Merge pull request #5 from fanick1/master

fix typo: quite->quote

.gitignore

@@ -0,0 +1,3 @@
+.test
+__pycache__
+adbfsc

Makefile

@@ -0,0 +1,42 @@
+# simple makefile for running tests for the adbfs plugin
+
+all: test_dir py3 flake8
+
+TEST_DIR='.test'
+PY3_VENV=$(TEST_DIR)/py3
+FL8_VENV=$(TEST_DIR)/flake8
+TST_EXISTS=$(shell [ -e $(TEST_DIR) ] && echo 1 || echo 0)
+PY3_EXISTS=$(shell [ -e $(PY3_VENV) ] && echo 1 || echo 0)
+FL8_EXISTS=$(shell [ -e $(FL8_VENV) ] && echo 1 || echo 0)
+
+py3: test_dir virtualenv3
+	.test/py3/bin/python test_adbfs.py
+
+flake8: test_dir virtualenv_flake8
+	.test/flake8/bin/flake8 adbfs test_adbfs.py
+
+ifeq ($(TST_EXISTS), 0)
+test_dir:
+	mkdir -p .test
+else
+test_dir:
+endif
+
+ifeq ($(PY3_EXISTS), 0)
+virtualenv3:
+	virtualenv -p python3 $(PY3_VENV)
+	$(PY3_VENV)/bin/pip install six
+else
+virtualenv3:
+endif
+
+ifeq ($(FL8_EXISTS), 0)
+virtualenv_flake8:
+	virtualenv -p python2 $(FL8_VENV)
+	$(FL8_VENV)/bin/pip install flake8
+else
+virtualenv_flake8:
+endif
+
+clean:
+	rm -fr $(TEST_DIR) __pycache__ adbfsc

README.rst

@@ -5,13 +5,15 @@ Midnight Commander adbfs external fs plugin
 This is Midnight Commander extfs plugin for browsing Android device through
 ``adb`` interface written in Python.
 
+
 Rquirements
 ===========
 
-* Python 2.7
+* Python 3.x (tested on 3.5.4, 3.6 and 3.7)
 * ``adb`` installed and in ``$PATH`` or provided via the config file
 * An Android device or emulator preferably rooted
-* Busybox installed and available in the path on the device
+* ``busybox`` (``toolbox``, ``toybox``) installed and available in the path on
+  the device
 
 Make sure, that issuing from command line:
 
@@ -23,6 +25,7 @@ Make sure, that issuing from command line:
 
 it should display files from root directory on the device.
 
+
 Features
 ========
 
@@ -34,12 +37,14 @@ Features
 * Symbolic links in lists are corrected to be relative to the file system
 * Symbolic links also point to the right target, skipping intermediate links
 
+
 Installation
 ============
 
 Copy adbfs into ``~/.local/share/mc/extfs.d/`` directory and make it executable
 if needed.
 
+
 Usage
 =====
 
@@ -47,13 +52,14 @@ To use it, just issue:
 
 .. code:: shell-session
 
-   cd adbfs://
+   $ cd adbfs://
 
 under MC - after some time you should see the files and directories on your
-device. For convenience you can add a bookmark (accessible under CTRL+\) for
+device. For convenience you can add a bookmark (accessible under CTRL+\\) for
 fast access. The time is depended on how many files and directories you have on
 your device and how fast it is :)
 
+
 Configuration
 =============
 
@@ -65,11 +71,12 @@ You can configure behaviour of this plugin using ``.ini`` file located under
 
    [adbfs]
    debug = false
-   skip_dirs = true
    dirs_to_skip = ["acct", "charger", "d", "dev", "proc", "sys"]
    suppress_colors = false
    root =
    adb_command = adb
+   adb_connect =
+   try_su = false
 
 where:
 
@@ -80,11 +87,52 @@ where:
   everything (slow!)
 * ``suppress_colors`` this option will make ``busybox`` not to display colors,
   helpful, if ``busybox ls`` is configured to display colors by default. Does
-  not affect ``toolbox``.
+  not affect ``toolbox`` or ``toybox``.
 * ``root`` root directory to read. Everything outside of that directory will be
   omitted. That would be the fastest way to access certain location on the
   device. Note, that ``dirs_to_skip`` still apply inside this directory.
-* ``adb_command`` = path to ``adb`` command.
+* ``adb_command`` absolute or relative path to ``adb`` command. ``~/`` or
+  environment variables are allowed.
+* ``adb_connect`` specifies if connection to specific device needs to be
+  performed before accessing shell. It is useful for *adb over network*
+  feature. Typical value here is a device IP address with optional port, which
+  defaults to 5555.
+* ``try_su`` specifies whether or not to try to detect if ``su`` command is
+  available and usable.
+
+
+Contribution
+============
+
+There is a ``Makefile`` in the top directory, which is basic helper for running
+the tests. Please use it, and adapt/add tests for provided fixes/functionality.
+The reason why `tox`_ wasn't used is, that there is no ``setup.py`` file, and
+it's difficult to install simple script, which isn't a python module (python
+interpreter will refuse to import module without ``.py`` extension).
+
+It requires GNU ``make`` program, and also ``virtualenv``. Using it is simple
+as running following command:
+
+.. code:: shell-session
+
+   $ make
+
+it will run `py3` and `flake8` jobs to check it against the code. For
+running tests against Python 3:
+
+.. code:: shell-session
+
+   $ make py3
+
+or flake 8:
+
+.. code:: shell-session
+
+   $ make flake8
+
+Exit status on any of those means that test fail. Appropriate message/traceback
+will also be visible.
+
 
 Limitations
 ===========
@@ -96,8 +144,11 @@ Limitations
 * The implementation is experimental and it's by now working with mine device;
   while it might not work with yours
 
+
 License
 =======
 
 This software is licensed under 3-clause BSD license. See LICENSE file for
 details.
+
+.. _tox: https://tox.readthedocs.io

adbfs

@@ -1,32 +1,43 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 """
 adbfs Virtual filesystem for Midnight Commander
 
 * Copyright (c) 2016, Roman Dobosz,
 * Published under 3-clause BSD-style license (see LICENSE file)
 """
-
-import ConfigParser
+import configparser
 import argparse
 from datetime import datetime
-import errno
 import json
 import os
 import re
 import subprocess
 import sys
+import shlex
 
-__version__ = 0.10
+__version__ = 0.14
 
 XDG_CONFIG_HOME = os.getenv('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
 
 
-class NoBoxFoundException(OSError):
+def check_output(command_list, stderr=None):
     """
-    Exception raised in case of not found either toolbox or busybox on remote
-    filesystem accessed via adb
+    For some reason, in py3 it was decided that command output should be bytes
+    instead of string. This little function will check if we have string or
+    bytes and in case of bytes it will convert it to string.
     """
-    pass
+    result = subprocess.check_output(command_list, stderr=stderr)
+    if not isinstance(result, str):
+        _result = []
+        for t in result.split(b'\n'):
+            if not t:
+                continue
+            try:
+                _result.append(t.decode('utf-8'))
+            except UnicodeDecodeError:
+                _result.append(t.decode('iso-8859-1'))
+        result = '\n'.join(_result) + '\n'
+    return result
 
 
 class Conf(object):
@@ -49,7 +60,17 @@ class Conf(object):
                                     r'(?P<size>\d+)?\s'
                                     r'(?P<date>\d{4}-\d{2}-\d{2}\s'
                                     r'\d{2}:\d{2})\s'
-                                    r'(?P<name>.*)'}}
+                                    r'(?P<name>.*)'},
+             'toybox': {'ls': 'toybox ls -anl',
+                        'rls':  'toybox ls -Ranl {}',
+                        'file_re': r'^(?P<perms>[-bcdlps][-rwxsStT]{9})\s+'
+                                   r'(?P<links>\d+)\s+'
+                                   r'(?P<uid>\d+)\s+'
+                                   r'(?P<gid>\d+)\s+'
+                                   r'(?P<size>\d+)?\s'
+                                   r'(?P<date>\d{4}-\d{2}-\d{2}\s'
+                                   r'\d{2}:\d{2})\s'
+                                   r'(?P<name>.*)'}}
 
     def __init__(self):
         self.box = None
@@ -58,8 +79,11 @@ class Conf(object):
         self.root = None
         self.suppress_colors = False
         self.adb_command = 'adb'
+        self.adb_connect = ''
+        self.try_su = False
 
         self.read()
+        self.connect()
         self.get_the_box()
 
     def get_the_box(self):
@@ -67,8 +91,7 @@ class Conf(object):
         cmd = [self.adb_command] + 'shell which'.split()
         try:
             with open(os.devnull, 'w') as fnull:
-                result = subprocess.check_output(cmd + ['busybox'],
-                                                 stderr=fnull)
+                result = check_output(cmd + ['busybox'], stderr=fnull)
             if 'busybox' in result:
                 self.box = Conf.boxes['busybox']
                 if self.suppress_colors:
@@ -82,8 +105,18 @@ class Conf(object):
 
         try:
             with open(os.devnull, 'w') as fnull:
-                result = subprocess.check_output(cmd + ['toolbox'],
-                                                 stderr=fnull)
+                result = check_output(cmd + ['toybox'], stderr=fnull)
+
+            if 'toybox' in result:
+                self.box = Conf.boxes['toybox']
+                Adb.file_re = re.compile(self.box['file_re'])
+                return
+        except subprocess.CalledProcessError:
+            pass
+
+        try:
+            with open(os.devnull, 'w') as fnull:
+                result = check_output(cmd + ['toolbox'], stderr=fnull)
 
             if 'toolbox' in result:
                 self.box = Conf.boxes['toolbox']
@@ -92,8 +125,56 @@ class Conf(object):
         except subprocess.CalledProcessError:
             pass
 
-        raise NoBoxFoundException(errno.ENOENT,
-                                  'There is no toolbox or busybox available')
+        sys.stderr.write('There is no toolbox or busybox available.\n')
+        sys.exit(1)
+
+    def get_attached_devices(self):
+        """Return a list of attached devices"""
+        cmd = [self.adb_command, 'devices']
+        devices = []
+
+        try:
+            with open(os.devnull, 'w') as fnull:
+                result = check_output(cmd, stderr=fnull)
+        except subprocess.CalledProcessError:
+            result = ''
+
+        for line in result.split('\n'):
+            if line.startswith('*'):
+                continue
+            if line.strip() == 'List of devices attached':
+                continue
+            if line.strip() == '':
+                continue
+            identifier, _ = line.split()
+            devices.append(identifier)
+
+        return devices
+
+    def connect(self):
+        """
+        If adb_connect is non empty string, perform connecting to specified
+        device over network using an address (or hostname).
+        """
+        if not self.adb_connect:
+            return
+
+        devices = self.get_attached_devices()
+
+        for device in devices:
+            if self.adb_connect in device:
+                return  # already connected, no need to reconnect
+
+        cmd = [self.adb_command, 'connect', self.adb_connect]
+        with open(os.devnull, 'w') as fnull:
+            result = check_output(cmd, stderr=fnull)
+        if result.split()[0] == 'connected':
+            subprocess.call([self.adb_command, 'wait-for-device'])
+            return
+
+        sys.stderr.write('Unable to connect to `%s\'. Is adb over network '
+                         'enabled on device?\n' % self.adb_connect)
+        sys.exit(2)
 
     def read(self):
         """
@@ -107,22 +188,24 @@ class Conf(object):
         if not os.path.exists(conf_fname):
             return
 
-        cfg = ConfigParser.SafeConfigParser()
+        cfg = configparser.ConfigParser()
         cfg_map = {'debug': (cfg.getboolean, 'debug'),
                    'dirs_to_skip': (cfg.get, 'dirs_to_skip'),
                    'suppress_colors': (cfg.get, 'suppress_colors'),
                    'root': (cfg.get, 'root'),
-                   'adb_command': (cfg.get, 'adb')}
+                   'adb_command': (cfg.get, 'adb_command'),
+                   'adb_connect': (cfg.get, 'adb_connect'),
+                   'try_su': (cfg.getboolean, 'try_su')}
         cfg.read(conf_fname)
 
         for key, (function, attr) in cfg_map.items():
             try:
                 setattr(self, attr, function('adbfs', key))
-            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            except (configparser.NoSectionError, configparser.NoOptionError):
                 pass
 
         if self.dirs_to_skip and isinstance(self.dirs_to_skip, str):
-            self.dirs_to_skip = json.loads(self.dirs_to_skip, encoding='ascii')
+            self.dirs_to_skip = json.loads(self.dirs_to_skip)
             self.dirs_to_skip = [x.encode('utf-8') for x in self.dirs_to_skip]
         else:
             self.dirs_to_skip = []
@@ -252,14 +335,26 @@ class Adb(object):
         self._links = {}
         self._got_root = False
 
-        self.__su_check()
+        if self.conf.try_su:
+            self.__su_check()
+
+    def _shell_cmd(self, with_root, *args):
+        cmd = [self.conf.adb_command, 'shell']
+
+        if with_root and self._got_root:
+            _args = [shlex.quote(x) for x in args]
+            cmd += ['su', '-c', shlex.quote(' '.join(_args))]
+        else:
+            cmd += args
+
+        return cmd
 
     def __su_check(self):
         """Check if we are able to get elevated privileges"""
-        cmd = [self.conf.adb_command] + 'shell su -c whoami'.split()
+        cmd = self._shell_cmd(False, 'su -c whoami')
         try:
             with open(os.devnull, 'w') as fnull:
-                result = subprocess.check_output(cmd, stderr=fnull)
+                result = check_output(cmd, stderr=fnull)
 
         except subprocess.CalledProcessError:
             return
@@ -306,16 +401,14 @@ class Adb(object):
 
     def _retrieve_single_dir_list(self, dir_):
         """Retrieve file list using adb"""
-        lscmd = self.conf.box['rls'].format(dir_)
-        if self._got_root:
-            lscmd = 'su -c "{}"'.format(lscmd)
-        command = [self.conf.adb_command, 'shell', lscmd]
+        lscmd = self.conf.box['rls'].format(shlex.quote(dir_))
+        command = self._shell_cmd(True, *shlex.split(lscmd))
 
         try:
             if self.conf.debug:
-                print 'executing', ' '.join(command)
+                print('executing', ' '.join(command))
 
-            lines = subprocess.check_output(command)
+            lines = check_output(command)
         except subprocess.CalledProcessError:
             sys.stderr.write('Cannot read directory. Is device connected?\n')
             return 1
@@ -366,21 +459,18 @@ class Adb(object):
         if not root:
             lscmd = self.conf.box['ls']
         else:
-            lscmd = self.conf.box['rls'].format(root.filepath)
+            lscmd = self.conf.box['rls'].format(shlex.quote(root.filepath))
 
-        if self._got_root:
-            lscmd = 'su -c "{}"'.format(lscmd)
-
-        command = [self.conf.adb_command, 'shell', lscmd]
+        command = self._shell_cmd(True, *shlex.split(lscmd))
 
         try:
             if self.conf.debug:
-                print 'executing', ' '.join(command)
+                print('executing', ' '.join(command))
 
-            lines = subprocess.check_output(command)
+            lines = check_output(command)
         except subprocess.CalledProcessError:
             sys.stderr.write('Cannot read directory. Is device connected?\n')
-            return 1
+            return 2
 
         current_dir = root.dirname if root else '/'
         for line in lines.split('\n'):
@@ -416,13 +506,13 @@ class Adb(object):
         """Not supported"""
         sys.stderr.write('Not supported - or maybe you are on compatible '
                          'architecture?\n')
-        return 1
+        return 3
 
     def list(self):
         """Output list contents directory"""
         if self.error:
             sys.stderr.write(self.error)
-            return 1
+            return 4
 
         if self.conf.root:
             self._retrieve_single_dir_list(self.conf.root)
@@ -437,7 +527,7 @@ class Adb(object):
         """Copy file form the device using adb."""
         if self.error:
             sys.stderr.write(self.error)
-            return 1
+            return 5
 
         cmd = [self.conf.adb_command, 'pull', src, dst]
         if self.conf.debug:
@@ -448,7 +538,7 @@ class Adb(object):
                 err = subprocess.call(cmd, stdout=fnull, stderr=fnull)
             except subprocess.CalledProcessError:
                 sys.stderr.write('Error executing adb shell')
-                return 1
+                return 6
 
         return err
 
@@ -456,7 +546,7 @@ class Adb(object):
         """Copy file to the device through adb."""
         if self.error:
             sys.stderr.write(self.error)
-            return 1
+            return 7
         if not dst.startswith('/'):
             dst = '/' + dst
 
@@ -469,69 +559,69 @@ class Adb(object):
                 err = subprocess.call(cmd, stdout=fnull, stderr=fnull)
             except subprocess.CalledProcessError:
                 sys.stderr.write('Error executing adb shell')
-                return 1
+                return 8
 
         if err != 0:
             sys.stderr.write('Cannot push the file, '
                              '%s, error %d' % (dst, err))
-            return 1
+            return 9
         return 0
 
     def rm(self, dst):
         """Remove file from device."""
         if self.error:
             sys.stderr.write(self.error)
-            return 1
+            return 10
 
-        cmd = [self.conf.adb_command, 'shell', 'rm', dst]
+        cmd = self._shell_cmd(False, 'rm %s' % shlex.quote(dst))
         try:
-            err = subprocess.check_output(cmd)
+            err = check_output(cmd).strip()
         except subprocess.CalledProcessError:
             sys.stderr.write('Error executing adb shell')
-            return 1
+            return 11
 
         if err != '':
             sys.stderr.write(err)
-            return 1
+            return 12
         return 0
 
     def rmdir(self, dst):
         """Remove directory from device."""
         if self.error:
             sys.stderr.write(self.error)
-            return 1
+            return 13
 
-        cmd = [self.conf.adb_command, 'shell', 'rm', '-r', dst]
+        cmd = self._shell_cmd(False, 'rm -r %s' % shlex.quote(dst))
         try:
-            err = subprocess.check_output(cmd)
+            err = check_output(cmd).strip()
         except subprocess.CalledProcessError:
             sys.stderr.write('Error executing adb shell')
-            return 1
+            return 14
 
         if err != '':
             sys.stderr.write(err)
-            return 1
+            return 15
         return 0
 
     def mkdir(self, dst):
         """Make directory on the device through adb."""
         if self.error:
             sys.stderr.write(self.error)
-            return 1
+            return 16
 
         if not dst.startswith('/'):
             dst = '/' + dst
 
-        cmd = [self.conf.adb_command, 'shell', 'mkdir', dst]
+        cmd = self._shell_cmd(False, 'mkdir %s' % shlex.quote(dst))
         try:
-            err = subprocess.check_output(cmd)
+            err = check_output(cmd).strip()
         except subprocess.CalledProcessError:
             sys.stderr.write('Error executing adb shell')
-            return 1
+            return 17
 
         if err != '':
             sys.stderr.write(err)
-            return 1
+            return 18
         return 0
 
 
@@ -590,7 +680,11 @@ def main():
 
     args = parser.parse_args()
 
-    return args.func(args)
+    try:
+        return args.func(args)
+    except AttributeError:
+        parser.print_help()
+        parser.exit()
 
 
 if __name__ == '__main__':

test_adbfs.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+import os
+from importlib.util import spec_from_loader, module_from_spec
+from importlib.machinery import SourceFileLoader
+import unittest
+from unittest import mock
+
+
+module_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'adbfs')
+spec = spec_from_loader("adbfs", SourceFileLoader("adbfs", module_path))
+adbfs = module_from_spec(spec)
+spec.loader.exec_module(adbfs)
+
+LISTING = '''\
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/Grüß Gott
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/\x80
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/Γεια σας
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/Здравствуйте
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/שָׁלוֹם
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/السَّلامُ عَلَيْكُمْ
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/გამარჯობა
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/こんにちは。
+-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 /storage/emulated/0/안녕하십니까
+'''  # noqa
+
+
+class TestCheckOutput(unittest.TestCase):
+
+    @mock.patch('subprocess.check_output')
+    def test_check_output(self, out):
+        """
+        As for Python2 (and its last version: 2.7), subprocess.check_output
+        always return string like objects, contrary to bytes - no conversion
+        to string is needed.
+        Python3 treats string as unicode objects, but subprocess.check_output
+        returns bytes object, which is equvalend for py2 string… annoying.
+        """
+        out.return_value = bytes(LISTING, 'utf-8')
+        result = adbfs.check_output(None)
+        self.assertEqual(result, LISTING)
+
+    @mock.patch('subprocess.check_output')
+    def test_check_output_py3_invalid_char(self, out):
+        """
+        Special case for py3. We have bytes with some weird character - like
+        some system write something with codepage, instead of utf8.
+        """
+        line = (b'-rw-rw----  1 0  1015  0 01/01/2010 22:11:01 '
+                b'/storage/emulated/0/\xe2\n')  # Latin 1 char â
+        out.return_value = bytes(line)
+        result = adbfs.check_output(None)
+        self.assertEqual(result, line.decode('iso-8859-1'))
+
+
+if __name__ == "__main__":
+    unittest.main()