Add tests to repository

tests/python/bcontroller.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is standalone Firefox Windows performance test.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Alice Nodelman <anodelman@mozilla.com> (original author)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+__author__ = 'anodelman@mozilla.com (Alice Nodelman)'
+
+import os
+import time
+import subprocess
+import threading
+import platform
+from ffprocess_linux import LinuxProcess
+from ffprocess_mac import MacProcess
+from ffprocess_win32 import Win32Process
+from utils import talosError
+import sys
+import getopt
+
+import stat
+
+
+if platform.system() == "Linux":
+    platform_type = 'linux_'
+    ffprocess = LinuxProcess()
+elif platform.system() in ("Windows", "Microsoft"):
+    import win32pdh
+    import win32api
+    import win32event
+    import win32con
+    platform_type = 'win_'
+    ffprocess = Win32Process()
+elif platform.system() == "Darwin":
+    platform_type = 'mac_'
+    ffprocess = MacProcess()
+
+class BrowserWaiter(threading.Thread):
+
+  def __init__(self, command, log, mod, deviceManager = None):
+     self.command = command
+     self.log = log
+     self.mod = mod
+     self.endTime = -1
+     self.returncode = -1
+     self.deviceManager = deviceManager
+     threading.Thread.__init__(self)
+     self.start()
+
+  def run(self):
+    if self.mod:
+      if (self.deviceManager.__class__.__name__ == "WinmoProcess"):
+        if (self.mod == "str(int(time.time()*1000))"):
+          self.command += self.deviceManager.getCurrentTime()
+      else:
+        self.command = self.command + eval(self.mod)
+
+    if (self.deviceManager.__class__.__name__ == "WinmoProcess"):
+      retVal = self.deviceManager.launchProcess(self.command, timeout=600)
+      if retVal <> None:
+        self.deviceManager.getFile(retVal, self.log)
+        self.returncode = 0
+      else:
+        self.returncode = 1
+    else:    #blocking call to system
+      self.returncode = os.system(self.command + " > " + self.log) 
+
+    self.endTime = int(time.time()*1000)
+
+  def hasTime(self):
+    return self.endTime > -1
+
+  def getTime(self):
+    return self.endTime
+
+  def getReturn(self):
+    def innerMethod(self):
+      pass
+    return self.returncode
+
+  def testMethod2(self):
+    class InnerClass:
+      def innerInnerMethod(self):
+        pass
+    return
+
+  class Test:
+    def testMethod(self):
+      pass
+
+class BrowserController:
+
+  def __init__(self, command, mod, name, child_process, 
+               timeout, log, host='', port=20701, root=''):
+    global ffprocess
+    self.command = command
+    self.mod = mod
+    self.process_name = name
+    self.child_process = child_process
+    self.browser_wait = timeout
+    self.log = log
+    self.timeout = 1200 #no output from the browser in 20 minutes = failure
+    self.host = host
+    self.port = port
+    self.root = root
+    if (host <> ''):
+      from ffprocess_winmo import WinmoProcess
+      platform_type = 'win_'
+      ffprocess = WinmoProcess(host, port, root)
+
+    self.ffprocess = ffprocess
+
+  def run(self):
+    self.bwaiter = BrowserWaiter(self.command, self.log, self.mod, self.ffprocess)
+    noise = 0
+    prev_size = 0
+    while not self.bwaiter.hasTime():
+      if noise > self.timeout: # check for frozen browser
+        try:
+          ffprocess.cleanupProcesses(self.process_name, self.child_process, self.browser_wait)
+        except talosError, te:
+          os.abort() #kill myself off because something horrible has happened
+        os.chmod(self.log, 0777)
+        results_file = open(self.log, "a")
+        results_file.write("\n__FAILbrowser frozen__FAIL\n")
+        results_file.close()
+        return
+      time.sleep(1)
+      try:
+        open(self.log, "r").close() #HACK FOR WINDOWS: refresh the file information
+        size = os.path.getsize(self.log)
+      except:
+        size = 0
+
+      if size > prev_size:
+        prev_size = size
+        noise = 0
+      else:
+        noise += 1
+
+    results_file = open(self.log, "a")
+    if self.bwaiter.getReturn() != 0:  #the browser shutdown, but not cleanly
+      results_file.write("\n__FAILbrowser non-zero return code (%d)__FAIL\n" % self.bwaiter.getReturn())
+      return
+    results_file.write("__startSecondTimestamp%d__endSecondTimestamp\n" % self.bwaiter.getTime())
+    results_file.close()
+    return
+
+
+def main(argv=None):
+
+   command = ""
+   name = "firefox" #default
+   child_process = "plugin-container" #default
+   timeout = ""
+   log = ""
+   mod = ""
+   host = ""
+   deviceRoot = ""
+   port = 20701
+
+   if argv is None:
+        argv = sys.argv
+   opts, args = getopt.getopt(argv[1:], "c:t:n:p:l:m:h:r:o", ["command=", "timeout=", "name=", "child_process=", "log=", "mod=", "host=", "deviceRoot=", "port="])
+
+   # option processing
+   for option, value in opts:
+     if option in ("-c", "--command"):
+       command = value
+     if option in ("-t", "--timeout"):
+       timeout = int(value)
+     if option in ("-n", "--name"):
+       name = value
+     if option in ("-p", "--child_process"):
+       child_process = value
+     if option in ("-l", "--log"):
+       log = value
+     if option in ("-m", "--mod"):
+       mod = value
+     if option in ("-h", "--host"):
+       host = value
+     if option in ("-r", "--deviceRoot"):
+       deviceRoot = value
+     if option in ("-o", "--port"):
+       port = value
+
+   if command and timeout and log:
+     bcontroller = BrowserController(command, mod, name, child_process, timeout, log, host, port, deviceRoot)
+     bcontroller.run()
+   else:
+     print "\nFAIL: no command\n"
+     sys.stdout.flush()
+
+   class mainClass:
+     def mainClassMethod(self):
+       pass
+     pass
+
+   def mainMethod(self):
+     class mainMethodClass:
+       pass
+     pass
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/python/functions.py

@@ -0,0 +1,697 @@
+"""
+Makefile functions.
+"""
+
+import parser, util
+import subprocess, os, logging
+from globrelative import glob
+from cStringIO import StringIO
+
+log = logging.getLogger('pymake.data')
+
+class Function(object):
+    """
+    An object that represents a function call. This class is always subclassed
+    with the following methods and attributes:
+
+    minargs = minimum # of arguments
+    maxargs = maximum # of arguments (0 means unlimited)
+
+    def resolve(self, makefile, variables, fd, setting)
+        Calls the function
+        calls fd.write() with strings
+    """
+
+    __slots__ = ('_arguments', 'loc')
+
+    def __init__(self, loc):
+        self._arguments = []
+        self.loc = loc
+        assert self.minargs > 0
+
+    def __getitem__(self, key):
+        return self._arguments[key]
+
+    def setup(self):
+        argc = len(self._arguments)
+
+        if argc < self.minargs:
+            raise data.DataError("Not enough arguments to function %s, requires %s" % (self.name, self.minargs), self.loc)
+
+        assert self.maxargs == 0 or argc <= self.maxargs, "Parser screwed up, gave us too many args"
+
+    def append(self, arg):
+        assert isinstance(arg, (data.Expansion, data.StringExpansion))
+        self._arguments.append(arg)
+
+    def __len__(self):
+        return len(self._arguments)
+
+class VariableRef(Function):
+    __slots__ = ('vname', 'loc')
+
+    def __init__(self, loc, vname):
+        self.loc = loc
+        assert isinstance(vname, (data.Expansion, data.StringExpansion))
+        self.vname = vname
+        
+    def setup(self):
+        assert False, "Shouldn't get here"
+
+    def resolve(self, makefile, variables, fd, setting):
+        vname = self.vname.resolvestr(makefile, variables, setting)
+        if vname in setting:
+            raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)
+
+        flavor, source, value = variables.get(vname)
+        if value is None:
+            log.debug("%s: variable '%s' was not set" % (self.loc, vname))
+            return
+
+        value.resolve(makefile, variables, fd, setting + [vname])
+
+class SubstitutionRef(Function):
+    """$(VARNAME:.c=.o) and $(VARNAME:%.c=%.o)"""
+
+    __slots__ = ('loc', 'vname', 'substfrom', 'substto')
+
+    def __init__(self, loc, varname, substfrom, substto):
+        self.loc = loc
+        self.vname = varname
+        self.substfrom = substfrom
+        self.substto = substto
+
+    def setup(self):
+        assert False, "Shouldn't get here"
+
+    def resolve(self, makefile, variables, fd, setting):
+        vname = self.vname.resolvestr(makefile, variables, setting)
+        if vname in setting:
+            raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)
+
+        substfrom = self.substfrom.resolvestr(makefile, variables, setting)
+        substto = self.substto.resolvestr(makefile, variables, setting)
+
+        flavor, source, value = variables.get(vname)
+        if value is None:
+            log.debug("%s: variable '%s' was not set" % (self.loc, vname))
+            return
+
+        f = data.Pattern(substfrom)
+        if not f.ispattern():
+            f = data.Pattern('%' + substfrom)
+            substto = '%' + substto
+
+        fd.write(' '.join([f.subst(substto, word, False)
+                           for word in value.resolvesplit(makefile, variables, setting + [vname])]))
+
+class SubstFunction(Function):
+    name = 'subst'
+    minargs = 3
+    maxargs = 3
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        s = self._arguments[0].resolvestr(makefile, variables, setting)
+        r = self._arguments[1].resolvestr(makefile, variables, setting)
+        d = self._arguments[2].resolvestr(makefile, variables, setting)
+        fd.write(d.replace(s, r))
+
+class PatSubstFunction(Function):
+    name = 'patsubst'
+    minargs = 3
+    maxargs = 3
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        s = self._arguments[0].resolvestr(makefile, variables, setting)
+        r = self._arguments[1].resolvestr(makefile, variables, setting)
+
+        p = data.Pattern(s)
+        fd.write(' '.join([p.subst(r, word, False)
+                           for word in self._arguments[2].resolvesplit(makefile, variables, setting)]))
+
+class StripFunction(Function):
+    name = 'strip'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        util.joiniter(fd, self._arguments[0].resolvesplit(makefile, variables, setting))
+
+class FindstringFunction(Function):
+    name = 'findstring'
+    minargs = 2
+    maxargs = 2
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        s = self._arguments[0].resolvestr(makefile, variables, setting)
+        r = self._arguments[1].resolvestr(makefile, variables, setting)
+        if r.find(s) == -1:
+            return
+        fd.write(s)
+
+class FilterFunction(Function):
+    name = 'filter'
+    minargs = 2
+    maxargs = 2
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        plist = [data.Pattern(p)
+                 for p in self._arguments[0].resolvesplit(makefile, variables, setting)]
+
+        fd.write(' '.join([w for w in self._arguments[1].resolvesplit(makefile, variables, setting)
+                           if util.any((p.match(w) for p in plist))]))
+
+class FilteroutFunction(Function):
+    name = 'filter-out'
+    minargs = 2
+    maxargs = 2
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        plist = [data.Pattern(p)
+                 for p in self._arguments[0].resolvesplit(makefile, variables, setting)]
+
+        fd.write(' '.join([w for w in self._arguments[1].resolvesplit(makefile, variables, setting)
+                           if not util.any((p.match(w) for p in plist))]))
+
+class SortFunction(Function):
+    name = 'sort'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        d = list(self._arguments[0].resolvesplit(makefile, variables, setting))
+        d.sort()
+        util.joiniter(fd, d)
+
+class WordFunction(Function):
+    name = 'word'
+    minargs = 2
+    maxargs = 2
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        n = self._arguments[0].resolvestr(makefile, variables, setting)
+        # TODO: provide better error if this doesn't convert
+        n = int(n)
+        words = list(self._arguments[1].resolvesplit(makefile, variables, setting))
+        if n < 1 or n > len(words):
+            return
+        fd.write(words[n - 1])
+
+class WordlistFunction(Function):
+    name = 'wordlist'
+    minargs = 3
+    maxargs = 3
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        nfrom = self._arguments[0].resolvestr(makefile, variables, setting)
+        nto = self._arguments[1].resolvestr(makefile, variables, setting)
+        # TODO: provide better errors if this doesn't convert
+        nfrom = int(nfrom)
+        nto = int(nto)
+
+        words = list(self._arguments[2].resolvesplit(makefile, variables, setting))
+
+        if nfrom < 1:
+            nfrom = 1
+        if nto < 1:
+            nto = 1
+
+        util.joiniter(fd, words[nfrom - 1:nto])
+
+class WordsFunction(Function):
+    name = 'words'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        fd.write(str(len(self._arguments[0].resolvesplit(makefile, variables, setting))))
+
+class FirstWordFunction(Function):
+    name = 'firstword'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        l = self._arguments[0].resolvesplit(makefile, variables, setting)
+        if len(l):
+            fd.write(l[0])
+
+class LastWordFunction(Function):
+    name = 'lastword'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        l = self._arguments[0].resolvesplit(makefile, variables, setting)
+        if len(l):
+            fd.write(l[-1])
+
+def pathsplit(path, default='./'):
+    """
+    Splits a path into dirpart, filepart on the last slash. If there is no slash, dirpart
+    is ./
+    """
+    dir, slash, file = util.strrpartition(path, '/')
+    if dir == '':
+        return default, file
+
+    return dir + slash, file
+
+class DirFunction(Function):
+    name = 'dir'
+    minargs = 1
+    maxargs = 1
+
+    def resolve(self, makefile, variables, fd, setting):
+        fd.write(' '.join([pathsplit(path)[0]
+                           for path in self._arguments[0].resolvesplit(makefile, variables, setting)]))
+
+class NotDirFunction(Function):
+    name = 'notdir'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        fd.write(' '.join([pathsplit(path)[1]
+                           for path in self._arguments[0].resolvesplit(makefile, variables, setting)]))
+
+class SuffixFunction(Function):
+    name = 'suffix'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    @staticmethod
+    def suffixes(words):
+        for w in words:
+            dir, file = pathsplit(w)
+            base, dot, suffix = util.strrpartition(file, '.')
+            if base != '':
+                yield dot + suffix
+
+    def resolve(self, makefile, variables, fd, setting):
+        util.joiniter(fd, self.suffixes(self._arguments[0].resolvesplit(makefile, variables, setting)))
+
+class BasenameFunction(Function):
+    name = 'basename'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    @staticmethod
+    def basenames(words):
+        for w in words:
+            dir, file = pathsplit(w, '')
+            base, dot, suffix = util.strrpartition(file, '.')
+            if dot == '':
+                base = suffix
+
+            yield dir + base
+
+    def resolve(self, makefile, variables, fd, setting):
+        util.joiniter(fd, self.basenames(self._arguments[0].resolvesplit(makefile, variables, setting)))
+
+class AddSuffixFunction(Function):
+    name = 'addprefix'
+    minargs = 2
+    maxargs = 2
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        suffix = self._arguments[0].resolvestr(makefile, variables, setting)
+
+        fd.write(' '.join([w + suffix for w in self._arguments[1].resolvesplit(makefile, variables, setting)]))
+
+class AddPrefixFunction(Function):
+    name = 'addsuffix'
+    minargs = 2
+    maxargs = 2
+
+    def resolve(self, makefile, variables, fd, setting):
+        prefix = self._arguments[0].resolvestr(makefile, variables, setting)
+
+        fd.write(' '.join([prefix + w for w in self._arguments[1].resolvesplit(makefile, variables, setting)]))
+
+class JoinFunction(Function):
+    name = 'join'
+    minargs = 2
+    maxargs = 2
+
+    __slots__ = Function.__slots__
+
+    @staticmethod
+    def iterjoin(l1, l2):
+        for i in xrange(0, max(len(l1), len(l2))):
+            i1 = i < len(l1) and l1[i] or ''
+            i2 = i < len(l2) and l2[i] or ''
+            yield i1 + i2
+
+    def resolve(self, makefile, variables, fd, setting):
+        list1 = list(self._arguments[0].resolvesplit(makefile, variables, setting))
+        list2 = list(self._arguments[1].resolvesplit(makefile, variables, setting))
+
+        util.joiniter(fd, self.iterjoin(list1, list2))
+
+class WildcardFunction(Function):
+    name = 'wildcard'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        patterns = self._arguments[0].resolvesplit(makefile, variables, setting)
+
+        fd.write(' '.join([x.replace('\\','/')
+                           for p in patterns
+                           for x in glob(makefile.workdir, p)]))
+
+    __slots__ = Function.__slots__
+
+class RealpathFunction(Function):
+    name = 'realpath'
+    minargs = 1
+    maxargs = 1
+
+    def resolve(self, makefile, variables, fd, setting):
+        fd.write(' '.join([os.path.realpath(os.path.join(makefile.workdir, path)).replace('\\', '/')
+                           for path in self._arguments[0].resolvesplit(makefile, variables, setting)]))
+
+class AbspathFunction(Function):
+    name = 'abspath'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        assert os.path.isabs(makefile.workdir)
+        fd.write(' '.join([util.normaljoin(makefile.workdir, path).replace('\\', '/')
+                           for path in self._arguments[0].resolvesplit(makefile, variables, setting)]))
+
+class IfFunction(Function):
+    name = 'if'
+    minargs = 1
+    maxargs = 3
+
+    __slots__ = Function.__slots__
+
+    def setup(self):
+        Function.setup(self)
+        self._arguments[0].lstrip()
+        self._arguments[0].rstrip()
+
+    def resolve(self, makefile, variables, fd, setting):
+        condition = self._arguments[0].resolvestr(makefile, variables, setting)
+
+        if len(condition):
+            self._arguments[1].resolve(makefile, variables, fd, setting)
+        elif len(self._arguments) > 2:
+            return self._arguments[2].resolve(makefile, variables, fd, setting)
+
+class OrFunction(Function):
+    name = 'or'
+    minargs = 1
+    maxargs = 0
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        for arg in self._arguments:
+            r = arg.resolvestr(makefile, variables, setting)
+            if r != '':
+                fd.write(r)
+                return
+
+class AndFunction(Function):
+    name = 'and'
+    minargs = 1
+    maxargs = 0
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        r = ''
+
+        for arg in self._arguments:
+            r = arg.resolvestr(makefile, variables, setting)
+            if r == '':
+                return
+
+        fd.write(r)
+
+class ForEachFunction(Function):
+    name = 'foreach'
+    minargs = 3
+    maxargs = 3
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        vname = self._arguments[0].resolvestr(makefile, variables, setting)
+        e = self._arguments[2]
+
+        v = data.Variables(parent=variables)
+        firstword = True
+
+        for w in self._arguments[1].resolvesplit(makefile, variables, setting):
+            if firstword:
+                firstword = False
+            else:
+                fd.write(' ')
+
+            v.set(vname, data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, w)
+            e.resolve(makefile, v, fd, setting)
+
+class CallFunction(Function):
+    name = 'call'
+    minargs = 1
+    maxargs = 0
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        vname = self._arguments[0].resolvestr(makefile, variables, setting)
+        if vname in setting:
+            raise data.DataError("Recursively setting variable '%s'" % (vname,))
+
+        v = data.Variables(parent=variables)
+        v.set('0', data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, vname)
+        for i in xrange(1, len(self._arguments)):
+            param = self._arguments[i].resolvestr(makefile, variables, setting)
+            v.set(str(i), data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, param)
+
+        flavor, source, e = variables.get(vname)
+
+        if e is None:
+            return
+
+        if flavor == data.Variables.FLAVOR_SIMPLE:
+            log.warning("%s: calling variable '%s' which is simply-expanded" % (self.loc, vname))
+
+        # but we'll do it anyway
+        e.resolve(makefile, v, fd, setting + [vname])
+
+class ValueFunction(Function):
+    name = 'value'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        varname = self._arguments[0].resolvestr(makefile, variables, setting)
+
+        flavor, source, value = variables.get(varname, expand=False)
+        if value is not None:
+            fd.write(value)
+
+class EvalFunction(Function):
+    name = 'eval'
+    minargs = 1
+    maxargs = 1
+
+    def resolve(self, makefile, variables, fd, setting):
+        if makefile.parsingfinished:
+            # GNU make allows variables to be set by recursive expansion during
+            # command execution. This seems really dumb to me, so I don't!
+            raise data.DataError("$(eval) not allowed via recursive expansion after parsing is finished", self.loc)
+
+        stmts = parser.parsestring(self._arguments[0].resolvestr(makefile, variables, setting),
+                                   'evaluation from %s' % self.loc)
+        stmts.execute(makefile)
+
+class OriginFunction(Function):
+    name = 'origin'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        vname = self._arguments[0].resolvestr(makefile, variables, setting)
+
+        flavor, source, value = variables.get(vname)
+        if source is None:
+            r = 'undefined'
+        elif source == data.Variables.SOURCE_OVERRIDE:
+            r = 'override'
+
+        elif source == data.Variables.SOURCE_MAKEFILE:
+            r = 'file'
+        elif source == data.Variables.SOURCE_ENVIRONMENT:
+            r = 'environment'
+        elif source == data.Variables.SOURCE_COMMANDLINE:
+            r = 'command line'
+        elif source == data.Variables.SOURCE_AUTOMATIC:
+            r = 'automatic'
+        elif source == data.Variables.SOURCE_IMPLICIT:
+            r = 'default'
+
+        fd.write(r)
+
+class FlavorFunction(Function):
+    name = 'flavor'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        varname = self._arguments[0].resolvestr(makefile, variables, setting)
+        
+        flavor, source, value = variables.get(varname)
+        if flavor is None:
+            r = 'undefined'
+        elif flavor == data.Variables.FLAVOR_RECURSIVE:
+            r = 'recursive'
+        elif flavor == data.Variables.FLAVOR_SIMPLE:
+            r = 'simple'
+        fd.write(r)
+
+class ShellFunction(Function):
+    name = 'shell'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        #TODO: call this once up-front somewhere and save the result?
+        shell, msys = util.checkmsyscompat()
+        cline = self._arguments[0].resolvestr(makefile, variables, setting)
+
+        log.debug("%s: running shell command '%s'" % (self.loc, cline))
+        if msys:
+            cline = [shell, "-c", cline]
+        p = subprocess.Popen(cline, shell=not msys, stdout=subprocess.PIPE, cwd=makefile.workdir)
+        stdout, stderr = p.communicate()
+
+        stdout = stdout.replace('\r\n', '\n')
+        if stdout.endswith('\n'):
+            stdout = stdout[:-1]
+        stdout = stdout.replace('\n', ' ')
+
+        fd.write(stdout)
+
+class ErrorFunction(Function):
+    name = 'error'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        v = self._arguments[0].resolvestr(makefile, variables, setting)
+        raise data.DataError(v, self.loc)
+
+class WarningFunction(Function):
+    name = 'warning'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        v = self._arguments[0].resolvestr(makefile, variables, setting)
+        log.warning(v)
+
+class InfoFunction(Function):
+    name = 'info'
+    minargs = 1
+    maxargs = 1
+
+    __slots__ = Function.__slots__
+
+    def resolve(self, makefile, variables, fd, setting):
+        v = self._arguments[0].resolvestr(makefile, variables, setting)
+        print v
+
+functionmap = {
+    'subst': SubstFunction,
+    'patsubst': PatSubstFunction,
+    'strip': StripFunction,
+    'findstring': FindstringFunction,
+    'filter': FilterFunction,
+    'filter-out': FilteroutFunction,
+    'sort': SortFunction,
+    'word': WordFunction,
+    'wordlist': WordlistFunction,
+    'words': WordsFunction,
+    'firstword': FirstWordFunction,
+    'lastword': LastWordFunction,
+    'dir': DirFunction,
+    'notdir': NotDirFunction,
+    'suffix': SuffixFunction,
+    'basename': BasenameFunction,
+    'addsuffix': AddSuffixFunction,
+    'addprefix': AddPrefixFunction,
+    'join': JoinFunction,
+    'wildcard': WildcardFunction,
+    'realpath': RealpathFunction,
+    'abspath': AbspathFunction,
+    'if': IfFunction,
+    'or': OrFunction,
+    'and': AndFunction,
+    'foreach': ForEachFunction,
+    'call': CallFunction,
+    'value': ValueFunction,
+    'eval': EvalFunction,
+    'origin': OriginFunction,
+    'flavor': FlavorFunction,
+    'shell': ShellFunction,
+    'error': ErrorFunction,
+    'warning': WarningFunction,
+    'info': InfoFunction,
+}
+
+import data

tests/python/mlstring.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+class ClassName(object):
+    """docstring for ClassName"""
+    def __init__(self, arg):
+        super(ClassName, self).__init__()
+        self.arg = arg
+    def function1(self):
+        foo = """
+foostring
+    """
+    def function2(self):
+        pass

tests/python/models.py

@@ -0,0 +1,50 @@
+from django.db import models
+from django.contrib.auth.models import User
+from django.contrib import admin
+from string import join
+from settings import MEDIA_ROOT
+
+class Forum(models.Model):
+    title = models.CharField(max_length=60)
+    def __unicode__(self):
+        return self.title
+
+class Thread(models.Model):
+    title = models.CharField(max_length=60)
+    created = models.DateTimeField(auto_now_add=True)
+    creator = models.ForeignKey(User, blank=True, null=True)
+    forum = models.ForeignKey(Forum)
+
+    def __unicode__(self):
+        return unicode(self.creator) + " - " + self.title
+
+class Post(models.Model):
+    title = models.CharField(max_length=60)
+    created = models.DateTimeField(auto_now_add=True)
+    creator = models.ForeignKey(User, blank=True, null=True)
+    thread = models.ForeignKey(Thread)
+    body = models.TextField(max_length=10000)
+
+    def __unicode__(self):
+        return u"%s - %s - %s" % (self.creator, self.thread, self.title)
+
+    def short(self):
+        return u"%s - %s\n%s" % (self.creator, self.title, self.created.strftime("%b %d, %I:%M %p"))
+    short.allow_tags = True
+
+### Admin
+
+class ForumAdmin(admin.ModelAdmin):
+    pass
+
+class ThreadAdmin(admin.ModelAdmin):
+    list_display = ["title", "forum", "creator", "created"]
+    list_filter = ["forum", "creator"]
+
+class PostAdmin(admin.ModelAdmin):
+    search_fields = ["title", "creator"]
+    list_display = ["title", "thread", "creator", "created"]
+
+admin.site.register(Forum, ForumAdmin)
+admin.site.register(Thread, ThreadAdmin)
+admin.site.register(Post, PostAdmin)

tests/python/multiline.py

@@ -0,0 +1,3 @@
+def foo(a, b, \
+        c, d):
+    pass

tests/python/tabindent.py

@@ -0,0 +1,3 @@
+def find_heading(self, position=0, direction=Direction.FORWARD, \
+		heading=Heading, connect_with_document=True):
+	pass

tests/python/test.py

@@ -0,0 +1,23 @@
+import foo
+import bar
+
+defaultdict(lambda: 0)
+classofhello = 0
+
+class test(foo, bar):
+    class Inner:
+#        pass
+        def foo(self):
+            print "Inner"
+#    pass
+
+def test():
+    class Inner2:
+        def bar(self):
+            print "Inner2"
+#    t = Inner2()
+#    pass
+
+#print "Test"
+#r = test.Inner2()
+#test()

tests/python/test2.py

@@ -0,0 +1,3 @@
+from test import *
+
+r = test.Inner()

tests/python/vis.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python2.6
+
+from __future__ import division
+
+import gtk
+
+import numpy as np
+import matplotlib
+matplotlib.use('GTKAgg')
+import matplotlib.cm as cm
+import matplotlib.pyplot as plt
+from matplotlib.widgets import Button, RadioButtons
+from matplotlib.backends.backend_gtkagg import FigureCanvasGTKAgg as FigureCanvas
+
+win = gtk.Window()
+win.connect("destroy", gtk.main_quit)
+win.set_default_size(600,600)
+win.set_title("Resource Visualisation")
+
+fig = plt.figure(figsize=(8,8))
+ax = fig.add_axes([0.1, 0.2, 0.8, 0.75], projection='polar')
+
+rax = fig.add_axes([0.7, 0.05, 0.2, 0.05])
+rax.grid(False)
+rax.set_xticks([])
+rax.set_yticks([])
+
+moax = fig.add_axes([0.1, 0.02, 0.25, 0.1])
+moax.grid(False)
+moax.set_xticks([])
+moax.set_yticks([])
+
+logax = fig.add_axes([0.4, 0.02, 0.25, 0.1])
+logax.grid(False)
+logax.set_xticks([])
+logax.set_yticks([])
+
+canvas = FigureCanvas(fig)
+win.add(canvas)
+
+class ResVis(object):
+    def __init__(self):
+        self.dirdata = self.load_data('dirs.dat')
+        self.userdata = self.load_data('users.dat')
+
+        self.mode = 'dir'
+        self.log = True
+        self.draw_dir('root')
+
+    def load_data(self, filename):
+        data = {}
+        for line in open(filename, 'r').readlines():
+            entry = line.split(None, 3)
+
+            if len(entry) > 3:      # current entry has subentries
+                entry[3] = entry[3].split()
+                entry[3].sort(key=str.lower)
+            else:
+                entry.append([])
+
+            data[entry[0]] = [entry[0], float(entry[1]), float(entry[2]),
+                              entry[3]]
+        return data
+
+    def load_dir(self, dirname):
+        curdirdata = []
+        for d in self.dirdata[dirname][3]:
+            curdirdata.append(self.dirdata[d])
+        return curdirdata
+
+    def load_user(self, username):
+        curdata = []
+        for u in self.userdata[username][3]:
+            curdata.append(self.userdata[u])
+        return curdata
+
+    def draw_dir(self, dirname='root'):
+        self.curdir = dirname
+        self.reset_ax()
+        ax.set_title('Directory size')
+        ax.set_xlabel(dirname, weight='bold')
+
+        curdir = self.load_dir(dirname)
+        self.draw_data(curdir)
+
+    def draw_user(self, username='root'):
+        self.curuser = username
+        self.reset_ax()
+        ax.set_title('Resource usage')
+        ax.set_xlabel(username, weight='bold')
+
+        user = self.load_user(username)
+        self.draw_data(user)
+
+    def reset_ax(self):
+        ax.cla()
+
+        #ax.axis('off')
+        #ax.set_axis_off()
+        #ax.set_yscale('log')
+        ax.grid(False)
+        ax.set_xticks([]) # edge
+        ax.set_yticks([]) # radius
+        #ax.set_xlabel('Size')
+        #ax.set_ylabel('Number of files')
+
+    def draw_data(self, data):
+        totalsize = sum(zip(*data)[1])     # get sum of subentry sizes
+        unit = 1.5 * np.pi / totalsize
+
+        angle = 0.5 * np.pi
+        if self.log:
+            maxy = max(map(np.log2, zip(*data)[2]))
+        else:
+            maxy = max(zip(*data)[2])
+        for d in data:
+            relangle = unit * d[1]
+
+            if len(d[3]) > 0 or self.mode == 'user':
+                # scale colours since the legend occupies a quarter
+                scaledangle = (angle - 0.5*np.pi) * (2 / 1.5)
+                colour = cm.hsv(scaledangle/(2*np.pi))
+            else:
+#                colour = cm.Greys(scaledangle/(2*np.pi))
+                colour = "#999999"
+
+            if self.log:
+                # take logarithm to accomodate for big differences
+                y = np.log2(d[2])
+            else:
+                y = d[2]
+
+            bar = ax.bar(angle, y, width=relangle, bottom=maxy*0.2,
+                         color=colour, label=d[0],
+                         picker=True)
+            angle += relangle
+
+            if self.mode == 'dir':
+                desc = '{0}\n{1}G'.format(d[0], d[1])
+            elif self.mode == 'user':
+                desc = '{0}\n{1}%'.format(d[0], d[1])
+            self.draw_desc(bar[0], d, desc)
+
+        self.draw_legend(maxy)
+
+        fig.canvas.draw()
+
+    def draw_desc(self, bar, data, text):
+        # show description in center of bar
+        bbox = bar.get_bbox()
+        x = bbox.xmin + (bbox.xmax - bbox.xmin) / 2
+        y = bbox.ymin + (bbox.ymax - bbox.ymin) / 2
+        ax.text(x, y, text, horizontalalignment='center',
+                verticalalignment='center', weight='bold')
+
+    def draw_legend(self, maxy):
+        ax.annotate('', xy=(0, maxy*0.3), xytext=(0.5*np.pi, maxy*0.3),
+                    arrowprops=dict(
+                        arrowstyle='<->',
+                        connectionstyle='angle3,angleA=0,angleB=-90',
+                        linewidth=3))
+        ax.annotate('', xy=(0.04*np.pi, maxy*0.35), xytext=(0.04*np.pi, maxy),
+                    arrowprops=dict(
+                        arrowstyle='<->',
+                        connectionstyle='arc3',
+                        linewidth=3))
+
+        if self.mode == 'dir':
+            xtext = 'Size'
+            ytext = 'Number of files'
+        elif self.mode == 'user':
+            xtext = 'Processor usage'
+            ytext = 'Memory usage'
+
+        ax.text(0.3*np.pi, maxy*0.35, xtext, weight='normal')
+        ax.text(0.06*np.pi, maxy*0.4, ytext, weight='normal', rotation=8)
+
+        if self.mode == 'dir':
+            ax.text(0.3*np.pi, maxy*0.6, 'Grey dirs do not\nhave subdirs.')
+
+    def on_pick(self, event):
+        clicked = event.artist.get_label()
+
+        if self.mode == 'dir' and len(self.dirdata[clicked][3]) > 0:
+                self.draw_dir(clicked)
+        elif self.mode == 'user' and len(self.userdata[clicked][3]) > 0:
+                self.draw_user(clicked)
+
+    def on_rootclick(self, event):
+        if self.mode == 'dir':
+            self.draw_dir('root')
+        elif self.mode == 'user':
+            self.draw_user('root')
+
+    def on_modeclick(self, mode):
+        if mode == 'Directory size':
+            self.mode = 'dir'
+            self.draw_dir('root')
+        elif mode == 'Resource usage':
+            self.mode = 'user'
+            self.draw_user('root')
+
+    def on_logclick(self, mode):
+        self.log = mode == 'Logarithmic'
+        if self.mode == 'dir':
+            self.draw_dir(self.curdir)
+        if self.mode == 'user':
+            self.draw_user(self.curuser)
+
+vis = ResVis()
+
+root = Button(rax, 'Home')
+root.on_clicked(vis.on_rootclick)
+mode = RadioButtons(moax, ('Directory size', 'Resource usage'))
+mode.on_clicked(vis.on_modeclick)
+log = RadioButtons(logax, ('Logarithmic', 'Linear'))
+log.on_clicked(vis.on_logclick)
+
+fig.canvas.mpl_connect('pick_event', vis.on_pick)
+
+#plt.show()
+
+win.show_all()
+gtk.main()
+