root/branches/mk/cheesecake/cheesecake_index.py

#!/usr/bin/env python
"""
Cheesecake: How tasty is your code?

The idea of the Cheesecake project is to rank Python packages
based on various empiric "kwalitee" factors, such as:

        * whether the package can be downloaded
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
        * existence of certain files such as README, INSTALL, LICENSE, setup.py etc.
        * existence of certain directories such as doc, test, demo, examples
        * percentage of modules/functions/classes/methods with docstrings
        * percentage of functions/methods that are unit tested
        * average pylint score for all non-test and non-demo modules
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
"""

import os, sys, re, shutil
import tempfile
from optparse import OptionParser
from urllib import urlretrieve
from urlparse import urlparse
from math import ceil

from util import pad_with_dots, pad_left_spaces, pad_right_spaces, pad_msg, pad_line
from util import run_cmd, command_successful
from util import unzip_package, untar_package
from util import mkdirs
from util import StdoutRedirector
import logger
from config import get_pkg_config
from codeparser import CodeParser

__docformat__ = 'reStructuredText en'


################################################################################
## Helpers.
################################################################################

def isiterable(obj):
    """Check whether object is iterable.

    >>> isiterable([1,2,3])
    True
    >>> isiterable("string")
    True
    >>> isiterable(object)
    False
    """
    return hasattr(obj, '__iter__') or isinstance(obj, basestring)

def has_extension(filename, ext):
    """Check if filename has given extension.

    >>> has_extension("foobar.py", ".py")
    True
    >>> has_extension("foo.bar.py", ".py")
    True
    >>> has_extension("foobar.pyc", ".py")
    False
    """
    return os.path.splitext(filename)[1] == ext

def discover_file_type(filename):
    """Discover type of a file according to its name and its parent directory.

    Currently supported file types:
        * pyc
        * pyo
        * module: .py files of an application
        * demo: .py files for documentation/demonstration purposes
        * test: .py files used for testing
        * special: .py file for special purposes

    :Note: This function only check file's name, and doesn't touch the
           filesystem. If you have to, check if file exists by yourself.

    >>> discover_file_type('module.py')
    'module'
    >>> discover_file_type('./setup.py')
    'special'
    >>> discover_file_type('some/directory/junk.pyc')
    'pyc'
    """
    dirs = filename.split(os.path.sep)
    dirs, filename = dirs[:-1], dirs[-1]

    if filename in ["setup.py", "ez_setup.py", "__pkginfo__.py"]:
        return 'special'

    if has_extension(filename, ".pyc"):
        return 'pyc'
    if has_extension(filename, ".pyo"):
        return 'pyo'
    if has_extension(filename, ".py"):
        for dir in dirs:
            if dir in ['test', 'tests']:
                return 'test'
            elif dir in ['docs', 'demo', 'example']:
                return 'demo'
        return 'module'

def get_files_of_type(file_list, file_type):
    """Return files from `file_list` that match given `file_type`.

    >>> file_list = ['test/test_foo.py', 'setup.py', 'README', 'test/test_bar.py']
    >>> get_files_of_type(file_list, 'test')
    ['test/test_foo.py', 'test/test_bar.py']
    """
    return filter(lambda x: discover_file_type(x) == file_type, file_list)

def get_method_arguments(method):
    """Return tuple of arguments for given method, excluding self.

    >>> class Class:
    ...     def method(s, arg1, arg2, other_arg):
    ...         pass
    >>> get_method_arguments(Class.method)
    ('arg1', 'arg2', 'other_arg')
    """
    return method.func_code.co_varnames[1:method.func_code.co_argcount]

def get_attributes(obj, names):
    """Return attributes dictionary with keys from `names`.

    Object is queried for each attribute name, if it doesn't have this
    attribute, default value None will be returned.

    >>> class Class:
    ...     pass
    >>> obj = Class()
    >>> obj.attr = True
    >>> obj.value = 13
    >>> obj.string = "Hello"

    >>> d = get_attributes(obj, ['attr', 'string', 'other'])
    >>> d == {'attr': True, 'string': "Hello", 'other': None}
    True
    """
    attrs = {}

    for name in names:
        attrs[name] = getattr(obj, name, None)

    return attrs

def camel2underscore(name):
    """Convert name from CamelCase to underscore_name.

    >>> camel2underscore('CamelCase')
    'camel_case'
    >>> camel2underscore('already_underscore_name')
    'already_underscore_name'
    >>> camel2underscore('BigHTMLClass')
    'big_html_class'
    >>> camel2underscore('')
    ''
    """
    if name and name[0].upper:
        name = name[0].lower() + name[1:]

    def capitalize(match):
        string = match.group(1).lower().capitalize()
        return string[:-1] + string[-1].upper()

    def underscore(match):
        return '_' + match.group(1).lower()

    name = re.sub(r'([A-Z]+)', capitalize, name)
    return re.sub(r'([A-Z])', underscore, name)

def index_class_to_name(clsname):
    """Covert index class name to index name.

    >>> index_class_to_name("IndexDownload")
    'download'
    >>> index_class_to_name("IndexUnitTests")
    'unit_tests'
    >>> index_class_to_name("IndexPyPIDownload")
    'py_pi_download'
    """
    return camel2underscore(clsname.replace('Index', '', 1))

def is_empty(path):
    """Returns True if file or directory pointed by `path` is empty.
    """
    if os.path.isfile(path) and os.path.getsize(path) == 0:
        return True
    if os.path.isdir(path) and os.listdir(path) == []:
        return True

    return False

def strip_dir_part(path, root):
    """Strip `root` part from `path`.

    >>> strip_dir_part('/home/ruby/file', '/home')
    'ruby/file'
    >>> strip_dir_part('/home/ruby/file', '/home/')
    'ruby/file'
    >>> strip_dir_part('/home/ruby/', '/home')
    'ruby/'
    >>> strip_dir_part('/home/ruby/', '/home/')
    'ruby/'
    """
    path = path.replace(root, '', 1)

    if path.startswith(os.path.sep):
        path = path[1:]

    return path

def get_files_dirs_list(root):
    """Return list of all files and directories below `root`.

    Root directory is excluded from files/directories paths.
    """
    files = []
    directories = []

    for dirpath, dirnames, filenames in os.walk(root):
        dirpath = strip_dir_part(dirpath, root)
        files.extend(map(lambda x: os.path.join(dirpath, x), filenames))
        directories.extend(map(lambda x: os.path.join(dirpath, x), dirnames))

    return files, directories

################################################################################
## Main index class.
################################################################################

class NameSetter(type):
    def __init__(cls, name, bases, dict):
        if 'name' not in dict:
            setattr(cls, 'name', index_class_to_name(name))

def make_indices_dict(indices):
    indices_dict = {}
    for index in indices:
        indices_dict[index.name] = index
    return indices_dict

class Index(object):
    """Class describing one index.

    Use it as a container index or subclass to create custom indices.

    During class initialization, special attribute `name` is magically
    set based on class name. See `index_class_to_name` and `NameSetter`
    definitions for details.
    """
    __metaclass__ = NameSetter

    subindices = None

    name = "unnamed"
    value = -1
    details = ""

    def __init__(self, indices=[]):
        if not self.subindices:
            self.subindices = []

        # Create dictionary for fast reference.
        self._indices_dict = make_indices_dict(self.subindices)

        for index in indices:
            self.add_subindex(index)

        self._compute_arguments = get_method_arguments(self.compute)

    def _iter_indices(self):
        """Iterate over each subindex and yield their values.
        """
        for index in self.subindices:
            # Pass Cheesecake instance to other indices.
            yield index.compute_with(self.cheesecake)
            # Print index info after computing.
            if not self.cheesecake.quiet:
                index.print_info()

    def compute_with(self, cheesecake):
        """Take given Cheesecake instance and compute index value.
        """
        self.cheesecake = cheesecake
        return self.compute(**get_attributes(cheesecake, self._compute_arguments))

    def compute(self):
        """Compute index value and return it.

        By default this method computes sum of all subindices. Override this
        method when subclassing for different behaviour.

        Parameters to this function are dynamically prepared with use of
        `get_attributes` function.

        :Warning: Don't use *args and **kwds arguments for this method.
        """
        self.value = sum(self._iter_indices())
        return self.value

    def _get_max_value(self):
        if self.subindices:
            return sum(map(lambda index: index.max_value,
                           self.subindices))
        return 0

    max_value = property(_get_max_value)

    def add_subindex(self, index):
        """Add subindex.

        :Parameters:
          `index` : Index instance
              Index instance for inclusion.
        """
        if not isinstance(index, Index):
            raise ValueError("subindex have to be instance of Index")

        self.subindices.append(index)
        self._indices_dict[index.name] = index

    def remove_subindex(self, index_name):
        """Remove subindex (refered by name).

        :Parameters:
          `index` : Index name
              Index name to be removed.
        """
        index = self._indices_dict[index_name]
        self.subindices.remove(index)
        del self._indices_dict[index_name]

    def _print_info_one(self):
        print "%s  (%s)" % (pad_msg(self.name, self.value), self.details)

    def _print_info_many(self):
        max_value = self.max_value
        if max_value == 0:
            return

        percentage = int(ceil(float(self.value) / float(max_value) * 100))
        print pad_line("-")

        print pad_msg("%s INDEX (ABSOLUTE)" % self.name, self.value)
        msg = pad_msg("%s INDEX (RELATIVE)" % self.name, percentage)
        msg += "  (%d out of a maximum of %d points is %d%%)" %\
             (self.value, max_value, percentage)

        print msg
        print

    def print_info(self):
        """Print index name padded with dots, followed by value and details.
        """
        if self.subindices:
            self._print_info_many()
        else:
            self._print_info_one()

    def __getitem__(self, name):
        return self._indices_dict[name]

################################################################################
## Index that computes scores based on files and directories.
################################################################################

class OneOf(object):
    def __init__(self, *possibilities):
        self.possibilities = possibilities
    def __str__(self):
        return 'one of %s' % (self.possibilities,)

def WithOptionalExt(name, extensions):
    """Handy way of writing Cheese rules for files with extensions.

    Instead of writing:
        >>> one_of = OneOf('readme', 'readme.html', 'readme.txt')

    Write this:
        >>> opt_ext = WithOptionalExt('readme', ['html', 'txt'])

    It means the same! (representation have a meaning)
        >>> str(one_of) == str(opt_ext)
        True
    """
    possibilities = [name]
    possibilities.extend(map(lambda x: name + '.' + x, extensions))

    return OneOf(*possibilities)

def Doc(name):
    return WithOptionalExt(name, ['html', 'txt'])

class FilesIndex(Index):
    _used_rules = []

    def _compute_from_rules(self, files_list, package_dir, files_rules):
        self._used_rules = []
        files_count = 0
        value = 0

        for filename in files_list:
            if not is_empty(os.path.join(package_dir, filename)):
                score = self.get_score(os.path.basename(filename), files_rules)
                if score != 0:
                    value += score
                    files_count += 1

        return files_count, value

    def get_score(self, name, specs):
        for entry, value in specs.iteritems():
            if self.match_filename(name, entry):
                self.cheesecake.log.debug("%d points entry found: %s (%s)" % \
                                          (value, name, entry))
                return value

        return 0

    def match_filename(self, name, rule):
        """Check if `name` matches given `rule`.
        """
        def equal(x, y):
            x_root, x_ext = os.path.splitext(x)
            y_root, y_ext = os.path.splitext(y.lower())
            if x_root in [y_root.lower(), y_root.upper(), y_root.capitalize()] \
                   and x_ext in [y_ext.lower(), y_ext.upper()]:
                return True
            return False

        if rule in self._used_rules:
            return False

        if isinstance(rule, basestring):
            if equal(name, rule):
                self._used_rules.append(rule)
                return True
        elif isinstance(rule, OneOf):
            for poss in rule.possibilities:
                if self.match_filename(name, poss):
                    self._used_rules.append(rule)
                    return True

        return False

################################################################################
## Installability index.
################################################################################

class IndexUrlDownload(Index):
    max_value = 25

    def compute(self, downloaded_from_url, package, url):
        if downloaded_from_url:
            self.details = "downloaded package %s from URL %s"  % (package, url)
            self.value = self.max_value
        else:
            self.value = 0

        return self.value

class IndexUnpack(Index):
    max_value = 25

    def compute(self, unpacked):
        if unpacked:
            self.details = "package unpacked successfully"
            self.value = self.max_value
        else:
            self.value = 0

        return self.value

class IndexUnpackDir(Index):
    max_value = 15

    def compute(self, unpack_dir, original_package_name):
        self.details = "unpack directory is " + unpack_dir

        if original_package_name:
            self.details += " instead of the expected " + original_package_name
            self.value = 0
        else:
            self.details += " as expected"
            self.value = self.max_value

        return self.value

class IndexSetupPy(FilesIndex):
    name = "setup.py"
    max_value = 25

    files_rules = {
        'setup.py': 25,
    }

    def compute(self, files_list, package_dir):
        setup_py_found, self.value = self._compute_from_rules(files_list, package_dir, self.files_rules)

        if setup_py_found:
            self.details = "setup.py found"
        else:
            self.details = "setup.py not found"

        return self.value

class IndexInstall(Index):
    max_value = 50

    def compute(self, installed, sandbox_install_dir):
        if installed:
            self.details = "package installed in %s" % sandbox_install_dir
            self.value = self.max_value
        else:
            self.details = "could not install package in %s" % sandbox_install_dir
            self.value = 0

        return self.value

class IndexPyPIDownload(Index):
    max_value = 50
    distance_penalty = -5

    def compute(self, package, found_on_cheeseshop, distance_from_pypi, download_url):
        if download_url:
            self.value = self.max_value

            self.details = "downloaded package " + package

            if not found_on_cheeseshop:
                self.value += (distance_from_pypi - 1) * self.distance_penalty

                if distance_from_pypi:
                    self.details += " following %d link" % distance_from_pypi
                    if distance_from_pypi > 1:
                        self.details += "s"
                        self.details += " from PyPI"
                    else:
                        self.details += " from " + download_url
            else:
                self.details += " directly from the Cheese Shop"
        else:
            self.value = 0

        return self.value

class IndexGeneratedFiles(Index):
    generated_files_penalty = -20

    def compute(self, files_list):
        self.value = 0

        pyc_files = len(get_files_of_type(files_list, 'pyc'))
        pyo_files = len(get_files_of_type(files_list, 'pyo'))

        if pyc_files > 0 or pyo_files > 0:
            self.value += self.generated_files_penalty

        self.details = "%d .pyc and %d .pyo files found" % \
                                  (pyc_files, pyo_files)

        return self.value

class IndexInstallability(Index):
    name = "INSTALLABILITY"

    subindices = [
        IndexUnpack(),
        IndexUnpackDir(),
        IndexSetupPy(),
        IndexInstall(),
        IndexGeneratedFiles(),
    ]

################################################################################
## Documentation index.
################################################################################

class IndexRequiredFiles(FilesIndex):
    cheese_files = {
        Doc('readme'): 30,
        OneOf(Doc('license'), Doc('copying')): 30,

        OneOf(Doc('announce'), Doc('changelog')): 20,
        Doc('install'): 20,

        Doc('authors'): 10,
        Doc('faq'): 10,
        Doc('news'): 10,
        Doc('thanks'): 10,
        Doc('todo'): 10,
    }

    cheese_dirs = {
        OneOf('doc', 'docs'): 30,
        OneOf('test', 'tests'): 30,

        'demo': 10,
        OneOf('example', 'examples'): 10,
    }

    max_value = sum(cheese_files.values() + cheese_dirs.values())

    def compute(self, files_list, dirs_list, package_dir):
        files_count, files_value = self._compute_from_rules(files_list, package_dir, self.cheese_files)
        dirs_count, dirs_value = self._compute_from_rules(dirs_list, package_dir, self.cheese_dirs)

        self.value = files_value + dirs_value

        self.details = "%d files and %d required directories found" % \
                       (files_count, dirs_count)

        return self.value

class IndexDocstrings(Index):
    max_value = 100

    def compute(self, object_cnt, docstring_cnt):
        percent = 0
        if object_cnt > 0:
            percent = float(docstring_cnt)/float(object_cnt)

        # Scale the result.
        self.value = int(ceil(percent * self.max_value))

        self.details = "found %d/%d=%.2f%% objects with docstrings" %\
                 (docstring_cnt, object_cnt, percent*100)

        return self.value

class IndexFormattedDocstrings(Index):
    max_value = 50

    def compute(self, object_cnt, docformat_cnt):
        percent = 0
        if object_cnt > 0:
            percent = float(docformat_cnt)/float(object_cnt)

        # Scale the result.
        # We give 20p for 25% of formatted docstrings, 35p for 50% and 50p for 75%.
        self.value = 0
        if percent > 0.75:
            self.value = 50
        elif percent > 0.50:
            self.value = 35
        elif percent > 0.25:
            self.value = 20

        self.details = "found %d/%d=%.2f%% objects with formatted docstrings" %\
                 (docformat_cnt, object_cnt, percent*100)

        return self.value

class IndexDocumentation(Index):
    name = "DOCUMENTATION"

    subindices = [
        IndexRequiredFiles(),
        IndexDocstrings(),
        IndexFormattedDocstrings(),
    ]

################################################################################
## Code "kwalitee" index.
################################################################################

class IndexUnitTests(Index):
    """Compute unittest index as percentage of methods/functions
    that are exercised in unit tests.
    """
    max_value = 50

    def compute(self, files_list, functions, package_dir):
        unittest_cnt = 0
        self.functions_tested = {}

        for testfile in get_files_of_type(files_list, 'test'):
            fullpath = os.path.join(package_dir, testfile)
            code = CodeParser(fullpath, self.cheesecake.log.debug)

            func_called = code.functions_called()

            for func in func_called:
                self.functions_tested[func] = 1

        for funcname in functions:
            if self.is_unit_tested(funcname):
                unittest_cnt += 1
                self.log.cheesecake.debug("%s is unit tested" % funcname)

        percent = 0
        if len(functions) > 0:
            percent = float(unittest_cnt)/float(len(functions))

        # Scale the result.
        self.value = int(ceil(percent * self.max_value))

        self.details = "found %d/%d=%.2f%% unit tested methods/functions." %\
                 (unittest_cnt, len(functions), percent*100)

        return self.value

    def is_unit_tested(self, funcname):
        elem = funcname.split(".")
        n1 = elem[-1]
        n2 = ""
        if len(elem) > 1:
            n2 = elem[-2] + "." + elem[-1]
        for key in self.functions_tested.keys():
            if key.startswith(n1) or (n2 and key.startswith(n2)):
                return True
        return False

class IndexPyLint(Index):
    """Compute pylint index as average of positive pylint scores obtained for
    the Python files identified in the package.
    """
    name = "pylint"
    max_value = 50

    disabled_messages = [
        'W0403', # relative import
        'W0406', # importing of self
    ]
    pylint_args = ' '.join(map(lambda x: '--disable-msg=%s' % x, disabled_messages))

    def compute(self, files_list, package_dir):
        self.value = 0

        # Try to run the pylint script
        if not command_successful("pylint --version"):
            self.details = "pylint not properly installed"
            return self.value

        pylint_value = 0
        cnt = 0
        for pyfile in get_files_of_type(files_list, 'module'):
            fullpath = os.path.join(package_dir, pyfile)
            path, filename = os.path.split(fullpath)
            module, ext = os.path.splitext(filename)

            self.cheesecake.log.debug("Running pylint on file " + fullpath)
            rc, output = run_cmd("pylint %s %s" % (self.pylint_args, fullpath))
            if rc:
                self.cheesecake.log.debug("encountered an error (%d)." % rc)
                continue

            score_line = output.split("\n")[-3]
            s = re.search(r" (\d+\.\d+)/10", score_line)
            # We only take positive scores into account
            if s:
                score = s.group(1)
                if score == "0.00":
                    self.cheesecake.log.debug("ignoring 0.00 score.")
                    continue
                else:
                    self.cheesecake.log.debug("pylint score for module %s: %s" % (module, score))
                pylint_value += float(score)
                cnt += 1

        avg_score = 0
        if cnt:
            avg_score = float(pylint_value)/float(cnt)

        self.value = int(ceil(avg_score/10.0 * self.max_value))
        self.details = "average pylint score is %.2f out of 10" % avg_score

        return self.value

class IndexCodeKwalitee(Index):
    name = "CODE KWALITEE"

    subindices = [
        IndexPyLint(),
        # IndexUnitTests(), TODO
    ]

################################################################################
## Main Cheesecake class.
################################################################################

class CheesecakeError(Exception):
    """Custom exception class for Cheesecake-specific errors.
    """
    pass


class CheesecakeIndex(Index):
    name = "Cheesecake"
    subindices = [
        IndexInstallability(),
        IndexDocumentation(),
        IndexCodeKwalitee(),
    ]


class Cheesecake(object):
    """
    Computes 'goodness' of Python packages

    Generates "cheesecake index" that takes into account things like:

        * whether the package can be downloaded
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
        * existence of certain files such as README, INSTALL, LICENSE, setup.py etc.
        * existence of certain directories such as doc, test, demo, examples
        * percentage of modules/functions/classes/methods with docstrings
        * percentage of functions/methods that are unit tested 
        * average pylint score for all non-test and non-demo modules
    """
    index = CheesecakeIndex()

    def __init__(self, name="", url="", path="", sandbox=None, config=None,
                logfile=None, verbose=False, quiet=False):
        """Initialize critical variables, download and unpack package,
        walk package tree.
        """
        self.name = name
        self.url = url
        self.package_path = path

        if not self.name and not self.url and not self.package_path:
            self.raise_exception("No package name, URL or path specified ... exiting")

        self.sandbox = sandbox or tempfile.mkdtemp(prefix='cheesecake')
        if not os.path.isdir(self.sandbox):
            os.mkdir(self.sandbox)

        self.config = config
        self.verbose = verbose
        self.quiet = quiet

        self.package_types = {
            "tar.gz": untar_package,
            "tgz": untar_package,
            "zip": unzip_package,
            "egg": unzip_package,
        }

        self.sandbox_pkg_file = ""
        self.sandbox_pkg_dir = ""
        self.sandbox_install_dir = ""

        # Include indices revelant to current situation.
        if self.name:
            self.index["INSTALLABILITY"].add_subindex(IndexPyPIDownload())
        if self.url:
            self.index["INSTALLABILITY"].add_subindex(IndexUrlDownload())

        # Get package name.
        self.determine_pkg_name()

        # Configure logging as soon as possible.
        self.configure_logging(logfile)

        # Get package, 
        self.retrieve_pkg()

        # Get package name and type.
        self.package_name, self.package_type = self.get_package_name_and_type(self.package)
        self.log.debug("Package name: " + self.package_name)
        self.log.debug("Package type: " + self.package_type)

        # Unpack package and list its files.
        self.unpack_pkg()
        self.walk_pkg()

        # Install package.
        self.install_pkg()

        # When checking an egg exclude irrelevant indices.
        if self.package_type == 'egg':
            self.index["INSTALLABILITY"].remove_subindex('setup.py')
            self.index["INSTALLABILITY"].remove_subindex('unpack_dir')
            self.index["INSTALLABILITY"].remove_subindex('generated_files')

    def raise_exception(self, msg):
        """Cleanup, print error message and raise CheesecakeError.

        Don't use logging, since it can be called before logging has been setup.
        """
        self.cleanup(remove_log_file=False)

        msg += "\nDetailed info available in log file %s" % self.logfile

        raise CheesecakeError(msg)

    def cleanup(self, remove_log_file=True):
        """Delete temporary directories and files that were created
        in the sandbox. At the end delete the sandbox itself.
        """
        if os.path.isfile(self.sandbox_pkg_file):
            self.log("Removing file %s" % self.sandbox_pkg_file)
            os.unlink(self.sandbox_pkg_file)

        def delete_dir(dirname):
            "Delete directory recursively and generate log message."
            if os.path.isdir(dirname):
                self.log("Removing directory %s" % dirname)
                shutil.rmtree(dirname)

        delete_dir(self.sandbox)

        if remove_log_file:
            os.unlink(os.path.join(self.sandbox, self.logfile))

    def set_defaults(self):
        """Set default values for variables that can also be defined
        in the config file.
        """
        pass

    def get_config(self, config_dir=None):
        """Retrieve values from configuration file.
        """
        pass

    def determine_pkg_name(self):
        if self.name:
            self.package = self.name
            self.short_pkg_name = self.name
        elif self.package_path:
            self.package = self.get_package_from_path(self.package_path)
        else:
            self.package = self.get_package_from_url()

    def get_package_from_url(self):
        """Use ``urlparse`` to obtain package path from URL.
        """
        (scheme,location,path,param,query,fragment_id) = urlparse(self.url)
        return self.get_package_from_path(path)

    def get_package_from_path(self, path):
        """Get package name as file portion of path.
        """
        dir, file = os.path.split(path)
        self.short_pkg_name = file
        for package_type in self.package_types:
            s = re.search("(.+)\.%s" % package_type, file)
            if s:
                self.short_pkg_name = s.group(1)
                break
        return file

    def configure_logging(self, logfile=None):
        """Default settings for logging.

        If verbose, log goes to console, else it goes to logfile.
        log.debug and log.info goes to logfile.
        log.warn and log.error go to both logfile and stdout.
        """
        if logfile:
            self.logfile = logfile
        else:
            self.logfile = os.path.join(tempfile.gettempdir(), self.short_pkg_name + ".log")

        logger.setconsumer('logfile', open(str(self.logfile), 'w', buffering=1))
        logger.setconsumer('console', logger.STDOUT)
        logger.setconsumer('null', None)

        if self.verbose:
            self.log = logger.MultipleProducer('cheesecake console')
        else:
            self.log = logger.MultipleProducer('cheesecake logfile')

        self.log.info = logger.MultipleProducer('cheesecake logfile')
        self.log.debug = logger.MultipleProducer('cheesecake logfile')
        self.log.warn = logger.MultipleProducer('cheesecake console')
        self.log.error = logger.MultipleProducer('cheesecake console')

    def retrieve_pkg(self):
        if self.name:
            self.get_pkg_from_pypi()
        elif self.url:
            self.download_pkg()
        else:
            self.copy_pkg()

    def get_pkg_from_pypi(self):
        """Download package using setuptools utilities.

        :Ivariables:
          download_url : str
              URL that package was downloaded from.
          distance_from_pypi : int
              How many hops setuptools had to make to download package.
          found_on_cheeseshop : bool
              Whenever package has been found on CheeseShop.
        """
        self.log.info("Trying to download package %s from PyPI using setuptools utilities" % self.name)

        try:
            from setuptools.package_index import PackageIndex
            from pkg_resources import Requirement
            from distutils import log
            from distutils.errors import DistutilsError

        except ImportError, e:
            msg = "Error: setuptools is not installed and is required for downloading a package by name\n"
            msg += "You can download and process a package by its full URL via the -u or --url option\n"
            msg += "Example: python cheesecake.py --url=http://www.mems-exchange.org/software/durus/Durus-3.1.tar.gz"
            self.raise_exception(msg)

        def drop_setuptools_info(stdout, error=None):
            """Drop all setuptools output as INFO.
            """
            self.log.info("*** Begin setuptools output")
            map(self.log.info, stdout.splitlines())
            if error:
                self.log.info(str(error))
            self.log.info("*** End setuptools output")

        try:
            # Temporarily set the log verbosity to INFO so we can capture setuptools info messages
            old_threshold = log.set_threshold(log.INFO)
            pkgindex = PackageIndex()
            old_stdout = sys.stdout
            sys.stdout = StdoutRedirector()
            output = pkgindex.fetch(Requirement.parse(self.name),
                                    self.sandbox,
                                    force_scan=True,
                                    source=False)
            captured_stdout = sys.stdout.read_buffer()
            sys.stdout = old_stdout
            log.set_threshold(old_threshold)

        except DistutilsError, e:
            # Bring back old stdout.
            captured_stdout = sys.stdout.read_buffer()
            sys.stdout = old_stdout
            log.set_threshold(old_threshold)

            drop_setuptools_info(captured_stdout, error)
            self.raise_exception("Error: setuptools returned an error: %s\n" % e)

        if output is None:
            drop_setuptools_info(captured_stdout)
            self.raise_exception("Error: Could not find distribution for " + self.name)

        # Defaults.
        self.download_url = ""
        self.distance_from_pypi = 0
        self.found_on_cheeseshop = False

        for line in captured_stdout.split('\n'):
            s = re.search(r"Reading http(.*)", line)
            if s:
                inspected_url = s.group(1)
                if not re.search(r"www.python.org\/pypi", inspected_url):
                    self.distance_from_pypi += 1
                continue
            s = re.search(r"Downloading (.*)", line)
            if s:
                self.download_url = s.group(1)
                break

        self.sandbox_pkg_file = output
        self.package = self.get_package_from_path(output)
        self.log.info("Downloaded package %s from %s" % (self.package, self.download_url))

        if re.search(r"cheeseshop.python.org", self.download_url):
            self.found_on_cheeseshop = True

    def download_pkg(self):
        """Use ``urllib.urlretrieve`` to download package to file in sandbox dir.
        """
        #self.log("Downloading package %s from URL %s" % (self.package, self.url))
        self.sandbox_pkg_file = os.path.join(self.sandbox, self.package)
        try:
            downloaded_filename, headers = urlretrieve(self.url, self.sandbox_pkg_file)
        except IOError, e:
            self.log.error("Error downloading package %s from URL %s"  % (self.package, self.url))
            self.raise_exception(str(e))
        #self.log("Downloaded package %s to %s" % (self.package, downloaded_filename))

        if headers.gettype() in ["text/html"]:
            f = open(downloaded_filename)
            if re.search("404 Not Found", "".join(f.readlines())):
                f.close()
                self.raise_exception("Got '404 Not Found' error while trying to download package ... exiting")
            f.close()

        self.downloaded_from_url = True
        
    def copy_pkg(self):
        """Copy package file to sandbox directory.
        """
        self.sandbox_pkg_file = os.path.join(self.sandbox, self.package)
        if not os.path.isfile(self.package_path):
            self.raise_exception("%s is not a valid file ... exiting" % self.package_path)
        self.log("Copying file %s to %s" % (self.package_path, self.sandbox_pkg_file))
        shutil.copyfile(self.package_path, self.sandbox_pkg_file)

    def get_package_name_and_type(self, package):
        """Return package name and type.

        Raise an exception when package type cannot be recognized.
        """
        for type in self.package_types.keys():
            s = re.search(r"(.+)\.%s" % type, package)
            if s:
                # Package name is name of package without file extension (ex. twill-7.3).
                return s.group(1), type

        msg = "Could not determine package type for package '%s'" % package
        msg += "\nCurrently recognized types: " + ", ".join(self.package_types.keys())
        self.raise_exception(msg)

    def unpack_pkg(self):
        """Unpack the package in the sandbox directory.

        Check `package_types` attribute for list of currently supported
        archive types.

        :Ivariables:
          original_package_name : str
        """
        self.sandbox_pkg_dir = os.path.join(self.sandbox, self.package_name)
        if os.path.isdir(self.sandbox_pkg_dir):
            shutil.rmtree(self.sandbox_pkg_dir)

        # Call appropriate function to unpack the package.
        unpack = self.package_types[self.package_type]
        self.unpack_dir = unpack(self.sandbox_pkg_file, self.sandbox)

        if self.unpack_dir is None:
            self.raise_exception("Could not unpack package %s ... exiting" % \
                                 self.sandbox_pkg_file)

        self.unpacked = True

        if self.unpack_dir != self.package_name:
            self.original_package_name = self.package_name
            self.package_name = self.unpack_dir

    def walk_pkg(self):
        """Get package files and directories.

        :Ivariables:
          dirs_list : list
              List of directories package contains.
          docstring_cnt : int
              Number of docstrings found in all package objects.
          docformat_cnt : int
              Number of formatted docstrings found in all package objects.
          files_list : list
              List of files package contains.
          functions : list
              List of all functions defined in package sources.
          object_cnt : int
              Number of documentable objects found in all package modules.
          package_dir : str
              Path to project directory.
        """
        self.package_dir = os.path.join(self.sandbox, self.package_name)

        self.files_list, self.dirs_list = get_files_dirs_list(self.package_dir)

        self.object_cnt = 0
        self.docstring_cnt = 0
        self.docformat_cnt = 0
        self.functions = []

        # Parse all application files and count objects
        # (modules/classes/functions) and their associated docstrings.
        for py_file in get_files_of_type(self.files_list, 'module'):
            pyfile = os.path.join(self.package_dir, py_file)
            code = CodeParser(pyfile, self.log.debug)

            self.object_cnt += code.object_count()
            self.docstring_cnt += code.docstring_count()
            self.docformat_cnt += code.formatted_docstrings_count
            self.functions += code.functions

        # Log a bit of debugging info.
        self.log.debug("Found %d files: %s." % (len(self.files_list),
                                                ', '.join(self.files_list)))
        self.log.debug("Found %d directories: %s." % (len(self.dirs_list),
                                                      ', '.join(self.dirs_list)))

    def install_pkg(self):
        """Verify that package can be installed in alternate directory.

        :Ivariables:
          installed : bool
              Describes whenever package has been succefully installed.
        """
        self.sandbox_install_dir = os.path.join(self.sandbox, "tmp_install_%s" % self.package_name)

        if self.package_type == 'egg':
            # Create dummy Python directories.
            mkdirs('%s/lib/python2.3/site-packages/' % self.sandbox_install_dir)
            mkdirs('%s/lib/python2.4/site-packages/' % self.sandbox_install_dir)

            environment = {'PYTHONPATH':
                           '%(sandbox)s/lib/python2.3/site-packages/:'\
                           '%(sandbox)s/lib/python2.4/site-packages/' % \
                           {'sandbox': self.sandbox_install_dir}}
            rc, output = run_cmd("easy_install --no-deps --prefix %s %s" % \
                                 (self.sandbox_install_dir, self.sandbox_pkg_file),
                                 environment)
        else:
            if os.path.isdir(self.package_name):
                package_dir = os.path.join(self.sandbox, self.package_name)
            else:
                package_dir = self.sandbox
            cwd = os.getcwd()
            os.chdir(package_dir)
            rc, output = run_cmd("python setup.py install --root=" + self.sandbox_install_dir)
            os.chdir(cwd)

        if rc:
            self.log('*** Installation failed. Captured output:')
            for output_line in output.splitlines():
                self.log(output_line)
            self.log('*** End of captured output.')
        else:
            self.installed = True

    def compute_cheesecake_index(self):
        """Compute overall Cheesecake index for the package by adding up
        specific indexes.
        """
        # Recursively compute all indices.
        max_cheesecake_index = self.index.max_value

        # Pass Cheesecake instance to the main Index object.
        cheesecake_index = self.index.compute_with(self)
        percentage = (cheesecake_index * 100) / max_cheesecake_index

        self.log.info("A given package can currently reach a MAXIMUM number of %d points" % max_cheesecake_index)
        self.log.info("Starting computation of Cheesecake index for package '%s'" % (self.package))

        # Print summary.
        if self.quiet:
            print "Cheesecake index: %d (%d / %d)" % (percentage,
                                                      cheesecake_index,
                                                      max_cheesecake_index)
        else:
            print
            print pad_line("=")
            print pad_msg("OVERALL CHEESECAKE INDEX (ABSOLUTE)", cheesecake_index)
            print "%s  (%d out of a maximum of %d points is %d%%)" % \
                  (pad_msg("OVERALL CHEESECAKE INDEX (RELATIVE)", percentage),
                   cheesecake_index,
                   max_cheesecake_index,
                   percentage)

        return cheesecake_index

################################################################################
## Command line.
################################################################################

def process_cmdline_args():
    """Parse command-line options.
    """
    parser = OptionParser()
    parser.add_option("-n", "--name", dest="name",
                      default="", help="package name (will be retrieved via setuptools utilities, if present)")
    parser.add_option("-u", "--url", dest="url",
                      default="", help="package URL")
    parser.add_option("-p", "--path", dest="path",
                      default="", help="path of tar.gz/zip package on local file system")
    parser.add_option("-s", "--sandbox", dest="sandbox",
                      default=None,
                      help="directory where package will be unpacked "\
                           "(default is to use random directory inside %s)" % tempfile.gettempdir())
    parser.add_option("-c", "--config", dest="config",
                      default=None,
                      help="directory with custom configuration (default=~/.cheesecake)")
    parser.add_option("-l", "--logfile", dest="logfile",
                      default=None,
                      help="file to log all cheesecake messages")
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
                      default=False, help="verbose output (default=False)")
    parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
                      default=False, help="only print Cheesecake index value (default=False)")

    (options, args) = parser.parse_args()
    return options

def main():
    """Display Cheesecake index for package specified via command-line options.
    """
    options = process_cmdline_args()
    name = options.name
    url = options.url
    path = options.path
    sandbox = options.sandbox
    config = options.config
    logfile = options.logfile
    verbose = options.verbose
    quiet = options.quiet

    if not name and not url and not path:
        print "Error: No package name, URL or path specified (see --help)"
        sys.exit(1)

    try:
        c = Cheesecake(name=name, url=url, path=path, sandbox=sandbox,
                       config=config, logfile=logfile, verbose=verbose,
                       quiet=quiet)
        c.compute_cheesecake_index()
        c.cleanup()
    except CheesecakeError, e:
        print str(e)

if __name__ == "__main__":
    main()