root/branches/mk/cheesecake/cheesecake_index.py

#!/usr/bin/env python
"""
Cheesecake: How tasty is your code?

The idea of the Cheesecake project is to rank Python packages
based on various empiric "kwalitee" factors, such as:

        * whether the package can be downloaded
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
        * existence of certain files such as README, INSTALL, LICENSE, setup.py etc.
        * existence of certain directories such as doc, test, demo, examples
        * percentage of modules/functions/classes/methods with docstrings
        * percentage of functions/methods that are unit tested
        * average pylint score for all non-test and non-demo modules
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
"""

import os, sys, re, shutil
import tarfile, zipfile
import tempfile
from optparse import OptionParser
from urllib import urlretrieve
from urlparse import urlparse
from math import ceil

from _util import run_cmd, pad_with_dots, pad_left_spaces, pad_msg, pad_line, command_successful
from _util import StdoutRedirector
import logger
from config import get_pkg_config
from codeparser import CodeParser

__docformat__ = 'reStructuredText en'


default_temp_directory = os.path.join(tempfile.gettempdir(),
                                      'cheesecake_sandbox')

################################################################################
## Helpers.
################################################################################

def isiterable(obj):
    return hasattr(obj, '__iter__')

def has_extension(filename, ext):
    """Check if filename has given extension.

    >>> has_extension("foobar.py", ".py")
    True
    >>> has_extension("foo.bar.py", ".py")
    True
    >>> has_extension("foobar.pyc", ".py")
    False
    """
    return os.path.splitext(filename)[1] == ext

def discover_file_type(filename):
    """Discover type of a file according to its name and its parent directory.

    Currently supported file types:
        * pyc
        * pyo
        * module: .py files of an application
        * demo: .py files for documentation/demonstration purposes
        * test: .py files used for testing
        * special: .py file for special purposes

    :Note: This function only check file's name, and doesn't touch the
           filesystem. If you have to, check if file exists by yourself.

    >>> discover_file_type('module.py')
    'module'
    >>> discover_file_type('./setup.py')
    'special'
    >>> discover_file_type('some/directory/junk.pyc')
    'pyc'
    """
    dirs = filename.split(os.path.sep)
    dirs, filename = dirs[:-1], dirs[-1]

    if filename in ["setup.py", "ez_setup.py", "__pkginfo__.py"]:
        return 'special'

    if has_extension(filename, ".pyc"):
        return 'pyc'
    if has_extension(filename, ".pyo"):
        return 'pyo'
    if has_extension(filename, ".py"):
        for dir in dirs:
            if dir in ['test', 'tests']:
                return 'test'
            elif dir in ['docs', 'demo', 'example']:
                return 'demo'
        return 'module'

def get_files_of_type(file_list, file_type):
    """Return files from `file_list` that match given `file_type`.

    >>> file_list = ['test/test_foo.py', 'setup.py', 'README', 'test/test_bar.py']
    >>> get_files_of_type(file_list, 'test')
    ['test/test_foo.py', 'test/test_bar.py']
    """
    return filter(lambda x: discover_file_type(x) == file_type, file_list)

def get_method_arguments(method):
    """Return tuple of arguments for given method, excluding self.

    >>> class Class:
    ...     def method(s, arg1, arg2, other_arg):
    ...         pass
    >>> get_method_arguments(Class.method)
    ('arg1', 'arg2', 'other_arg')
    """
    return method.func_code.co_varnames[1:method.func_code.co_argcount]

def get_attributes(obj, names):
    """Return attributes dictionary with keys from `names`.

    Object is queried for each attribute name, if it doesn't have this
    attribute, default value None will be returned.

    >>> class Class:
    ...     pass
    >>> obj = Class()
    >>> obj.attr = True
    >>> obj.value = 13
    >>> obj.string = "Hello"

    >>> d = get_attributes(obj, ['attr', 'string', 'other'])
    >>> d == {'attr': True, 'string': "Hello", 'other': None}
    True
    """
    attrs = {}

    for name in names:
        attrs[name] = getattr(obj, name, None)

    return attrs

def camel2underscore(name):
    """Convert name from CamelCase to underscore_name.

    >>> camel2underscore('CamelCase')
    'camel_case'
    >>> camel2underscore('already_underscore_name')
    'already_underscore_name'
    >>> camel2underscore('BigHTMLClass')
    'big_html_class'
    >>> camel2underscore('')
    ''
    """
    if name and name[0].upper:
        name = name[0].lower() + name[1:]

    def capitalize(match):
        string = match.group(1).lower().capitalize()
        return string[:-1] + string[-1].upper()

    def underscore(match):
        return '_' + match.group(1).lower()

    name = re.sub(r'([A-Z]+)', capitalize, name)
    return re.sub(r'([A-Z])', underscore, name)

def index_class_to_name(clsname):
    """Covert index class name to index name.

    >>> index_class_to_name("IndexDownload")
    'download'
    >>> index_class_to_name("IndexUnitTests")
    'unit_tests'
    >>> index_class_to_name("IndexPyPIDownload")
    'py_pi_download'
    """
    return camel2underscore(clsname.replace('Index', '', 1))

def is_empty(path):
    """Returns True if file or directory pointed by `path` is empty.
    """
    if os.path.isfile(path) and os.path.getsize(path) == 0:
        return True
    if os.path.isdir(path) and os.listdir(path) == []:
        return True

    return False

def strip_dir_part(path, root):
    """Strip `root` part from `path`.

    >>> strip_dir_part('/home/ruby/file', '/home')
    'ruby/file'
    >>> strip_dir_part('/home/ruby/file', '/home/')
    'ruby/file'
    >>> strip_dir_part('/home/ruby/', '/home')
    'ruby/'
    >>> strip_dir_part('/home/ruby/', '/home/')
    'ruby/'
    """
    path = path.replace(root, '', 1)

    if path.startswith(os.path.sep):
        path = path[1:]

    return path

def get_files_dirs_list(root):
    """Return list of all files and directories below `root`.

    Root directory is excluded from files/directories paths.
    """
    files = []
    directories = []

    for dirpath, dirnames, filenames in os.walk(root):
        dirpath = strip_dir_part(dirpath, root)
        files.extend(map(lambda x: os.path.join(dirpath, x), filenames))
        directories.extend(map(lambda x: os.path.join(dirpath, x), dirnames))

    return files, directories

################################################################################
## Main index class.
################################################################################

class NameSetter(type):
    def __init__(cls, name, bases, dict):
        if 'name' not in dict:
            setattr(cls, 'name', index_class_to_name(name))

def make_indices_dict(indices):
    indices_dict = {}
    for index in indices:
        indices_dict[index.name] = index
    return indices_dict

class Index(object):
    """Class describing one index.

    Use it as a container index or subclass to create custom indices.

    During class initialization, special attribute `name` is magically
    set based on class name. See `index_class_to_name` and `NameSetter`
    definitions for details.
    """
    __metaclass__ = NameSetter

    subindices = None

    name = "unnamed"
    value = -1
    details = ""

    def __init__(self, indices=[]):
        if not self.subindices:
            self.subindices = []

        # Create dictionary for fast reference.
        self._indices_dict = make_indices_dict(self.subindices)

        for index in indices:
            self.add_subindex(index)

        self._compute_arguments = get_method_arguments(self.compute)

    def _iter_indices(self):
        """Iterate over each subindex and yield their values.
        """
        for index in self.subindices:
            # Pass Cheesecake instance to other indices.
            yield index.compute_with(self.cheesecake)
            # Print index info after computing.
            index.print_info()

    def compute_with(self, cheesecake):
        """Take given Cheesecake instance and compute index value.
        """
        self.cheesecake = cheesecake
        return self.compute(**get_attributes(cheesecake, self._compute_arguments))

    def compute(self):
        """Compute index value and return it.

        By default this method computes sum of all subindices. Override this
        method when subclassing for different behaviour.

        Parameters to this function are dynamically prepared with use of
        `get_attributes` function.

        :Warning: Don't use *args and **kwds arguments for this method.
        """
        self.value = sum(self._iter_indices())
        return self.value

    def _get_max_value(self):
        if self.subindices:
            return sum(map(lambda index: index.max_value,
                           self.subindices))
        return 0

    max_value = property(_get_max_value)

    def add_subindex(self, index):
        """Add subindex.

        :Parameters:
          `index` : Index instance
              Index instance for inclusion.
        """
        if not isinstance(index, Index):
            raise ValueError("subindex have to be instance of Index")

        self.subindices.append(index)
        self._indices_dict[index.name] = index

    def print_info(self):
        """Print index name padded with dots, followed by value and details.
        """
        print "%s%s (%s)" % (pad_with_dots(self.name),
                             pad_left_spaces(self.value),
                             self.details)

    def __getitem__(self, name):
        return self._indices_dict[name]

class MegaIndex(Index):
    """Index with special information schema, suitable for composite indices.
    """
    def print_info(self):
        max_value = self.max_value
        if max_value == 0:
            return

        percentage = int(ceil(float(self.value) / float(max_value) * 100))
        print pad_line("-")

        print pad_msg("%s INDEX (ABSOLUTE)" % self.name, self.value)
        msg = pad_msg("%s INDEX (RELATIVE)" % self.name, percentage)
        msg += " (%d out of a maximum of %d points is %d%%)" %\
             (self.value, max_value, percentage)

        print msg
        print

################################################################################
## Installability index.
################################################################################

class IndexUrlDownload(Index):
    max_value = 25

    def compute(self, downloaded_from_url, package, url):
        if downloaded_from_url:
            self.details = "downloaded package %s from URL %s"  % (package, url)
            self.value = self.max_value
        else:
            self.value = 0

        return self.value

class IndexUnpack(Index):
    max_value = 25

    def compute(self, unpacked):
        if unpacked:
            self.details = "package unpacked successfully"
            self.value = self.max_value
        else:
            self.value = 0

        return self.value

class IndexUnpackDir(Index):
    max_value = 15

    def compute(self, unpack_dir, original_package_name):
        self.details = "unpack directory is " + unpack_dir

        if original_package_name:
            self.details += " instead of the expected " + original_package_name
            self.value = 0
        else:
            self.details += " as expected"
            self.value = self.max_value

        return self.value

class IndexInstall(Index):
    max_value = 50

    def compute(self, installed, sandbox_install_dir):
        if installed:
            self.details = "package installed in %s" % sandbox_install_dir
            self.value = self.max_value
        else:
            self.details = "could not install package in %s" % sandbox_install_dir
            self.value = 0

        return self.value

class IndexPyPIDownload(Index):
    max_value = 50
    distance_penalty = -5

    def compute(self, package, found_on_cheeseshop, distance_from_pypi, download_url):
        if download_url:
            self.value = self.max_value

            self.details = "downloaded package " + package

            if not found_on_cheeseshop:
                self.value += distance_from_pypi * self.distance_penalty

                if distance_from_pypi:
                    self.details += " following %d link" % distance_from_pypi
                    if distance_from_pypi > 1:
                        self.details += "s"
                        self.details += " from PyPI"
                    else:
                        self.details += " from " + download_url
            else:
                self.details += " directly from the Cheese Shop"
        else:
            self.value = 0

        return self.value

class IndexGeneratedFiles(Index):
    generated_files_penalty = -20

    def compute(self, files_list):
        self.value = 0

        pyc_files = len(get_files_of_type(files_list, 'pyc'))
        pyo_files = len(get_files_of_type(files_list, 'pyo'))

        if pyc_files > 0 or pyo_files > 0:
            self.value += self.generated_files_penalty

        self.details = "%d .pyc and %d .pyo files found" % \
                                  (pyc_files, pyo_files)

        return self.value

class IndexInstallability(MegaIndex):
    name = "INSTALLABILITY"

    subindices = [
        IndexUnpack(),
        IndexUnpackDir(),
        IndexInstall(),
        IndexGeneratedFiles(),
    ]

################################################################################
## Documentation index.
################################################################################

def match_filename(name, rule):
    """Check if `name` matches given `rule`.
    """
    def equal(x, y):
        x_root, x_ext = os.path.splitext(x)
        y_root, y_ext = os.path.splitext(y.lower())
        if x_root in [y_root.lower(), y_root.upper(), y_root.capitalize()] \
               and x_ext in [y_ext.lower(), y_ext.upper()]:
            return True
        return False

    if isinstance(rule, basestring):
        if equal(name, rule):
            return True
    elif isinstance(rule, OneOf) and not rule.used:
        for poss in rule.possibilities:
            if match_filename(name, poss):
                rule.used = True
                return True

    return False

class OneOf(object):
    def __init__(self, *possibilities):
        self.possibilities = possibilities
        self.used = False
    def __str__(self):
        return 'one of %s' % (self.possibilities,)

def WithOptionalExt(name, extensions):
    """Handy way of writing Cheese rules for files with extensions.

    Instead of writing:
        >>> one_of = OneOf('readme', 'readme.html', 'readme.txt')

    Write this:
        >>> opt_ext = WithOptionalExt('readme', ['html', 'txt'])

    It means the same! (representation have a meaning)
        >>> str(one_of) == str(opt_ext)
        True
    """
    possibilities = [name]
    possibilities.extend(map(lambda x: name + '.' + x, extensions))

    return OneOf(*possibilities)

def Doc(name):
    return WithOptionalExt(name, ['html', 'txt'])

class IndexRequiredFiles(Index):
    cheese_files = {
        'setup.py': 15,
        Doc('readme'): 15,
        OneOf(Doc('license'), Doc('copying')): 15,

        Doc('authors'): 10,
        Doc('announce'): 10,
        Doc('changelog'): 10,
        Doc('faq'): 10,
        Doc('install'): 10,
        Doc('news'): 10,
        Doc('thanks'): 10,
        Doc('todo'): 10,
    }

    cheese_dirs = {
        'demo': 20,
        'doc': 25,
        'example': 20,
        OneOf('test', 'tests'): 25,
    }

    max_value = sum(cheese_files.values() + cheese_dirs.values())

    def compute(self, files_list, dirs_list, package_dir):
        self.value = 0
        self.reset_rules(self.cheese_files.keys() + self.cheese_dirs.keys())

        files_count = 0
        for filename in files_list:
            if not is_empty(os.path.join(package_dir, filename)):
                score = self.get_score(os.path.basename(filename), self.cheese_files)
                if score != 0:
                    self.value += score
                    files_count += 1

        directories_count = 0
        for directory in dirs_list:
            if not is_empty(os.path.join(package_dir, directory)):
                score = self.get_score(os.path.basename(directory), self.cheese_dirs)
                if score != 0:
                    self.value += score
                    directories_count += 1

        self.details = "%d files and %d required directories found" % \
                       (files_count, directories_count)

        return self.value

    def get_score(self, name, specs):
        for entry, value in specs.iteritems():
            if match_filename(name, entry):
                self.cheesecake.log.debug("%d points entry found: %s (%s)" % \
                                          (value, name, entry))
                return value

        return 0

    def reset_rules(self, rules):
        if isiterable(rules):
            for rule in rules:
                self.reset_rules(rule)
        elif isinstance(rules, OneOf):
            rules.used = False
            self.reset_rules(rules.possibilities)

class IndexDocstrings(Index):
    max_value = 100

    def compute(self, object_cnt, docstring_cnt):
        percent = 0
        if object_cnt > 0:
            percent = float(docstring_cnt)/float(object_cnt)

        # Scale the result.
        self.value = int(ceil(percent * self.max_value))

        self.details = "found %d/%d=%.2f%% objects with docstrings" %\
                 (docstring_cnt, object_cnt, percent*100)

        return self.value

class IndexFormattedDocstrings(Index):
    max_value = 50

    def compute(self, object_cnt, docformat_cnt):
        percent = 0
        if object_cnt > 0:
            percent = float(docformat_cnt)/float(object_cnt)

        # Scale the result.
        self.value = int(ceil(percent * self.max_value))

        self.details = "found %d/%d=%.2f%% objects with formatted docstrings" %\
                 (docformat_cnt, object_cnt, percent*100)

        return self.value

class IndexDocumentation(MegaIndex):
    name = "DOCUMENTATION"

    subindices = [
        IndexRequiredFiles(),
        IndexDocstrings(),
        IndexFormattedDocstrings(),
    ]

################################################################################
## Code "kwalitee" index.
################################################################################

class IndexUnitTests(Index):
    """Compute unittest index as percentage of methods/functions
    that are exercised in unit tests.
    """
    max_value = 50

    def compute(self, files_list, functions, package_dir):
        unittest_cnt = 0
        self.functions_tested = {}

        for testfile in get_files_of_type(files_list, 'test'):
            fullpath = os.path.join(package_dir, testfile)
            code = CodeParser(fullpath, self.cheesecake.log.debug)

            func_called = code.functions_called()

            for func in func_called:
                self.functions_tested[func] = 1

        for funcname in functions:
            if self.is_unit_tested(funcname):
                unittest_cnt += 1
                self.log.cheesecake.debug("%s is unit tested" % funcname)

        percent = 0
        if len(functions) > 0:
            percent = float(unittest_cnt)/float(len(functions))

        # Scale the result.
        self.value = int(ceil(percent * self.max_value))

        self.details = "found %d/%d=%.2f%% unit tested methods/functions." %\
                 (unittest_cnt, len(functions), percent*100)

        return self.value

    def is_unit_tested(self, funcname):
        elem = funcname.split(".")
        n1 = elem[-1]
        n2 = ""
        if len(elem) > 1:
            n2 = elem[-2] + "." + elem[-1]
        for key in self.functions_tested.keys():
            if key.startswith(n1) or (n2 and key.startswith(n2)):
                return True
        return False

class IndexPyLint(Index):
    """Compute pylint index as average of positive pylint scores obtained for
    the Python files identified in the package.
    """
    name = "pylint"
    max_value = 50

    def compute(self, files_list, package_dir):
        self.value = 0

        # Try to run the pylint script
        if not command_successful("pylint --version"):
            self.details = "pylint not properly installed"
            return self.value

        pylint_value = 0
        cnt = 0
        for pyfile in get_files_of_type(files_list, 'module'):
            fullpath = os.path.join(package_dir, pyfile)
            path, filename = os.path.split(fullpath)
            module, ext = os.path.splitext(filename)

            self.cheesecake.log.debug("Running pylint on file " + fullpath)
            rc, output = run_cmd("pylint " + fullpath)
            if rc:
                self.cheesecake.log.debug("encountered an error (%d)." % rc)
                continue

            score_line = output.split("\n")[-3]
            s = re.search(r" (\d+\.\d+)/10", score_line)
            # We only take positive scores into account
            if s:
                score = s.group(1)
                if score == "0.00":
                    self.cheesecake.log.debug("ignoring 0.00 score.")
                    continue
                else:
                    self.cheesecake.log.debug("pylint score for module %s: %s" % (module, score))
                pylint_value += float(score)
                cnt += 1

        avg_score = 0
        if cnt:
            avg_score = float(pylint_value)/float(cnt)

        self.value = int(ceil(avg_score/10.0 * self.max_value))
        self.details = "average pylint score is %.2f out of 10" % avg_score

        return self.value

class IndexCodeKwalitee(MegaIndex):
    name = "CODE KWALITEE"

    subindices = [
        IndexPyLint(),
        # IndexUnitTests(), TODO
    ]

################################################################################
## Main Cheesecake class.
################################################################################

class CheesecakeError(Exception):
    """
    Custom exception class for Cheesecake-specific errors
    """
    pass


class CheesecakeIndex(Index):
    name = "Cheesecake"
    subindices = [
        IndexInstallability(),
        IndexDocumentation(),
        IndexCodeKwalitee(),
    ]


class Cheesecake(object):
    """
    Computes 'goodness' of Python packages

    Generates "cheesecake index" that takes into account things like:

        * whether the package can be downloaded
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
        * existence of certain files such as README, INSTALL, LICENSE, setup.py etc.
        * existence of certain directories such as doc, test, demo, examples
        * percentage of modules/functions/classes/methods with docstrings
        * percentage of functions/methods that are unit tested 
        * average pylint score for all non-test and non-demo modules
    """
    index = CheesecakeIndex()

    def __init__(self, name="", url="", path="", sandbox=None, config=None,
                logfile=None, verbose=False, quiet=False):
        """Initialize critical variables, download and unpack package,
        walk package tree.
        """
        self.name = name
        self.url = url
        self.package_path = path

        if not self.name and not self.url and not self.package_path:
            self.raise_exception("No package name, URL or path specified ... exiting")

        self.sandbox = sandbox or tempfile.mkdtemp(prefix='cheesecake')
        if not os.path.isdir(self.sandbox):
            os.mkdir(self.sandbox)

        self.config = config
        self.verbose = verbose
        self.quiet = quiet

        self.package_types = ["tar.gz", "tgz", "zip"]
        self.sandbox_pkg_file = ""
        self.sandbox_pkg_dir = ""
        self.sandbox_install_dir = ""

        # Include indices revelant to current situation.
        if self.name:
            self.index["INSTALLABILITY"].add_subindex(IndexPyPIDownload())
        if self.url:
            self.index["INSTALLABILITY"].add_subindex(IndexUrlDownload())

        self.determine_pkg_name()
        self.configure_logging(logfile)
        #self.set_defaults()
        #self.get_config()
        self.retrieve_pkg()
        self.unpack_pkg()
        self.walk_pkg()
        self.install_pkg()

    def raise_exception(self, msg):
        """Cleanup, print error message and raise CheesecakeError.

        Don't use logging, since it can be called before logging has been setup.
        """
        self.cleanup()
        os.unlink(os.path.join(self.sandbox, self.logfile))

        msg += "\n" + pad_msg("CHEESECAKE INDEX", 0)
        raise CheesecakeError(msg)
 
    def cleanup(self):
        """Delete temporary directories and files that were created
        in the sandbox. At the end delete the sandbox itself.
        """
        if os.path.isfile(self.sandbox_pkg_file):
            self.log("Removing file %s" % self.sandbox_pkg_file)
            os.unlink(self.sandbox_pkg_file)

        def delete_dir(dirname):
            "Delete directory recursively and generate log message."
            if os.path.isdir(dirname):
                self.log("Removing directory %s" % dirname)
                shutil.rmtree(dirname)

        delete_dir(self.sandbox)

    def set_defaults(self):
        """Set default values for variables that can also be defined
        in the config file.
        """
        pass

    def get_config(self, config_dir=None):
        """Retrieve values from configuration file.
        """
        pass

    def determine_pkg_name(self):
        if self.name:
            self.package = self.name
            self.short_pkg_name = self.name
        elif self.package_path:
            self.package = self.get_package_from_path(self.package_path)
        else:
            self.package = self.get_package_from_url()

    def get_package_from_url(self):
        """Use ``urlparse`` to obtain package path from URL.
        """
        (scheme,location,path,param,query,fragment_id) = urlparse(self.url)
        return self.get_package_from_path(path)

    def get_package_from_path(self, path):
        """Get package name as file portion of path.
        """
        dir, file = os.path.split(path)
        self.short_pkg_name = file
        for package_type in self.package_types:
            s = re.search("(.+)\.%s" % package_type, file)
            if s:
                self.short_pkg_name = s.group(1)
                break
        return file

    def configure_logging(self, logfile=None):
        """Default settings for logging.

        If verbose, log goes to console, else it goes to logfile
        log.debug goes to logfile
        log.info goes to console
        log.warn and log.error go to both logfile and stdout
        """
        if logfile:
            self.logfile = logfile
        else:
            self.logfile = os.path.join(tempfile.gettempdir(), self.short_pkg_name + ".log")

        logger.setconsumer('logfile', open(str(self.logfile), 'w', buffering=1))
        logger.setconsumer('console', logger.STDOUT)
        logger.setconsumer('null', None)

        if self.verbose:
            self.log = logger.MultipleProducer('cheesecake console')
        else:
            self.log = logger.MultipleProducer('cheesecake logfile')
        if self.quiet:
            self.log.info = logger.MultipleProducer('cheesecake logfile')
        else:
            self.log.info = logger.MultipleProducer('cheesecake console')
        self.log.debug = logger.MultipleProducer('cheesecake logfile')
        self.log.warn = logger.MultipleProducer('cheesecake console')
        self.log.error = logger.MultipleProducer('cheesecake console')

        self.log.debug("package = ", self.short_pkg_name)

    def retrieve_pkg(self):
        if self.name:
            self.get_pkg_from_pypi()
        elif self.url:
            self.download_pkg()
        else:
            self.copy_pkg()

    def get_package_from_url(self):
        """Use ``urlparse`` to obtain package path from URL.
        """
        (scheme,location,path,param,query,fragment_id) = urlparse(self.url)
        return self.get_package_from_path(path)        

    def get_package_from_path(self, path):
        """Get package name as file portion of path.
        """
        dir, file = os.path.split(path)
        self.short_pkg_name = file
        for package_type in self.package_types:
            s = re.search("(.+)\.%s" % package_type, file)
            if s:
                self.short_pkg_name = s.group(1)
                break
        return file

    def get_pkg_from_pypi(self):
        """Download package using setuptools utilities.

        :Ivariables:
          download_url : str
              URL that package was downloaded from.
          distance_from_pypi : int
              How many hops setuptools had to make to download package.
          found_on_cheeseshop : bool
              Whenever package has been found on CheeseShop.
        """
        try:
            self.log.info("Trying to download package %s from PyPI using setuptools utilities" % self.name)
            from setuptools.package_index import PackageIndex
            from pkg_resources import Requirement
            from distutils import log

            # Temporarily set the log verbosity to INFO so we can capture setuptools info messages
            old_threshold = log.set_threshold(log.INFO)
            pkgindex = PackageIndex()
            old_stdout = sys.stdout
            sys.stdout = StdoutRedirector()
            output = pkgindex.fetch(Requirement.parse(self.name),
                                    self.sandbox,
                                    force_scan=True,
                                    source=True)
            captured_stdout = sys.stdout.read_buffer()
            sys.stdout = old_stdout
            log.set_threshold(old_threshold)

            if output is None:
                self.raise_exception("Error: Could not find distribution for " + self.name)

            # Defaults.
            self.download_url = ""
            self.distance_from_pypi = 0
            self.found_on_cheeseshop = False

            for line in captured_stdout.split('\n'):
                s = re.search(r"Reading http(.*)", line)
                if s:
                    inspected_url = s.group(1)
                    if not re.search(r"www.python.org\/pypi", inspected_url):
                        self.distance_from_pypi += 1
                    continue
                s = re.search(r"Downloading (.*)", line)
                if s:
                    self.download_url = s.group(1)
                    break

            self.sandbox_pkg_file = output
            self.package = self.get_package_from_path(output)
            self.log.info("Downloaded package %s from %s" % (self.package, self.download_url))

            if re.search(r"cheeseshop.python.org", self.download_url):
                self.found_on_cheeseshop = True

        except ImportError, e:
            msg = "Error: setuptools is not installed and is required for downloading a package by name\n"
            msg += "You can donwload and process a package by its full URL via the -u or --url option\n"
            msg += "Example: python cheesecake.py --url=http://www.mems-exchange.org/software/durus/Durus-3.1.tar.gz"
            self.raise_exception(msg)

    def download_pkg(self):
        """Use ``urllib.urlretrieve`` to download package to file in sandbox dir.
        """
        #self.log("Downloading package %s from URL %s" % (self.package, self.url))
        self.sandbox_pkg_file = os.path.join(self.sandbox, self.package)
        try:
            downloaded_filename, headers = urlretrieve(self.url, self.sandbox_pkg_file)
        except IOError, e:
            self.log.error("Error downloading package %s from URL %s"  % (self.package, self.url))
            self.raise_exception(str(e))
        #self.log("Downloaded package %s to %s" % (self.package, downloaded_filename))

        if re.search("Content-Type: details/html", str(headers)):
            f = open(downloaded_filename)
            if re.search("404 Not Found", "".join(f.readlines())):
                f.close()
                self.raise_exception("Got '404 Not Found' error while trying to download package ... exiting")
            f.close()

        self.downloaded_from_url = True
        
    def copy_pkg(self):
        """Copy package file to sandbox directory.
        """
        self.sandbox_pkg_file = os.path.join(self.sandbox, self.package)
        if not os.path.isfile(self.package_path):
            self.raise_exception("%s is not a valid file ... exiting" % self.package_path)
        self.log("Copying file %s to %s" % (self.package_path, self.sandbox_pkg_file))
        shutil.copyfile(self.package_path, self.sandbox_pkg_file)

    def unpack_pkg(self):
        """Unpack the package in the sandbox directory.
        
        Currently supported archive types:

        * .tar.gz (handled with ``tarfile`` module)
        * .zip (handled with ``zipfile`` module)

        :Ivariables:
          original_package_name : str
        """
        self.package_type = ""

        for type in self.package_types:
            s = re.search(r"(.+)\.%s" % type, self.package)
            if s:
                # package_name is name of package without file extension (ex. twill-7.3)
                self.package_name = s.group(1)
                self.package_type = type
                break
        if not self.package_type:
            msg = "Could not determine package type for package '%s'" % self.package
            msg += "\nCurrently recognized types: " + " ".join(self.package_types)
            self.raise_exception(msg)
        self.log.debug("Package name: " + self.package_name)
        self.log.debug("Package type: " + self.package_type)

        self.sandbox_pkg_dir = os.path.join(self.sandbox, self.package_name)
        if os.path.isdir(self.sandbox_pkg_dir):
            shutil.rmtree(self.sandbox_pkg_dir)

        if self.package_type in ["tar.gz", "tgz"]:
            self.untar_pkg()
        elif self.package_type == "zip":
            self.unzip_pkg()

        if self.unpack_dir != self.package_name:
            self.original_package_name = self.package_name
            self.package_name = self.unpack_dir

        if not self.quiet:
            self.log.info("Detailed info available in log file %s" % self.logfile)

    def untar_pkg(self):
        """Untar the package in the sandbox directory.

        Uses tarfile module.
        """
        try:
            t = tarfile.open(self.sandbox_pkg_file)
        except tarfile.ReadError, e:
            self.raise_exception("Could not read tar file %s ... exiting" % self.sandbox_pkg_file)

        for member in t.getmembers():
            t.extract(member, self.sandbox)

        tarinfo = t.members[0]
        self.unpack_dir = tarinfo.name.split(os.sep)[0]

        self.unpacked = True
            
    def unzip_pkg(self):
        """Unzip the package in the sandbox directory.

        Uses zipfile module.
        """
        try:
            z = zipfile.ZipFile(self.sandbox_pkg_file)
        except zipfile.error:
            self.raise_exception("Error unzipping file %s ... exiting" % self.sandbox_pkg_file)

        # Get directory structure from zip and create it in sandbox
        for name in z.namelist():
            (dir, file) = os.path.split(name)
            unpack_dir = dir
            target_dir = os.path.join(self.sandbox, dir)
            if not os.path.exists(target_dir):
                os.makedirs(target_dir)

        # Extract files to directory structure
        for i, name in enumerate(z.namelist()):
            if not name.endswith('/'):
                outfile = open(os.path.join(self.sandbox, name), 'wb')
                outfile.write(z.read(name))
                outfile.flush()
                outfile.close()

        self.unpack_dir = unpack_dir.split(os.sep)[0]

        self.unpacked = True

    def walk_pkg(self):
        """Get package files and directories.

        :Ivariables:
          dirs_list : list
              List of directories package contains.
          docstring_cnt : int
              Number of docstrings found in all package objects.
          docformat_cnt : int
              Number of formatted docstrings found in all package objects.
          files_list : list
              List of files package contains.
          functions : list
              List of all functions defined in package sources.
          object_cnt : int
              Number of documentable objects found in all package modules.
          package_dir : str
              Path to project directory.
        """
        self.package_dir = os.path.join(self.sandbox, self.package_name)

        self.files_list, self.dirs_list = get_files_dirs_list(self.package_dir)

        self.object_cnt = 0
        self.docstring_cnt = 0
        self.docformat_cnt = 0
        self.functions = []

        # Parse all application files and count objects
        # (modules/classes/functions) and their associated docstrings.
        for py_file in get_files_of_type(self.files_list, 'module'):
            pyfile = os.path.join(self.package_dir, py_file)
            code = CodeParser(pyfile, self.log.debug)

            self.object_cnt += code.object_count()
            self.docstring_cnt += code.docstring_count()
            self.docformat_cnt += code.formatted_docstrings_count
            self.functions += code.functions

        # Log a bit of debugging info.
        self.log.debug("Found %d files: %s." % (len(self.files_list),
                                                ', '.join(self.files_list)))
        self.log.debug("Found %d directories: %s." % (len(self.dirs_list),
                                                      ', '.join(self.dirs_list)))

    def install_pkg(self):
        """Verify that package can be installed in alternate directory.

        :Ivariables:
          installed : bool
              Describes whenever package has been succefully installed.
        """
        self.sandbox_install_dir = os.path.join(self.sandbox, "tmp_install_%s" % self.package_name)

        cwd = os.getcwd()
        os.chdir(os.path.join(self.sandbox, self.package_name))

        rc, output = run_cmd("python setup.py install --root=" + self.sandbox_install_dir)

        # Install succeeded
        if not rc:
            self.installed = True

        os.chdir(cwd)

    def compute_cheesecake_index(self):
        """Compute overall Cheesecake index for the package by adding up
        specific indexes.
        """
        # Recursively compute all indices.
        max_cheesecake_index = self.index.max_value

        # Pass Cheesecake instance to the main Index object.
        cheesecake_index = self.index.compute_with(self)
        percentage = (cheesecake_index * 100) / max_cheesecake_index

        self.log.info("A given package can currently reach a MAXIMUM number of %d points" % max_cheesecake_index)
        self.log.info("Starting computation of Cheesecake index for package '%s'" % (self.package))

        # Print summary.
        print
        print pad_line("=")
        print pad_msg("OVERALL CHEESECAKE INDEX (ABSOLUTE)", cheesecake_index)
        print "%s (%d out of a maximum of %d points is %d%%)" % \
              (pad_msg("OVERALL CHEESECAKE INDEX (RELATIVE)", percentage),
               cheesecake_index,
               max_cheesecake_index,
               percentage)

        return cheesecake_index


def process_cmdline_args():
    """
    Parse command-line options
    """
    parser = OptionParser()
    parser.add_option("-n", "--name", dest="name",
                      default="", help="package name (will be retrieved via setuptools utilities, if present)")
    parser.add_option("-u", "--url", dest="url",
                      default="", help="package URL")
    parser.add_option("-p", "--path", dest="path",
                      default="", help="path of tar.gz/zip package on local file system")
    parser.add_option("-s", "--sandbox", dest="sandbox",
                      default=None,
                      help="directory where package will be unpacked "\
                           "(default is to use random directory inside %s)" % tempfile.gettempdir())
    parser.add_option("-c", "--config", dest="config",
                      default=None,
                      help="directory with custom configuration (default=~/.cheesecake)")
    parser.add_option("-l", "--logfile", dest="logfile",
                      default=None,
                      help="file to log all cheesecake messages")
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
                      default=False, help="verbose output (default=False)")
    parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
                      default=False, help="only print Cheesecake index value (default=False)")

    (options, args) = parser.parse_args()
    return options

def main():
    """
    Display Cheesecake index for package specified via command-line options
    """
    options = process_cmdline_args()
    name = options.name
    url = options.url
    path = options.path
    sandbox = options.sandbox
    config = options.config
    logfile = options.logfile
    verbose = options.verbose
    quiet = options.quiet

    if not name and not url and not path:
        print "Error: No package name, URL or path specified (see --help)"
        sys.exit(1)

    try:
        c = Cheesecake(name=name, url=url, path=path, sandbox=sandbox,
                       config=config, logfile=logfile, verbose=verbose,
                       quiet=quiet)
        c.compute_cheesecake_index()
        c.cleanup()
    except CheesecakeError, e:
        print str(e)

if __name__ == "__main__":
    main()