root/branches/mk/cheesecake/cheesecake_index.py

#!/usr/bin/env python
"""
Cheesecake: How tasty is your code?

The idea of the Cheesecake project is to rank Python packages
based on various empiric "kwalitee" factors, such as:

        * whether the package can be downloaded
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
        * existence of certain files such as README, INSTALL, LICENSE, setup.py etc.
        * existence of certain directories such as doc, test, demo, examples
        * percentage of modules/functions/classes/methods with docstrings
        * percentage of functions/methods that are unit tested
        * average pylint score for all non-test and non-demo modules
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
"""

import os, sys, re, shutil
import tarfile, zipfile
import tempfile
from optparse import OptionParser
from urllib import urlretrieve
from urlparse import urlparse
from math import ceil

from _util import run_cmd, pad_with_dots, pad_left_spaces, pad_msg, pad_line
from _util import StdoutRedirector
import logger
from config import get_pkg_config
from codeparser import CodeParser

default_temp_directory = os.path.join(tempfile.gettempdir(), 'cheesecake_sandbox')


class Index(object):
    """
    Encapsulates index attributes such as name, value, details
    """

    def __init__(self, type, name="", value=0, details=""):
        self.type = "index_" + type
        self.name = self.type
        if name: self.name += "_" + name
        self.value = value
        self.details = details
        
    def print_info(self):
        """
        Print index name padded with dots, followed by value and details
        """
        msg = pad_with_dots(self.name)
        msg += pad_left_spaces(self.value)
        msg += " (" + self.details + ")"
        print msg

class CompositeIndex(object):
    """
    Collection of indexes of same type (e.g. files, dirs)
    """

    def __init__(self, type):
        """
        Indexes is a dict mapping names to Index objects
        """
        self.type = type
        self.indexes = {}

    def set_index(self, name, value=0, details=""):
        """
        Create new index or update existing index with specified attributes
        """
        if self.indexes.has_key(name):
            index = self.indexes[name]
            index.value = value
            index.details = details
        else:
            self.indexes[name] = Index(self.type, name, value, details)
        
    def print_info(self):
        """
        Print index info for all indexes sorted alphanumerically by name
        """
        names = self.indexes.keys()
        names.sort()
        for name in names:
            index = self.indexes[name]
            index.print_info()

    def get_value(self):
        """
        Return sum of individual index values
        """
        value = 0
        for key in self.indexes.keys():
            index = self.indexes[key]
            value += index.value
        return value

    value = property(get_value)

class CheesecakeError(Exception):
    """
    Custom exception class for Cheesecake-specific errors
    """
    pass

class Cheesecake(object):
    """
    Computes 'goodness' of Python packages

    Generates "cheesecake index" that takes into account things like:

        * whether the package can be downloaded
        * whether the package can be unpacked
        * whether the package can be installed into an alternate directory
        * existence of certain files such as README, INSTALL, LICENSE, setup.py etc.
        * existence of certain directories such as doc, test, demo, examples
        * percentage of modules/functions/classes/methods with docstrings
        * percentage of functions/methods that are unit tested 
        * average pylint score for all non-test and non-demo modules
    """

    def __init__(self, name="", url="", path="", sandbox=None, config=None,
                logfile=None, verbose=False, quiet=False):
        """
        Initialize critical variables, download and unpack package, walk package tree

        """
        self.name = name
        self.url = url
        self.package_path = path
        if not self.name and not self.url and not self.package_path:
            self.raise_exception("No package name, URL or path specified ... exiting")
        self.sandbox = sandbox or default_temp_directory
        if not os.path.isdir(self.sandbox):
            os.mkdir(self.sandbox)
        self.config = config
        self.verbose = verbose
        self.quiet = quiet

        self.package_types = ["tar.gz", "tgz", "zip"]
        self.sandbox_pkg_file = ""
        self.sandbox_pkg_dir = ""
        self.sandbox_install_dir = ""

        self.determine_pkg_name()
        self.configure_logging(logfile)
        self.set_defaults()
        self.get_config()
        self.init_indexes()
        self.retrieve_pkg()
        self.unpack_pkg()
        self.walk_pkg()

    def raise_exception(self, msg):
        """
        Cleanup, print error message and raise CheesecakeError

        Don't use logging, since it can be called before logging has been setup
        """
        self.cleanup()
        os.unlink(os.path.join(self.sandbox, self.logfile))

        msg += "\n" + pad_msg("CHEESECAKE INDEX", 0)
        raise CheesecakeError(msg)
 
    def cleanup(self):
        """
        Delete temporary directories and files that were
        created in the sandbox. At the end delete the sandbox itself.
        """
        if os.path.isfile(self.sandbox_pkg_file):
            self.log("Removing file %s" % self.sandbox_pkg_file)
            os.unlink(self.sandbox_pkg_file)

        def delete_dir(dirname):
            "Delete directory recursively and generate log message."
            if os.path.isdir(dirname):
                self.log("Removing directory %s" % dirname)
                shutil.rmtree(dirname)

        for dir in [self.sandbox_pkg_dir, self.sandbox_install_dir,
                    self.sandbox]:
            delete_dir(dir)

    def set_defaults(self):
        """
        Set default values for variables that can also be defined
        in the config file
        """
        self.INDEX_PYPI_DOWNLOAD = 50
        self.INDEX_PYPI_DISTANCE = 5
        self.INDEX_URL_DOWNLOAD  = 25
        self.INDEX_UNPACK        = 25
        self.INDEX_UNPACK_DIR    = 15
        self.INDEX_INSTALL       = 50
        self.INDEX_FILE_CRITICAL = 15
        self.INDEX_FILE          = 10
        self.INDEX_REQUIRED_FILES = 100
        self.INDEX_FILE_PYC      = -20
        self.INDEX_DIR_CRITICAL  = 25
        self.INDEX_DIR           = 20
        self.INDEX_DIR_EMPTY     = 5
        self.MAX_INDEX_DOCSTRINGS = 100 # max. percentage of modules/classes/methods/functions with docstrings
        self.MAX_INDEX_UNITTESTS  = 100 # max. percentage of methods/functions that are unit tested
        self.MAX_INDEX_PYLINT     = 100 # max. pylint score
        self.cheese_files = ["readme", "install", "changelog",
                            "news", "faq",
                            "todo", "thanks",
                            "license", "announce",
                            "setup.py",
                            ]
        self.critical_cheese_files = ["readme", "license", "setup.py"]
        self.cheese_dirs = ["doc", "test", "example", "demo"]
        self.critical_cheese_dirs = ["doc", "test"]

    def get_config(self, config_dir=None):
        """
        Retrieve values from configuration file
        """
        self.config = get_pkg_config(self.short_pkg_name, config_dir)
        for config_var in ["INDEX_PYPI_DOWNLOAD", "INDEX_PYPI_DISTANCE",
            "INDEX_URL_DOWNLOAD", "INDEX_UNPACK", "INDEX_UNPACK_DIR",
            "INDEX_INSTALL", "INDEX_FILE_CRITICAL", "INDEX_FILE",
            "INDEX_REQUIRED_FILES", "INDEX_FILE_PYC",
            "INDEX_DIR_CRITICAL", "INDEX_DIR", "INDEX_DIR_EMPTY",
            "MAX_INDEX_DOCSTRINGS", "MAX_INDEX_PYLINT",
            "cheese_files", "critical_cheese_files", 
            "cheese_dirs", "critical_cheese_dirs",
            ]:
            value = self.config.get(config_var)
            if value: setattr(self, config_var, value)

    def determine_pkg_name(self):
        if self.name:
            self.package = self.name
            self.short_pkg_name = self.name
        elif self.package_path:
            self.package = self.get_package_from_path(self.package_path)
        else:
            self.package = self.get_package_from_url()

    def get_package_from_url(self):
        """
        Use ``urlparse`` to obtain package path from URL
        """
        (scheme,location,path,param,query,fragment_id) = urlparse(self.url)
        return self.get_package_from_path(path)


    def get_package_from_path(self, path):
        """
        Get package name as file portion of path
        """
        dir, file = os.path.split(path)
        self.short_pkg_name = file
        for package_type in self.package_types:
            s = re.search("(.+)\.%s" % package_type, file)
            if s:
                self.short_pkg_name = s.group(1)
                break
        return file

    def configure_logging(self, logfile=None):
        """
        Default settings for logging

        if verbose, log goes to console, else it goes to logfile
        log.debug goes to logfile
        log.info goes to console
        log.warn and log.error go to both logfile and stdout
        """
        if logfile:
            self.logfile = logfile
        else:
            self.logfile = os.path.join(tempfile.gettempdir(), self.short_pkg_name + ".log")

        logger.setconsumer('logfile', open(str(self.logfile), 'w', buffering=1))
        logger.setconsumer('console', logger.STDOUT)
        logger.setconsumer('null', None)

        if self.verbose:
            self.log = logger.MultipleProducer('cheesecake console')
        else:
            self.log = logger.MultipleProducer('cheesecake logfile')
        if self.quiet:
            self.log.info = logger.MultipleProducer('cheesecake logfile')
        else:
            self.log.info = logger.MultipleProducer('cheesecake console')
        self.log.debug = logger.MultipleProducer('cheesecake logfile')
        self.log.warn = logger.MultipleProducer('cheesecake console')
        self.log.error = logger.MultipleProducer('cheesecake console')

        self.log.debug("package = ", self.short_pkg_name)

    def init_indexes(self):
        """
        Initialize variables used in index computation

        * cheesecake_index: overall index for the package
        * index: dict holding Index or CompositeIndex objects of various types
        """
        self.cheesecake_index = 0
        self.cheesecake_index_installability = 0
        self.cheesecake_index_documentation = 0
        self.cheesecake_index_codekwalitee = 0
        self.max_cheesecake_index = self.INDEX_PYPI_DOWNLOAD + \
                                    self.INDEX_UNPACK + \
                                    self.INDEX_UNPACK_DIR + \
                                    self.INDEX_INSTALL + \
                                    self.MAX_INDEX_DOCSTRINGS + \
                                    self.MAX_INDEX_PYLINT 
#                                    self.MAX_INDEX_UNITTESTS
        self.max_cheesecake_index_installability = self.INDEX_PYPI_DOWNLOAD + \
                                            self.INDEX_UNPACK + \
                                            self.INDEX_UNPACK_DIR + \
                                            self.INDEX_INSTALL
        self.max_cheesecake_index_documentation = self.INDEX_REQUIRED_FILES + \
                                        self.MAX_INDEX_DOCSTRINGS
        self.max_cheesecake_index_codekwalitee = self.MAX_INDEX_PYLINT 
#                                        self.MAX_INDEX_UNITTESTS
                                        
        self.index = {}
        for index_type in ["file", "dir"]:
            self.index[index_type] = CompositeIndex(index_type)
        for index_type in ["pypi_download", "url_download", 
                           "unpack_dir", "unpack", "install",
                           "docstrings", "unittests", "pylint"]:
            self.index[index_type] = Index(index_type)

        for cheese_file in self.cheese_files:
            self.index["file"].set_index(name=cheese_file, details="file not found")
            if cheese_file in self.critical_cheese_files:
                self.max_cheesecake_index += self.INDEX_FILE_CRITICAL
                self.max_cheesecake_index_documentation += self.INDEX_FILE_CRITICAL
            else:
                self.max_cheesecake_index += self.INDEX_FILE
                self.max_cheesecake_index_documentation += self.INDEX_FILE
        self.log.debug("cheese_files: " + ",".join(self.cheese_files))
        self.log.debug("critical_cheese_files: " + ",".join(self.critical_cheese_files))

        for cheese_dir in self.cheese_dirs:
            self.index["dir"].set_index(name=cheese_dir, details="directory not found")
            if cheese_dir in self.critical_cheese_dirs:
                self.max_cheesecake_index += self.INDEX_DIR_CRITICAL
                self.max_cheesecake_index_documentation += self.INDEX_DIR_CRITICAL
            else:
                self.max_cheesecake_index += self.INDEX_DIR
                self.max_cheesecake_index_documentation += self.INDEX_DIR
        self.log.debug("cheese_dirs: " + ",".join(self.cheese_dirs))
        self.log.debug("critical_cheese_dirs: " + ",".join(self.critical_cheese_dirs))

        self.pkg_files = {}
        self.pkg_dirs = {}
        self.file_types =   ["py", "pyc", "test",
                            ]
        for type in self.file_types:
            self.pkg_files[type] = []

        self.object_cnt = 0  # Number of modules/functions/classes/methods in .py files found
        self.docstring_cnt = 0
        self.functions = [] # List of methods/functions found in .py files

    def retrieve_pkg(self):
        if self.name:
            self.get_pkg_from_pypi()
        elif self.url:
            self.download_pkg()
        else:
            self.copy_pkg()

    def get_package_from_url(self):
        """
        Use ``urlparse`` to obtain package path from URL
        """
        (scheme,location,path,param,query,fragment_id) = urlparse(self.url)
        return self.get_package_from_path(path)
        

    def get_package_from_path(self, path):
        """
        Get package name as file portion of path
        """
        dir, file = os.path.split(path)
        self.short_pkg_name = file
        for package_type in self.package_types:
            s = re.search("(.+)\.%s" % package_type, file)
            if s:
                self.short_pkg_name = s.group(1)
                break
        return file

    def get_pkg_from_pypi(self):
        """
        Download package using setuptools utilities
        """
        try:
            self.log.info("Trying to download package %s from PyPI using setuptools utilities" % self.name)
            from setuptools.package_index import PackageIndex
            from pkg_resources import Requirement
            from distutils import log
            # Temporarily set the log verbosity to INFO so we can capture setuptools info messages
            old_threshold = log.set_threshold(log.INFO)
            pkgindex = PackageIndex()
            old_stdout = sys.stdout
            sys.stdout = StdoutRedirector()
            output = pkgindex.fetch(Requirement.parse(self.name),
                                    self.sandbox,
                                    force_scan=True,
                                    source=True)
            captured_stdout = sys.stdout.read_buffer()
            sys.stdout = old_stdout
            log.set_threshold(old_threshold)
            if output is None:
                self.raise_exception("Error: Could not find distribution for " + self.name)
            download_url = ""
            distance_from_pypi = 0
            #print captured_stdout
            for line in captured_stdout.split('\n'):
                s = re.search(r"Reading http(.*)", line)
                if s:
                    inspected_url = s.group(1)
                    if not re.search(r"www.python.org\/pypi", inspected_url):
                        distance_from_pypi += 1
                    continue
                s = re.search(r"Downloading (.*)", line)
                if s:
                    download_url = s.group(1)
                    break
            self.sandbox_pkg_file = output
            self.package = self.get_package_from_path(output)
            self.log.info("Downloaded package %s from %s" % (self.package, download_url))
            index_type = "pypi_download"
            found_on_cheeseshop = False
            if re.search(r"cheeseshop.python.org", download_url):
                value = self.INDEX_PYPI_DOWNLOAD
                found_on_cheeseshop = True
            else:
                value = self.INDEX_PYPI_DOWNLOAD - distance_from_pypi * self.INDEX_PYPI_DISTANCE
            self.index[index_type].value = value
            details = "downloaded package " + self.package
            if found_on_cheeseshop:
                details += " directly from the Cheese Shop"
            elif distance_from_pypi:
                details += " following %d link" % distance_from_pypi
                if distance_from_pypi > 1:
                    details += "s"
                details += " from PyPI"
            else:
                details += "from " + download_url
            self.index[index_type].details = details
        except ImportError, e:
            msg = "Error: setuptools is not installed and is required for downloading a package by name\n"
            msg += "You can donwload and process a package by its full URL via the -u or --url option\n"
            msg += "Example: python cheesecake.py --url=http://www.mems-exchange.org/software/durus/Durus-3.1.tar.gz"
            self.raise_exception(msg)
        
    def download_pkg(self):
        """
        Use ``urllib.urlretrieve`` to download package to file in sandbox dir
        """
        #self.log("Downloading package %s from URL %s" % (self.package, self.url))
        self.sandbox_pkg_file = os.path.join(self.sandbox, self.package)
        try:
            downloaded_filename, headers = urlretrieve(self.url, self.sandbox_pkg_file)
        except IOError, e:
            self.log.error("Error downloading package %s from URL %s"  % (self.package, self.url))
            self.raise_exception(str(e))
        #self.log("Downloaded package %s to %s" % (self.package, downloaded_filename))
        if re.search("Content-Type: details/html", str(headers)):
            f = open(downloaded_filename)
            if re.search("404 Not Found", "".join(f.readlines())):
                f.close()
                self.raise_exception("Got '404 Not Found' error while trying to download package ... exiting")
            f.close()
        index_type = "url_download"
        self.index[index_type].value = self.INDEX_URL_DOWNLOAD
        self.index[index_type].details = "downloaded package %s from URL %s"  % (self.package, self.url)
        
    def copy_pkg(self):
        """
        Copy package file to sandbox directory
        """
        self.sandbox_pkg_file = os.path.join(self.sandbox, self.package)
        if not os.path.isfile(self.package_path):
            self.raise_exception("%s is not a valid file ... exiting" % self.package_path)
        self.log("Copying file %s to %s" % (self.package_path, self.sandbox_pkg_file))
        shutil.copyfile(self.package_path, self.sandbox_pkg_file)

    def unpack_pkg(self):
        """
        Unpack the package in the sandbox directory
        
        Currently supported archive types:

        * .tar.gz (handled with ``tarfile`` module)
        * .zip (handled with ``zipfile`` module)
        """
        self.package_type = ""
        for type in self.package_types:
            s = re.search(r"(.+)\.%s" % type, self.package)
            if s:
                # package_name is name of package without file extension (ex. twill-7.3)
                self.package_name = s.group(1)
                self.package_type = type
                break
        if not self.package_type:
            msg = "Could not determine package type for package '%s'" % self.package
            msg += "\nCurrently recognized types: " + " ".join(self.package_types)
            self.raise_exception(msg)
        self.log.debug("Package name: " + self.package_name)
        self.log.debug("Package type: " + self.package_type)

        self.sandbox_pkg_dir = os.path.join(self.sandbox, self.package_name)
        if os.path.isdir(self.sandbox_pkg_dir):
            shutil.rmtree(self.sandbox_pkg_dir)

        if self.package_type in ["tar.gz", "tgz"]:
            self.untar_pkg()
        elif self.package_type == "zip":
            self.unzip_pkg()

        index_type = "unpack_dir"
        details = "unpack directory is " + self.unpack_dir
        if self.unpack_dir != self.package_name:
            details += " instead of the expected " + self.package_name
            self.package_name = self.unpack_dir
        else:
            details += " as expected"
            self.index[index_type].value = self.INDEX_UNPACK_DIR
        self.index[index_type].details = details

        if not self.quiet:
            self.log.info("Detailed info available in log file %s" % self.logfile)

    def untar_pkg(self):
        """
        Untar the package in the sandbox directory

        Uses tarfile module
        """
        try:
            t = tarfile.open(self.sandbox_pkg_file)
        except tarfile.ReadError, e:
            self.raise_exception("Could not read tar file %s ... exiting" % self.sandbox_pkg_file)

        for member in t.getmembers():
            t.extract(member, self.sandbox)

        tarinfo = t.members[0]
        self.unpack_dir = tarinfo.name.split(os.sep)[0]

        index_type = "unpack"
        self.index[index_type].value = self.INDEX_UNPACK
        self.index[index_type].details = "package untar-ed successfully"
            
    def unzip_pkg(self):
        """
        Unzip the package in the sandbox directory

        Uses zipfile module
        """
        try:
            z = zipfile.ZipFile(self.sandbox_pkg_file)
        except zipfile.error:
            self.raise_exception("Error unzipping file %s ... exiting" % self.sandbox_pkg_file)

        # Get directory structure from zip and create it in sandbox
        for name in z.namelist():
            (dir, file) = os.path.split(name)
            unpack_dir = dir
            target_dir = os.path.join(self.sandbox, dir)
            if not os.path.exists(target_dir):
                os.makedirs(target_dir)

        # Extract files to directory structure
        for i, name in enumerate(z.namelist()):
            if not name.endswith('/'):
                outfile = open(os.path.join(self.sandbox, name), 'wb')
                outfile.write(z.read(name))
                outfile.flush()
                outfile.close()

        self.unpack_dir = unpack_dir.split(os.sep)[0]

        index_type = "unpack"
        self.index[index_type].value = self.INDEX_UNPACK
        self.index[index_type].details = "package unzipped successfully"

    def walk_pkg(self):
        """
        Traverse the file system tree rooted at sandbox/package_name

        * Compute indexes for special files and directories
        * Identify Python files, test files, etc.
        """
        cwd = os.getcwd()
        os.chdir(self.sandbox)
        for rootdir, dirs, files in os.walk(self.package_name):
            head, tail = os.path.split(rootdir)
            dirs_in_rootdir = rootdir.split(os.path.sep)
            for cheese_dir in self.cheese_dirs:
                if re.search("^%s" % cheese_dir, tail):
                    if files or dirs:
                        if cheese_dir in self.critical_cheese_dirs:
                            value = self.INDEX_DIR_CRITICAL
                            details = "critical directory found"
                            self.log.debug("critical_cheese_dir found: " + cheese_dir)
                        else:
                            value = self.INDEX_DIR
                            details = "directory found"
                            self.log.debug("cheese_dir found: " + cheese_dir)
                    else:
                        value = self.INDEX_DIR_EMPTY
                        details = "empty directory found"
                        self.log.debug("empty cheese_dir found: " + cheese_dir)
                    self.index["dir"].set_index(cheese_dir, value, details)
            for file in files:
                fullpath = os.path.join(rootdir, file)
                for cheese_file in self.cheese_files:
                    if re.search(r"^%s(\.txt)*" % cheese_file, file, re.IGNORECASE):
                        if cheese_file in self.critical_cheese_files:
                            value = self.INDEX_FILE_CRITICAL
                            details = "critical file found"
                            self.log.debug("critical_cheese_file found: " + cheese_file)
                        else:
                            value = self.INDEX_FILE
                            details = "file found"
                            self.log.debug("cheese_file found: " + cheese_file)
                        self.index["file"].set_index(cheese_file, value, details)

                if self.is_py_file(file, dirs_in_rootdir):
                    self.pkg_files["py"].append(fullpath)
                    self.log.debug("py file found: " + fullpath)
                    pyfile = os.path.join(self.sandbox, fullpath)
                    # Parse the file and count objects (modules/classes/functions)
                    # and their associated docstrings
                    code = CodeParser(pyfile, self.log.debug)
                    self.object_cnt += code.object_count()
                    self.docstring_cnt += code.docstring_count()
                    self.functions += code.functions

                if os.path.splitext(file)[1] == ".pyc":
                    self.pkg_files["pyc"].append(fullpath)
                    self.log.debug("pyc file found: " + fullpath)

                if self.is_test_file(file, dirs_in_rootdir):
                    self.pkg_files["test"].append(fullpath)
                    self.log.debug("test file found: " + fullpath)

        len_pyc_list = len(self.pkg_files["pyc"])
        if len_pyc_list:
            self.index["file"].set_index("pyc", value=self.INDEX_FILE_PYC,
                details="%d .pyc files found" % len_pyc_list)
        self.log.debug("Found %d py files" % len(self.pkg_files["py"]))
        self.log.debug("Found %d pyc files" % len(self.pkg_files["pyc"]))
        self.log.debug("Found %d test files" % len(self.pkg_files["test"]))

        os.chdir(cwd)

    def is_py_file(self, file, dirs):
        """
        Return True if file ends with .py and it is not a special file and it is not
        in special directory
        """
        if os.path.splitext(file)[1] != ".py":
            return False
        if file in ["setup.py", "ez_setup.py", "__init__.py", "__pkginfo__.py"]:
            return False
        for dir in dirs:
            if dir.startswith("test") or \
                dir.startswith("docs") or \
                dir.startswith("demo") or \
                dir.startswith("example"):
                return False
        return True

    def is_test_file(self, file, dirs):
        """
        Return True is file is in directory rooted at "test" or "tests"
        """
        if not file.endswith(".py"):
            return False
        if file in ["__init__.py"]:
            return False
        for dir in dirs:
            if dir.startswith("test"):
                return True
        return False

    def index_file(self):
        """
        Return CompositeIndex object of type "file"
        """
        return self.index["file"]

    def index_dir(self):
        """
        Return CompositeIndex object of type "dir"
        """
        return self.index["dir"]

    def index_pypi_download(self):
        """
        Verify that package can be downloaded from PyPI

        Return Index object of type "pypi_download"
        """
        index_type = "pypi_download"
        if self.url:
            # Package was downloaded directly from URL
            self.index[index_type].value = 0
            self.index[index_type].details = "package was downloaded directly from URL"

        if self.package_path:
            # Package was processed from file system path
            self.index[index_type].value = 0
            self.index[index_type].details = "package was processed from file system path"
            
        # Otherwise, index["pypi_download"] was already set in get_pkg_from_pypi()
        return self.index["pypi_download"]

    def index_url_download(self):
        """
        Verify that package can be downloaded from an URL

        Return Index object of type "download"
        """
        # index["download"] is already set in download_pkg()
        return self.index["url_download"]

    def index_unpack(self):
        """
        Verify that package can be unpacked

        Return Index object of type "unpack"
        """
        # index["unpack"] is already set in unpack_pkg()
        return self.index["unpack"]


    def index_unpack_dir(self):
        """
        Verify that unpack directory has same name as package

        Return Index object of type "unpack_dir"
        """
        # index["unpack_dir"] is already set in unpack_pkg()
        return self.index["unpack_dir"]

    def index_install(self):
        """
        Verify that package can be installed in alternate directory

        Return Index object of type "install"
        """
        index_type = "install"
        self.sandbox_install_dir = os.path.join(self.sandbox, "tmp_install_%s" % self.package_name)
        cwd = os.getcwd()
        os.chdir(os.path.join(self.sandbox, self.package_name))
        rc, output = run_cmd("python setup.py install --root=" + self.sandbox_install_dir)
        if not rc:
            # Install succeeded
            self.index[index_type].value = self.INDEX_INSTALL
            self.index[index_type].details = details="package installed in %s" % self.sandbox_install_dir
        else:
            # Install failed
            self.index[index_type].details = "could not install package in %s" % self.sandbox_install_dir
        os.chdir(cwd)
        return self.index[index_type]

    def index_docstrings(self):
        """
        Compute docstring index as percentage of modules/classes/methods/functions
        that have docstrings associated with them

        Return Index object of type "docstrings"
        """
        index_type = "docstrings"
        if self.object_cnt:
            percent = float(self.docstring_cnt)/float(self.object_cnt)
        else:
            percent = 0
        index_value = int(ceil(percent*100))
        details = "found %d/%d=%.2f%% modules/classes/methods/functions with docstrings" %\
                 (self.docstring_cnt, self.object_cnt, percent*100)
        self.index[index_type].value = index_value
        self.index[index_type].details = details
        return self.index[index_type]

    def index_unittests(self):
        """
        Compute unittest index as percentage of methods/functions
        that are exercised in unit tests

        Return Index object of type "unittests"
        """
        unittest_cnt = 0
        index_type = "unittests"
        self.functions_tested = {}
        for testfile in self.pkg_files["test"]:
            fullpath = os.path.join(self.sandbox, testfile)
            code = CodeParser(fullpath, self.log.debug)
            func_called = code.functions_called()
            self.log.debug("Functions called in unit test:")
            self.log.debug(func_called)
            for func in func_called:
                self.functions_tested[func] = 1
        self.log.debug("FUNCTIONS TO BE CHECKED WHETHER THEY ARE UNIT TESTED:")
        self.log.debug(self.functions)
        self.log.debug("FUNCTIONS THAT ARE UNIT TESTED:")
        self.log.debug(self.functions_tested.keys())
        for funcname in self.functions:
            if self.is_unit_tested(funcname):
                unittest_cnt += 1
                self.log.debug("%s is unit tested" % funcname)
        cnt = len(self.functions)
        if cnt:
            percent = float(unittest_cnt)/float(cnt)
        else:
            percent = 0
        index_value = int(ceil(percent*100))
        details = "found %d/%d=%.2f%% unit tested methods/functions" % (unittest_cnt, cnt, percent*100)
        self.index[index_type].value = index_value
        self.index[index_type].details = details
        return self.index[index_type]

    def is_unit_tested(self, funcname):
        elem = funcname.split(".")
        n1 = elem[-1]
        n2 = ""
        if len(elem) > 1:
            n2 = elem[-2] + "." + elem[-1]
        for key in self.functions_tested.keys():
            if key.startswith(n1) or (n2 and key.startswith(n2)):
                return True
        return False

    def index_pylint(self):
        """
        Compute pylint index as average of positive pylint scores obtained for
        the Python files identified in the package

        Return Index object of type "pylint"
        """
        index_type = "pylint"
            # Try to run the pylint script
        rc, output = run_cmd("pylint --version")
        if rc:
            # We encountered an error
            self.index[index_type].details = "pylint not properly installed"
            return self.index[index_type]
        index_pylint = 0
        cnt = 0
        for pyfile in self.pkg_files["py"]:
            (path, filename) = os.path.split(pyfile)
            (module, ext) = os.path.splitext(filename)
            if module == "setup" or module == "ez_setup" or module.startswith("__"):
                continue
            fullpath = os.path.join(self.sandbox, pyfile)
            self.log.debug("Running pylint on file " + fullpath)
            rc, output = run_cmd("pylint " + fullpath)
            if rc:
                # We encountered an error
                continue
            score_line = output.split("\n")[-3]
            s = re.search(r" (\d+\.\d+)/10", score_line)
            # We only take positive scores into account
            if s:
                score = s.group(1)
                self.log.debug("pylint score for module %s: %s" % (module, score))
                if score == "0.00":
                    self.log.debug("Ignoring scores of 0.00")
                    continue
                index_pylint += float(score)
                cnt += 1
        if cnt:
            avg_value = float(index_pylint)/float(cnt)
        else:
            avg_value = 0
        index_value = int(ceil(avg_value*10))
        self.index[index_type].value = index_value
        self.index[index_type].details = "average score is %.2f out of 10" % avg_value
        return self.index[index_type]

    def compute_cheesecake_index(self):
        """
        Compute overall Cheesecake index for the package by adding up 
        specific indexes
        """
        self.log.info("A given package can currently reach a MAXIMUM number of %d points" % self.max_cheesecake_index)
        self.log.info("Starting computation of Cheesecake index for package '%s'" % (self.package))

        index_types = []
        #if self.name:
        #    index_types.append("pypi_download")
        index_types.append("pypi_download")
        if self.url:
            index_types.append("url_download")
        index_types += ["unpack", "unpack_dir", "install"]
        self.cheesecake_index_installability = self.process_partial_index("INSTALLABILITY",\
                                         index_types, self.max_cheesecake_index_installability)

        index_types = ["file", "dir", "docstrings"]
        self.cheesecake_index_documentation = self.process_partial_index("DOCUMENTATION",\
                                         index_types, self.max_cheesecake_index_documentation)

        index_types = [ 
                        #"unittests",
                        "pylint",
                        ]
        self.cheesecake_index_codekwalitee = self.process_partial_index("CODE KWALITEE",\
                                         index_types, self.max_cheesecake_index_codekwalitee)

        print
        self.print_separator_line("=")
        print pad_msg("OVERALL CHEESECAKE INDEX (ABSOLUTE)", self.cheesecake_index)
        percentage = (self.cheesecake_index * 100) / self.max_cheesecake_index
        msg = pad_msg("OVERALL CHEESECAKE INDEX (RELATIVE)", percentage)
        msg += " (%d out of a maximum of %d points is %d%%)" %\
             (self.cheesecake_index, self.max_cheesecake_index, percentage)
        print msg
        self.cleanup()

        return self.cheesecake_index

    def process_partial_index(self, partial_index_name, index_types, max_value):
        print
        self.log.info("Starting computation of %s index (max. points = %d)" % \
                            (partial_index_name, max_value))
        partial_index_value = 0
        for index_type in index_types:
            partial_index_value += self.process_index(index_type)

        self.print_separator_line()
        print pad_msg("%s INDEX (ABSOLUTE)" % partial_index_name, partial_index_value)
        percentage = (partial_index_value * 100) / max_value
        msg = pad_msg("%s INDEX (RELATIVE)" % partial_index_name, percentage)
        msg += " (%d out of a maximum of %d points is %d%%)" %\
             (partial_index_value, max_value, percentage)
        print msg
        return partial_index_value

    def process_index(self, index_type):
        """
        Compute and print index of specified type
        """
        index = self.index[index_type]
        index_method = "index_" + index_type
        getattr(self, index_method)()
        if not self.quiet:
            index.print_info()
        self.cheesecake_index += index.value
        return index.value

    def print_separator_line(self, char="-"):
        """
        Print line of text, unless quiet flag was given
        """
        if self.quiet:
            return
        print pad_line(char)


def process_cmdline_args():
    """
    Parse command-line options
    """
    parser = OptionParser()
    parser.add_option("-n", "--name", dest="name",
                      default="", help="package name (will be retrieved via setuptools utilities, if present)")
    parser.add_option("-u", "--url", dest="url",
                      default="", help="package URL")
    parser.add_option("-p", "--path", dest="path",
                      default="", help="package path on local file system")
    parser.add_option("-s", "--sandbox", dest="sandbox",
                      default=default_temp_directory,
                      help="directory where package will be unpacked (default=%s)" % default_temp_directory)
    parser.add_option("-c", "--config", dest="config",
                      default=None,
                      help="directory with custom configuration (default=~/.cheesecake)")
    parser.add_option("-l", "--logfile", dest="logfile",
                      default=None,
                      help="file to log all cheesecake messages")
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
                      default=False, help="verbose output (default=False)")
    parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
                      default=False, help="only print Cheesecake index value (default=False)")

    (options, args) = parser.parse_args()
    return options

def main():
    """
    Display Cheesecake index for package specified via command-line options
    """
    options = process_cmdline_args()
    name = options.name
    url = options.url
    path = options.path
    sandbox = options.sandbox
    config = options.config
    logfile = options.logfile
    verbose = options.verbose
    quiet = options.quiet

    if not name and not url and not path:
        print "Error: No package name, URL or path specified (see --help)"
        sys.exit(1)

    try:
        c = Cheesecake(name=name, url=url, path=path, sandbox=sandbox,
                       config=config, logfile=logfile, verbose=verbose,
                       quiet=quiet)
        c.compute_cheesecake_index()
    except CheesecakeError, e:
        print str(e)

if __name__ == "__main__":
    main()