Changeset 80

Timestamp:

07/06/06 18:15:29 (2 years ago)

Author:

mk

Message:

Indices can now hold dependencies (via decide() interface).
Certain steps (like installation of a package) during scoring are run only

on demand - to satisfy indices dependencies.

Implemented "static" profile by setting appropriate dependencies on

IndexInstall? (closes ticket #37).

Files:

• branches/mk/cheesecake/cheesecake_index.py (modified) (28 diffs)
• branches/mk/tests/unit/_mockup_cheesecake.py (modified) (3 diffs)
• branches/mk/tests/unit/test_index_class.py (modified) (1 diff)
• branches/mk/tests/unit/test_index_installability.py (modified) (3 diffs)

branches/mk/cheesecake/cheesecake_index.py

@@ -18 +18 @@
-"""
-
-
+
+
+
+
-import tempfile
+
-from optparse import OptionParser
-from urllib import urlretrieve
-from urlparse import urlparse
-from math import ceil
+
+import logger
-
-from util import pad_with_dots, pad_left_spaces, pad_right_spaces, pad_msg, pad_line
@@ -30 +36 @@
-from util import mkdirs
-from util import StdoutRedirector
-import logger
-from config import get_pkg_config
-from codeparser import CodeParser
@@ -40 +45 @@
-## Helpers.
-################################################################################
+
+if 'sorted' not in dir(__builtins__):
+    def sorted(L):
+        new_list = L[:]
+        new_list.sort()
+        return new_list
-
-def isiterable(obj):
@@ -113 +124 @@
-    return filter(lambda x: discover_file_type(x) == file_type, file_list)
-
+def get_package_name_from_path(path):
+    """Get package name as file portion of path.
+
+    >>> get_package_name_from_path('/some/random/path/package.tar.gz')
+    'package.tar.gz'
+    >>> get_package_name_from_path('/path/underscored_name.zip')
+    'underscored_name.zip'
+    >>> get_package_name_from_path('/path/unknown.extension.txt')
+    'unknown.extension.txt'
+    """
+    dir, filename = os.path.split(path)
+    return filename
+
+def get_package_name_from_url(url):
+    """Use ``urlparse`` to obtain package name from URL.
+
+    >>> get_package_name_from_url('http://www.example.com/file.tar.bz2')
+    'file.tar.bz2'
+    >>> get_package_name_from_url('https://www.example.com/some/dir/file.txt')
+    'file.txt'
+    """
+    (scheme,location,path,param,query,fragment_id) = urlparse(url)
+    return get_package_name_from_path(path)
+
+def get_package_name_and_type(package, known_extensions):
+    """Return package name and type.
+
+    Package type must exists in known_extensions list. Otherwise None is
+    returned.
+
+    >>> extensions = ['tar.gz', 'zip']
+    >>> get_package_name_and_type('underscored_name.zip', extensions)
+    ('underscored_name', 'zip')
+    >>> get_package_name_and_type('unknown.extension.txt', extensions)
+    """
+    for package_type in known_extensions:
+        if package.endswith('.'+package_type):
+            # Package name is name of package without file extension (ex. twill-7.3).
+            return package[:package.rfind('.'+package_type)], package_type
+
-def get_method_arguments(method):
-    """Return tuple of arguments for given method, excluding self.
@@ -261 +312 @@
-    details = ""
-
-
-
+
+
+
-            self.subindices = []
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-        self._compute_arguments = get_method_arguments(self.compute)
@@ -303 +365 @@
-        return self.value
-
+    def decide(self, cheesecake, when):
+        """Decide if this index should be computed.
+
+        If index has children, it will automatically remove all for which
+        decide() return false.
+        """
+        if self.subindices:
+            # Iterate over copy, as we may remove some elements.
+            for index in self.subindices[:]:
+                print "Trying %s." % index.name
+                if not getattr(index, 'decide_' + when)(cheesecake):
+                    self.remove_subindex(index.name)
+            return self.subindices
+        return True
+
+    def decide_before_download(self, cheesecake):
+        return self.decide(cheesecake, 'before_download')
+
+    def decide_after_download(self, cheesecake):
+        return self.decide(cheesecake, 'after_download')
+
-    def _get_max_value(self):
-        if self.subindices:
@@ -310 +393 @@
-
-    max_value = property(_get_max_value)
+
+    def _get_requirements(self):
+        if self.subindices:
+            return list(self._compute_arguments) + \
+                   reduce(lambda x,y: x + y.requirements, self.subindices, [])
+        return list(self._compute_arguments)
+
+    requirements = property(_get_requirements)
-
-    def add_subindex(self, index):
@@ -464 +555 @@
-        return self.value
-
+    def decide_before_download(self, cheesecake):
+        return cheesecake.url
+
-class IndexUnpack(Index):
-    max_value = 25
@@ -491 +585 @@
-        return self.value
-
+    def decide_after_download(self, cheesecake):
+        return cheesecake.package_type != 'egg'
+
-class IndexSetupPy(FilesIndex):
-    name = "setup.py"
@@ -508 +605 @@
-
-        return self.value
+
+    def decide_after_download(self, cheesecake):
+        return cheesecake.package_type != 'egg'
-
-class IndexInstall(Index):
@@ -521 +621 @@
-
-        return self.value
+
+    def decide_before_download(self, cheesecake):
+        return not cheesecake.static_only
-
-class IndexPyPIDownload(Index):
@@ -551 +654 @@
-        return self.value
-
+    def decide_before_download(self, cheesecake):
+        return cheesecake.name
+
-class IndexGeneratedFiles(Index):
-    generated_files_penalty = -20
+    max_value = 0
-
-    def compute(self, files_list):
@@ -568 +675 @@
-        return self.value
-
+    def decide_after_download(self, cheesecake):
+        return cheesecake.package_type != 'egg'
+
-class IndexInstallability(Index):
-    name = "INSTALLABILITY"
-
-    subindices = [
-
-
-
-
-
+
+
+
+
+
+
+
-    ]
-
@@ -662 +774 @@
-
-    subindices = [
-()
-()
-()
+
+
+
-    ]
-
@@ -778 +890 @@
-
-    subindices = [
-()
- IndexUnitTests()
+
+IndexUnitTests
-    ]
-
@@ -795 +907 @@
-    name = "Cheesecake"
-    subindices = [
-()
-()
-()
+
+
+
-    ]
-
-
+class Step(object):
+    """Single step during computation of package score.
+    """
+    def __init__(self, provides):
+        self.provides = provides
+
+    def decide(self, cheesecake):
+        """Decide if step should be run.
+
+        It checks if there's at least one index from current profile that need
+        variables provided by this step. Override this method for other behaviour.
+        """
+        for provide in self.provides:
+            if provide in cheesecake.index.requirements:
+                return True
+        return False
+
+class StepByVariable(Step):
+    """Step which is always run if given Cheesecake instance variable is true.
+    """
+    def __init__(self, variable_name, provides):
+        self.variable_name = variable_name
+        Step.__init__(self, provides)
+
+    def decide(self, cheesecake):
+        if getattr(cheesecake, self.variable_name, None):
+            return True
+
+        # Fallback to the default.
+        return Step.decide(self, cheesecake)
+
-class Cheesecake(object):
-
-
+
-
-    Generates "cheesecake index" that takes into account things like:
@@ -816 +958 @@
-        * average pylint score for all non-test and non-demo modules
-    """
-
+
+
+
+
+
+
+
+
+
-
-    def __init__(self, name="", url="", path="", sandbox=None, config=None,
@@ -827 +977 @@
-        self.package_path = path
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-        self.sandbox = sandbox or tempfile.mkdtemp(prefix='cheesecake')
-        if not os.path.isdir(self.sandbox):
@@ -839 +996 @@
-        self.static_only = static_only
-
-        self.package_types = {
-            "tar.gz": untar_package,
-            "tgz": untar_package,
-            "zip": unzip_package,
-            "egg": unegg_package,
-        }
-
-        self.sandbox_pkg_file = ""
-        self.sandbox_pkg_dir = ""
-        self.sandbox_install_dir = ""
-
-        # Include indices revelant to current situation.
-        if self.name:
-            self.index["INSTALLABILITY"].add_subindex(IndexPyPIDownload())
-        if self.url:
-            self.index["INSTALLABILITY"].add_subindex(IndexUrlDownload())
-
-        # Get package name.
-        self.determine_pkg_name()
-
-        # Configure logging as soon as possible.
-        self.configure_logging(logfile)
-
-
-
+
+
+
+
+
+
+
+
+
+
-
-        # Get package name and type.
-
+
+
+
+
+
+
+
+
-        self.log.debug("Package name: " + self.package_name)
-        self.log.debug("Package type: " + self.package_type)
-
+        # Make last indices decisions.
+        self.index.decide_after_download(self)
+
-        # Unpack package and list its files.
-unpack_pkg(
-walk_pkg(
+run_step('unpack_pkg'
+run_step('walk_pkg'
-
-        # Install package.
-
-
-
-
-
-
-
-
-
-
+
-
-    def raise_exception(self, msg):
@@ -927 +1077 @@
-        pass
-
-    def determine_pkg_name(self):
-        if self.name:
-            self.package = self.name
-            self.short_pkg_name = self.name
-        elif self.package_path:
-            self.package = self.get_package_from_path(self.package_path)
-        else:
-            self.package = self.get_package_from_url()
-
-    def get_package_from_url(self):
-        """Use ``urlparse`` to obtain package path from URL.
-        """
-        (scheme,location,path,param,query,fragment_id) = urlparse(self.url)
-        return self.get_package_from_path(path)
-
-    def get_package_from_path(self, path):
-        """Get package name as file portion of path.
-        """
-        dir, file = os.path.split(path)
-        self.short_pkg_name = file
-        for package_type in self.package_types:
-            s = re.search("(.+)\.%s" % package_type, file)
-            if s:
-                self.short_pkg_name = s.group(1)
-                break
-        return file
-
-    def configure_logging(self, logfile=None):
-        """Default settings for logging.
@@ -964 +1087 @@
-            self.logfile = logfile
-        else:
-short_pkg_nam
+packag
-
-        logger.setconsumer('logfile', open(str(self.logfile), 'w', buffering=1))
@@ -980 +1103 @@
-        self.log.error = logger.MultipleProducer('cheesecake console')
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    def get_pkg_from_pypi(self):
-        """Download package using setuptools utilities.
@@ -1070 +1199 @@
-
-        self.sandbox_pkg_file = output
-self.get_packag
+get_package_nam
-        self.log.info("Downloaded package %s from %s" % (self.package, self.download_url))
-
@@ -1079 +1208 @@
-            self.found_on_cheeseshop = True
-
+    steps['download_pkg'] = StepByVariable('url',
+                                           ['sandbox_pkg_file',
+                                            'downloaded_from_url'])
-    def download_pkg(self):
-        """Use ``urllib.urlretrieve`` to download package to file in sandbox dir.
@@ -1099 +1231 @@
-
-        self.downloaded_from_url = True
-
+
+
+
-    def copy_pkg(self):
-        """Copy package file to sandbox directory.
@@ -1109 +1243 @@
-        shutil.copyfile(self.package_path, self.sandbox_pkg_file)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-    def unpack_pkg(self):
-        """Unpack the package in the sandbox directory.
@@ -1151 +1274 @@
-            self.package_name = self.unpack_dir
-
+    steps['walk_pkg'] = Step(['dirs_list',
+                              'docstring_cnt',
+                              'docformat_cnt',
+                              'files_list',
+                              'functions',
+                              'object_cnt',
+                              'package_dir'])
-    def walk_pkg(self):
-        """Get package files and directories.
@@ -1196 +1326 @@
-                                                      ', '.join(self.dirs_list)))
-
+    steps['install_pkg'] = Step(['installed'])
-    def install_pkg(self):
-        """Verify that package can be installed in alternate directory.

branches/mk/tests/unit/_mockup_cheesecake.py

@@ -6 +6 @@
-import _path_cheesecake
-from cheesecake.cheesecake_index import Cheesecake
+from cheesecake.cheesecake_index import CheesecakeIndex
-
-
@@ -12 +13 @@
-
-    class CheesecakeMockup(Cheesecake):
+        def run_step(self, step_name):
+            if step_name == 'install_pkg':
+                return
+            Cheesecake.run_step(self, step_name)
+
-        def __init__(self, sandbox, package_name, logfile):
-            self.name = package_name
-            self.package_name = package_name
+            self.package = package_name
-            self.sandbox = sandbox
-
@@ -22 +29 @@
-            self.unpack_dir = sandbox
-
-            self.determine_pkg_name()
-            self.configure_logging(logfile)
-set_defaults
+index = CheesecakeIndex
-
-    def setUp(self):

branches/mk/tests/unit/test_index_class.py

@@ -24 +24 @@
-    >>> index in big_index.subindices
-    False
+
+Test requirements.
+    >>> class NewIndex(Index):
+    ...     def compute(self, one, two, three):
+    ...         pass
+    >>> new = NewIndex()
+    >>> new.requirements
+    ['one', 'two', 'three']
+
+Now create other index and add it to the NewIndex.
+    >>> class OtherIndex(Index):
+    ...     def compute(self, four):
+    ...         pass
+    >>> other = OtherIndex()
+    >>> other.requirements
+    ['four']
+    >>> new.add_subindex(other)
+    >>> new.requirements
+    ['one', 'two', 'three', 'four']
-"""

branches/mk/tests/unit/test_index_installability.py

@@ -11 +11 @@
-from cheesecake.cheesecake_index import IndexInstall
-from cheesecake.cheesecake_index import IndexUrlDownload
+from cheesecake.cheesecake_index import IndexGeneratedFiles
-
-
@@ -26 +27 @@
-
-        index = self.cheesecake.index["INSTALLABILITY"]
-
+, IndexGeneratedFiles
-
-        assert index.max_value == sum(map(lambda x: x.max_value, parts))
@@ -37 +38 @@
-
-        index = self.cheesecake.index["INSTALLABILITY"]
-npack, IndexUnpackDir, IndexSetupPy, IndexInstall, IndexUrlDownload
+rlDownload, IndexUnpack, IndexUnpackDir, IndexSetupPy, IndexInstall, IndexGeneratedFiles
-
-        assert index.max_value == sum(map(lambda x: x.max_value, parts))