cheesecake_index.py

Revision 6, 39.4 kB (checked in by grig, 7 years ago)
Verify that pylint script can actually be executed properly.
Property svn:executable set to

Line
1	#!/usr/bin/env python
2	"""
3	Cheesecake: How tasty is your code?
4
5	The idea of the Cheesecake project is to rank Python packages
6	based on various empiric "kwalitee" factors, such as:
7
8	* whether the package can be downloaded
9	* whether the package can be unpacked
10	* whether the package can be installed into an alternate directory
11	* existence of certain files such as README, INSTALL, LICENSE, setup.py etc.
12	* existence of certain directories such as doc, test, demo, examples
13	* percentage of modules/functions/classes/methods with docstrings
14	* percentage of functions/methods that are unit tested
15	* average pylint score for all non-test and non-demo modules
16	* whether the package can be unpacked
17	* whether the package can be installed into an alternate directory
18	"""
19
20	import os, sys, re, shutil
21	import tarfile, zipfile
22	from optparse import OptionParser
23	from urllib import urlretrieve
24	from urlparse import urlparse
25	from subprocess import call, Popen, PIPE, STDOUT
26	from inspect import isclass, ismethod, isfunction
27	from math import ceil
28	import logger
29
30	INDEX_PYPI_DOWNLOAD = 50
31	INDEX_PYPI_DISTANCE = 5
32	INDEX_URL_DOWNLOAD = 25
33	INDEX_UNPACK = 25
34	INDEX_UNPACK_DIR = 15
35	INDEX_INSTALL = 50
36	INDEX_FILE_CRITICAL = 15
37	INDEX_FILE = 10
38	INDEX_REQUIRED_FILES = 100
39	INDEX_FILE_PYC = -20
40	INDEX_DIR_CRITICAL = 25
41	INDEX_DIR = 20
42	INDEX_DIR_EMPTY = 5
43
44	MAX_INDEX_DOCSTRINGS = 100 # max. percentage of modules/classes/methods/functions with docstrings
45	MAX_INDEX_PYLINT = 100 # max. pylint score
46
47	PAD_TEXT = 40
48	PAD_VALUE = 4
49
50	class Index(object):
51	"""
52	Encapsulates index attributes such as name, value, details
53	"""
54
55	def __init__(self, type, name="", value=0, details=""):
56	self.type = "index_" + type
57	self.name = self.type
58	if name: self.name += "_" + name
59	self.value = value
60	self.details = details
61
62	def print_info(self):
63	"""
64	Print index name padded with dots, followed by value and details
65	"""
66	msg = pad_with_dots(self.name)
67	msg += pad_left_spaces(self.value)
68	msg += " (" + self.details + ")"
69	print msg
70
71	class CompositeIndex(object):
72	"""
73	Collection of indexes of same type (e.g. files, dirs)
74	"""
75
76	def __init__(self, type):
77	"""
78	Indexes is a dict mapping names to Index objects
79	"""
80	self.type = type
81	self.indexes = {}
82
83	def set_index(self, name, value=0, details=""):
84	"""
85	Create new index or update existing index with specified attributes
86	"""
87	if self.indexes.has_key(name):
88	index = self.indexes[name]
89	index.value = value
90	index.details = details
91	else:
92	self.indexes[name] = Index(self.type, name, value, details)
93
94	def print_info(self):
95	"""
96	Print index info for all indexes sorted alphanumerically by name
97	"""
98	names = self.indexes.keys()
99	names.sort()
100	for name in names:
101	index = self.indexes[name]
102	index.print_info()
103
104	def get_value(self):
105	"""
106	Return sum of individual index values
107	"""
108	value = 0
109	for key in self.indexes.keys():
110	index = self.indexes[key]
111	value += index.value
112	return value
113
114	value = property(get_value)
115
116	class CheesecakeError(Exception):
117	"""
118	Custom exception class for Cheesecake-specific errors
119	"""
120	pass
121
122	class Cheesecake(object):
123	"""
124	Computes 'goodness' of Python packages
125
126	Generates "cheesecake index" that takes into account things like:
127
128	* whether the package can be downloaded
129	* whether the package can be unpacked
130	* whether the package can be installed into an alternate directory
131	* existence of certain files such as README, INSTALL, LICENSE, setup.py etc.
132	* existence of certain directories such as doc, test, demo, examples
133	* percentage of modules/functions/classes/methods with docstrings
134	* percentage of functions/methods that are unit tested
135	* average pylint score for all non-test and non-demo modules
136	"""
137
138	def __init__(self, name="", url="", path="", sandbox=None,
139	verbose=False, quiet=False):
140	"""
141	Initialize critical variables, download and unpack package, walk package tree
142
143	"""
144	self.name = name
145	self.url = url
146	self.package_path = path
147	if not self.name and not self.url and not self.package_path:
148	self.raise_exception("No package name, URL or path specified ... exiting")
149	self.sandbox = sandbox or "/tmp/cheesecake_sandbox"
150	if not os.path.isdir(self.sandbox):
151	os.mkdir(self.sandbox)
152	self.verbose = verbose
153	self.quiet = quiet
154
155	self.package_types = ["tar.gz", "tgz", "zip"]
156	self.sandbox_pkg_file = ""
157	self.sandbox_pkg_dir = ""
158	self.sandbox_install_dir = ""
159
160	self.determine_pkg_name()
161	self.configure_logging()
162	self.init_indexes()
163	self.retrieve_pkg()
164	self.unpack_pkg()
165	self.walk_pkg()
166
167	def raise_exception(self, msg):
168	"""
169	Cleanup, print error message and raise CheesecakeError
170
171	Don't use logging, since it can be called before logging has been setup
172	"""
173	self.cleanup()
174	os.unlink(os.path.join(self.sandbox, self.logfile))
175
176	msg += "\n" + pad_msg("CHEESECAKE INDEX", 0)
177	raise CheesecakeError(msg)
178
179	def cleanup(self):
180	"""
181	Delete temporary directories and files that were
182	created in the sandbox
183	"""
184	if os.path.isfile(self.sandbox_pkg_file):
185	self.log("Removing file %s" % self.sandbox_pkg_file)
186	os.unlink(self.sandbox_pkg_file)
187	if os.path.isdir(self.sandbox_pkg_dir):
188	self.log("Removing directory %s" % self.sandbox_pkg_dir)
189	shutil.rmtree(self.sandbox_pkg_dir)
190	if os.path.isdir(self.sandbox_install_dir):
191	self.log("Removing directory %s" % self.sandbox_install_dir)
192	shutil.rmtree(self.sandbox_install_dir)
193
194	def determine_pkg_name(self):
195	if self.name:
196	self.package = self.name
197	elif self.package_path:
198	self.package = self.get_package_from_path(self.package_path)
199	else:
200	self.package = self.get_package_from_url()
201
202	def configure_logging(self):
203	"""
204	Default settings for logging
205
206	if verbose, log goes to console, else it goes to logfile
207	log.debug goes to logfile
208	log.info goes to console
209	log.warn and log.error go to both logfile and stdout
210	"""
211	self.logfile = os.path.join(self.sandbox, self.package + ".log")
212
213	logger.setconsumer('logfile', open(str(self.logfile), 'w', buffering=1))
214	logger.setconsumer('console', logger.STDOUT)
215	logger.setconsumer('null', None)
216
217	if self.verbose:
218	self.log = logger.MultipleProducer('cheesecake console')
219	else:
220	self.log = logger.MultipleProducer('cheesecake logfile')
221	if self.quiet:
222	self.log.info = logger.MultipleProducer('cheesecake logfile')
223	else:
224	self.log.info = logger.MultipleProducer('cheesecake console')
225	self.log.debug = logger.MultipleProducer('cheesecake logfile')
226	self.log.warn = logger.MultipleProducer('cheesecake console')
227	self.log.error = logger.MultipleProducer('cheesecake console')
228
229	def init_indexes(self):
230	"""
231	Initialize variables used in index computation
232
233	* cheesecake_index: overall index for the package
234	* index: dict holding Index or CompositeIndex objects of various types
235	"""
236	self.cheesecake_index = 0
237	self.cheesecake_index_installability = 0
238	self.cheesecake_index_documentation = 0
239	self.cheesecake_index_codekwalitee = 0
240	self.max_cheesecake_index = INDEX_PYPI_DOWNLOAD + \
241	INDEX_UNPACK + \
242	INDEX_UNPACK_DIR + \
243	INDEX_INSTALL + \
244	MAX_INDEX_DOCSTRINGS + \
245	MAX_INDEX_PYLINT
246	self.max_cheesecake_index_installability = INDEX_PYPI_DOWNLOAD + \
247	INDEX_UNPACK + \
248	INDEX_UNPACK_DIR + \
249	INDEX_INSTALL
250	self.max_cheesecake_index_documentation = INDEX_REQUIRED_FILES + \
251	MAX_INDEX_DOCSTRINGS
252	self.max_cheesecake_index_codekwalitee = MAX_INDEX_PYLINT
253	self.index = {}
254	for index_type in ["file", "dir"]:
255	self.index[index_type] = CompositeIndex(index_type)
256	for index_type in ["pypi_download", "url_download",
257	"unpack_dir", "unpack", "install",
258	"docstrings", "pylint"]:
259	self.index[index_type] = Index(index_type)
260
261	self.cheese_files = ["readme", "install", "changelog",
262	"news", "faq",
263	"todo", "thanks",
264	"license", "announce",
265	"setup.py", "ez_setup.py",
266	]
267	self.critical_cheese_files = ["readme", "license", "setup.py"]
268	for cheese_file in self.cheese_files:
269	self.index["file"].set_index(name=cheese_file, details="file not found")
270	if cheese_file in self.critical_cheese_files:
271	self.max_cheesecake_index += INDEX_FILE_CRITICAL
272	self.max_cheesecake_index_documentation += INDEX_FILE_CRITICAL
273	else:
274	self.max_cheesecake_index += INDEX_FILE
275	self.max_cheesecake_index_documentation += INDEX_FILE
276	self.log.debug("cheese_files: " + ",".join(self.cheese_files))
277	self.log.debug("critical_cheese_files: " + ",".join(self.critical_cheese_files))
278
279	self.cheese_dirs = ["doc", "test", "example", "demo"]
280	self.critical_cheese_dirs = ["doc", "test"]
281	for cheese_dir in self.cheese_dirs:
282	self.index["dir"].set_index(name=cheese_dir, details="directory not found")
283	if cheese_dir in self.critical_cheese_dirs:
284	self.max_cheesecake_index += INDEX_DIR_CRITICAL
285	self.max_cheesecake_index_documentation += INDEX_DIR_CRITICAL
286	else:
287	self.max_cheesecake_index += INDEX_DIR
288	self.max_cheesecake_index_documentation += INDEX_DIR
289	self.log.debug("cheese_dirs: " + ",".join(self.cheese_dirs))
290	self.log.debug("critical_cheese_dirs: " + ",".join(self.critical_cheese_dirs))
291
292	self.pkg_files = {}
293	self.pkg_dirs = {}
294	self.file_types = ["py", "pyc", "test",
295	]
296	for type in self.file_types:
297	self.pkg_files[type] = []
298
299	def retrieve_pkg(self):
300	if self.name:
301	self.get_pkg_from_pypi()
302	elif self.url:
303	self.download_pkg()
304	else:
305	self.copy_pkg()
306
307	def get_package_from_url(self):
308	"""
309	Use ``urlparse`` to obtain package path from URL
310	"""
311	(scheme,location,path,param,query,fragment_id) = urlparse(self.url)
312	return self.get_package_from_path(path)
313
314
315	def get_package_from_path(self, path):
316	"""
317	Get package name as file portion of path
318	"""
319	dir, file = os.path.split(path)
320	return file
321
322	def get_pkg_from_pypi(self):
323	"""
324	Download package using setuptools utilities
325	"""
326	try:
327	self.log.info("Trying to download package %s from PyPI using setuptools utilities" % self.name)
328	from setuptools.package_index import PackageIndex
329	from pkg_resources import Requirement
330	from distutils import log
331	# Temporarily set the log verbosity to INFO so we can capture setuptools info messages
332	old_threshold = log.set_threshold(log.INFO)
333	pkgindex = PackageIndex()
334	old_stdout = sys.stdout
335	sys.stdout = StdoutRedirector()
336	output = pkgindex.fetch(Requirement.parse(self.name),
337	self.sandbox,
338	force_scan=True,
339	source=True)
340	captured_stdout = sys.stdout.read_buffer()
341	sys.stdout = old_stdout
342	log.set_threshold(old_threshold)
343	if output is None:
344	self.raise_exception("Error: Could not find distribution for " + self.name)
345	download_url = ""
346	distance_from_pypi = 0
347	#print captured_stdout
348	for line in captured_stdout.split('\n'):
349	s = re.search(r"Reading http(.*)", line)
350	if s:
351	inspected_url = s.group(1)
352	if not re.search(r"www.python.org\/pypi", inspected_url):
353	distance_from_pypi += 1
354	continue
355	s = re.search(r"Downloading (.*)", line)
356	if s:
357	download_url = s.group(1)
358	break
359	self.sandbox_pkg_file = output
360	self.package = self.get_package_from_path(output)
361	self.log.info("Downloaded package %s from %s" % (self.package, download_url))
362	index_type = "pypi_download"
363	found_on_cheeseshop = False
364	if re.search(r"cheeseshop.python.org", download_url):
365	value = INDEX_PYPI_DOWNLOAD
366	found_on_cheeseshop = True
367	else:
368	value = INDEX_PYPI_DOWNLOAD - distance_from_pypi * INDEX_PYPI_DISTANCE
369	self.index[index_type].value = value
370	details = "downloaded package " + self.package
371	if found_on_cheeseshop:
372	details += " directly from the Cheese Shop"
373	elif distance_from_pypi:
374	details += " following %d link" % distance_from_pypi
375	if distance_from_pypi > 1:
376	details += "s"
377	details += " from PyPI"
378	else:
379	details += "from " + download_url
380	self.index[index_type].details = details
381	except ImportError, e:
382	msg = "Error: setuptools is not installed and is required for downloading a package by name\n"
383	msg += "You can donwload and process a package by its full URL via the -u or --url option\n"
384	msg += "Example: python cheesecake.py --url=http://www.mems-exchange.org/software/durus/Durus-3.1.tar.gz"
385	self.raise_exception(msg)
386
387	def download_pkg(self):
388	"""
389	Use ``urllib.urlretrieve`` to download package to file in sandbox dir
390	"""
391	#self.log("Downloading package %s from URL %s" % (self.package, self.url))
392	self.sandbox_pkg_file = os.path.join(self.sandbox, self.package)
393	try:
394	downloaded_filename, headers = urlretrieve(self.url, self.sandbox_pkg_file)
395	except IOError, e:
396	self.log.error("Error downloading package %s from URL %s" % (self.package, self.url))
397	self.raise_exception(str(e))
398	#self.log("Downloaded package %s to %s" % (self.package, downloaded_filename))
399	if re.search("Content-Type: details/html", str(headers)):
400	f = open(downloaded_filename)
401	if re.search("404 Not Found", "".join(f.readlines())):
402	f.close()
403	self.raise_exception("Got '404 Not Found' error while trying to download package ... exiting")
404	f.close()
405	index_type = "url_download"
406	self.index[index_type].value = INDEX_URL_DOWNLOAD
407	self.index[index_type].details = "downloaded package %s from URL %s" % (self.package, self.url)
408
409	def copy_pkg(self):
410	"""
411	Copy package file to sandbox directory
412	"""
413	self.sandbox_pkg_file = os.path.join(self.sandbox, self.package)
414	if not os.path.isfile(self.package_path):
415	self.raise_exception("%s is not a valid file ... exiting" % self.package_path)
416	self.log("Copying file %s to %s" % (self.package_path, self.sandbox_pkg_file))
417	shutil.copyfile(self.package_path, self.sandbox_pkg_file)
418
419	def unpack_pkg(self):
420	"""
421	Unpack the package in the sandbox directory
422
423	Currently supported archive types:
424
425	* .tar.gz (handled with ``tarfile`` module)
426	* .zip (handled with ``zipfile`` module)
427	"""
428	self.package_type = ""
429	for type in self.package_types:
430	s = re.search(r"(.+)\.%s" % type, self.package)
431	if s:
432	# package_name is name of package without file extension (ex. twill-7.3)
433	self.package_name = s.group(1)
434	self.package_type = type
435	break
436	if not self.package_type:
437	msg = "Could not determine package type for package '%s'" % self.package
438	msg += "\nCurrently recognized types: " + " ".join(self.package_types)
439	self.raise_exception(msg)
440	self.log.debug("Package name: " + self.package_name)
441	self.log.debug("Package type: " + self.package_type)
442
443	self.sandbox_pkg_dir = os.path.join(self.sandbox, self.package_name)
444	if os.path.isdir(self.sandbox_pkg_dir):
445	shutil.rmtree(self.sandbox_pkg_dir)
446
447	if self.package_type in ["tar.gz", "tgz"]:
448	self.untar_pkg()
449	elif self.package_type == "zip":
450	self.unzip_pkg()
451
452	index_type = "unpack_dir"
453	details = "unpack directory is " + self.unpack_dir
454	if self.unpack_dir != self.package_name:
455	details += " instead of the expected " + self.package_name
456	self.package_name = self.unpack_dir
457	else:
458	details += " as expected"
459	self.index[index_type].value = INDEX_UNPACK_DIR
460	self.index[index_type].details = details
461
462	if not self.quiet:
463	self.log.info("Detailed info available in log file %s" % self.logfile)
464
465	def untar_pkg(self):
466	"""
467	Untar the package in the sandbox directory
468
469	Uses tarfile module
470	"""
471	try:
472	t = tarfile.open(self.sandbox_pkg_file)
473	except tarfile.ReadError, e:
474	self.raise_exception("Could not read tar file %s ... exiting" % self.sandbox_pkg_file)
475
476	for member in t.getmembers():
477	t.extract(member, self.sandbox)
478
479	tarinfo = t.members[0]
480	self.unpack_dir = tarinfo.name.split(os.sep)[0]
481
482	index_type = "unpack"
483	self.index[index_type].value = INDEX_UNPACK
484	self.index[index_type].details = "package untar-ed successfully"
485
486	def unzip_pkg(self):
487	"""
488	Unzip the package in the sandbox directory
489
490	Uses zipfile module
491	"""
492	try:
493	z = zipfile.ZipFile(self.sandbox_pkg_file)
494	except zipfile.error:
495	self.raise_exception("Error unzipping file %s ... exiting" % self.sandbox_pkg_file)
496
497	# Get directory structure from zip and create it in sandbox
498	for name in z.namelist():
499	(dir, file) = os.path.split(name)
500	unpack_dir = dir
501	target_dir = os.path.join(self.sandbox, dir)
502	if not os.path.exists(target_dir):
503	os.makedirs(target_dir)
504
505	# Extract files to directory structure
506	for i, name in enumerate(z.namelist()):
507	if not name.endswith('/'):
508	outfile = open(os.path.join(self.sandbox, name), 'wb')
509	outfile.write(z.read(name))
510	outfile.flush()
511	outfile.close()
512
513	self.unpack_dir = unpack_dir.split(os.sep)[0]
514
515	index_type = "unpack"
516	self.index[index_type].value = INDEX_UNPACK
517	self.index[index_type].details = "package unzipped successfully"
518
519	def walk_pkg(self):
520	"""
521	Traverse the file system tree rooted at sandbox/package_name
522
523	* Compute indexes for special files and directories
524	* Identify Python files, test files, etc.
525	"""
526	cwd = os.getcwd()
527	os.chdir(self.sandbox)
528	for rootdir, dirs, files in os.walk(self.package_name):
529	head, tail = os.path.split(rootdir)
530	dirs_in_rootdir = rootdir.split(os.path.sep)
531	for cheese_dir in self.cheese_dirs:
532	if re.search("^%s" % cheese_dir, tail):
533	if files or dirs:
534	if cheese_dir in self.critical_cheese_dirs:
535	value = INDEX_DIR_CRITICAL
536	details = "critical directory found"
537	self.log.debug("critical_cheese_dir found: " + cheese_dir)
538	else:
539	value = INDEX_DIR
540	details = "directory found"
541	self.log.debug("cheese_dir found: " + cheese_dir)
542	else:
543	value = INDEX_DIR_EMPTY
544	details = "empty directory found"
545	self.log.debug("empty cheese_dir found: " + cheese_dir)
546	self.index["dir"].set_index(cheese_dir, value, details)
547	for file in files:
548	fullpath = os.path.join(rootdir, file)
549	for cheese_file in self.cheese_files:
550	if re.search(r"^%s(\.txt)*" % cheese_file, file, re.IGNORECASE):
551	if cheese_file in self.critical_cheese_files:
552	value = INDEX_FILE_CRITICAL
553	details = "critical file found"
554	self.log.debug("critical_cheese_file found: " + cheese_file)
555	else:
556	value = INDEX_FILE
557	details = "file found"
558	self.log.debug("cheese_file found: " + cheese_file)
559	self.index["file"].set_index(cheese_file, value, details)
560
561	if self.is_py_file(file, dirs_in_rootdir):
562	self.pkg_files["py"].append(fullpath)
563	self.log.debug("py file found: " + fullpath)
564	if os.path.splitext(file)[1] == ".pyc":
565	self.pkg_files["pyc"].append(fullpath)
566	self.log.debug("pyc file found: " + fullpath)
567	if self.is_test_file(file, dirs_in_rootdir):
568	self.pkg_files["test"].append(fullpath)
569	self.log.debug("test file found: " + fullpath)
570	len_pyc_list = len(self.pkg_files["pyc"])
571	if len_pyc_list:
572	self.index["file"].set_index("pyc", value=-INDEX_FILE_PYC,
573	details="%d .pyc files found" % len_pyc_list)
574	self.log.debug("Found %d py files" % len(self.pkg_files["py"]))
575	self.log.debug("Found %d pyc files" % len(self.pkg_files["pyc"]))
576	self.log.debug("Found %d test files" % len(self.pkg_files["test"]))
577
578	os.chdir(cwd)
579
580	def is_py_file(self, file, dirs):
581	"""
582	Return True if file ends with .py and it is not a special file and it is not
583	in special directory
584	"""
585	if os.path.splitext(file)[1] != ".py":
586	return False
587	if file in ["setup.py", "ez_setup.py", "__init__.py", "__pkginfo__.py"]:
588	return False
589	for dir in dirs:
590	if dir.startswith("test") or \
591	dir.startswith("docs") or \
592	dir.startswith("demo") or \
593	dir.startswith("example"):
594	return False
595	return True
596
597	def is_test_file(self, file, dirs):
598	"""
599	Return True is file is in directory rooted at "test" or "tests"
600	"""
601	if file in ["__init__.py"]:
602	return False
603	for dir in dirs:
604	if dir.startswith("test"):
605	return True
606	return False
607
608	def index_file(self):
609	"""
610	Return CompositeIndex object of type "file"
611	"""
612	return self.index["file"]
613
614	def index_dir(self):
615	"""
616	Return CompositeIndex object of type "dir"
617	"""
618	return self.index["dir"]
619
620	def index_pypi_download(self):
621	"""
622	Verify that package can be downloaded from PyPI
623
624	Return Index object of type "pypi_download"
625	"""
626	index_type = "pypi_download"
627	if self.url:
628	# Package was downloaded directly from URL
629	self.index[index_type].value = 0
630	self.index[index_type].details = "package was downloaded directly from URL"
631
632	if self.package_path:
633	# Package was processed from file system path
634	self.index[index_type].value = 0
635	self.index[index_type].details = "package was processed from file system path"
636
637	# Otherwise, index["pypi_download"] was already set in get_pkg_from_pypi()
638	return self.index["pypi_download"]
639
640	def index_url_download(self):
641	"""
642	Verify that package can be downloaded from an URL
643
644	Return Index object of type "download"
645	"""
646	# index["download"] is already set in download_pkg()
647	return self.index["url_download"]
648
649	def index_unpack(self):
650	"""
651	Verify that package can be unpacked
652
653	Return Index object of type "unpack"
654	"""
655	# index["unpack"] is already set in unpack_pkg()
656	return self.index["unpack"]
657
658
659	def index_unpack_dir(self):
660	"""
661	Verify that unpack directory has same name as package
662
663	Return Index object of type "unpack_dir"
664	"""
665	# index["unpack_dir"] is already set in unpack_pkg()
666	return self.index["unpack_dir"]
667
668	def index_install(self):
669	"""
670	Verify that package can be installed in alternate directory
671
672	Return Index object of type "install"
673	"""
674	index_type = "install"
675	self.sandbox_install_dir = os.path.join(self.sandbox, "tmp_install_%s" % self.package_name)
676	cwd = os.getcwd()
677	os.chdir(os.path.join(self.sandbox, self.package_name))
678	p = Popen(["python", "setup.py", "install", "--home=%s" % self.sandbox_install_dir], stdout=PIPE, stderr=STDOUT)
679	output = p.communicate()[0]
680	if not p.returncode:
681	# Install succeeded
682	self.index[index_type].value = INDEX_INSTALL
683	self.index[index_type].details = details="package installed in %s" % self.sandbox_install_dir
684	else:
685	# Install failed
686	self.index[index_type].details = "could not install package in %s" % self.sandbox_install_dir
687	os.chdir(cwd)
688	return self.index[index_type]
689
690	def index_docstrings(self):
691	"""
692	Compute docstring index as percentage of modules/classes/methods/functions
693	that have docstrings associated with them
694
695	Return Index object of type "docstrings"
696	"""
697	cnt = 0
698	docstring_cnt = 0
699	index_type = "docstrings"
700	for pyfile in self.pkg_files["py"]:
701	fullpath = os.path.join(self.sandbox, pyfile)
702	code = CodeParser(fullpath, self.log.debug)
703	cnt += code.object_count()
704	docstring_cnt += code.docstring_count()
705	if cnt:
706	percent = float(docstring_cnt)/float(cnt)
707	else:
708	percent = 0
709	index_value = int(ceil(percent*100))
710	details = "found %d/%d=%.2f%% modules/classes/methods/functions with docstrings" % (docstring_cnt, cnt, percent*100)
711	self.index[index_type].value = index_value
712	self.index[index_type].details = details
713	return self.index[index_type]
714
715	def index_pylint(self):
716	"""
717	Compute pylint index as average of positive pylint scores obtained for
718	the Python files identified in the package
719
720	Return Index object of type "pylint"
721	"""
722	index_type = "pylint"
723	try:
724	import pylint
725	except ImportError, e:
726	self.index[index_type].details = "pylint not available"
727	return self.index[index_type]
728	# Now try to actually run the pylint script
729	p = Popen(["pylint"], stdout=PIPE, stderr=STDOUT)
730	output = p.communicate()[0]
731	rc = p.returncode
732	if rc:
733	# We encountered an error
734	self.index[index_type].details = "pylint not properly installed"
735	return self.index[index_type]
736	index_pylint = 0
737	cnt = 0
738	for pyfile in self.pkg_files["py"]:
739	(path, filename) = os.path.split(pyfile)
740	(module, ext) = os.path.splitext(filename)
741	if module == "setup" or module == "ez_setup" or module.startswith("__"):
742	continue
743	fullpath = os.path.join(self.sandbox, pyfile)
744	self.log.debug("Running pylint on file " + fullpath)
745	p = Popen(["pylint", fullpath], stdout=PIPE, stderr=STDOUT)
746	output = p.communicate()[0]
747	rc = p.returncode
748	if rc:
749	# We encountered an error
750	continue
751	score_line = output.split("\n")[-3]
752	s = re.search(r" (\d+\.\d+)/10", score_line)
753	# We only take positive scores into account
754	if s:
755	score = s.group(1)
756	self.log.debug("pylint score for module %s: %s" % (module, score))
757	if score == "0.00":
758	self.log.debug("Ignoring scores of 0.00")
759	continue
760	index_pylint += float(score)
761	cnt += 1
762	avg_value = float(index_pylint)/float(cnt)
763	index_value = int(ceil(avg_value*10))
764	self.index[index_type].value = index_value
765	self.index[index_type].details = "average score is %.2f out of 10" % avg_value
766	return self.index[index_type]
767
768	def compute_cheesecake_index(self):
769	"""
770	Compute overall Cheesecake index for the package by adding up
771	specific indexes
772	"""
773	self.log.info("A given package can currently reach a MAXIMUM number of %d points" % self.max_cheesecake_index)
774	self.log.info("Starting computation of Cheesecake index for package '%s'" % (self.package))
775
776	index_types = []
777	#if self.name:
778	# index_types.append("pypi_download")
779	index_types.append("pypi_download")
780	if self.url:
781	index_types.append("url_download")
782	index_types += ["unpack", "unpack_dir", "install"]
783	self.cheesecake_index_installability = self.process_partial_index("INSTALLABILITY",\
784	index_types, self.max_cheesecake_index_installability)
785
786	index_types = ["file", "dir", "docstrings"]
787	self.cheesecake_index_documentation = self.process_partial_index("DOCUMENTATION",\
788	index_types, self.max_cheesecake_index_documentation)
789
790	index_types = ["pylint"]
791	self.cheesecake_index_codekwalitee = self.process_partial_index("CODE KWALITEE",\
792	index_types, self.max_cheesecake_index_codekwalitee)
793
794	print
795	self.print_line("=" * (PAD_TEXT + PAD_VALUE + 1))
796	print pad_msg("OVERALL CHEESECAKE INDEX (ABSOLUTE)", self.cheesecake_index)
797	percentage = (self.cheesecake_index * 100) / self.max_cheesecake_index
798	msg = pad_msg("OVERALL CHEESECAKE INDEX (RELATIVE)", percentage)
799	msg += " (%d out of a maximum of %d points is %d%%)" %\
800	(self.cheesecake_index, self.max_cheesecake_index, percentage)
801	print msg
802	self.cleanup()
803
804	return self.cheesecake_index
805
806	def process_partial_index(self, partial_index_name, index_types, max_value):
807	print
808	self.log.info("Starting computation of %s index (max. points = %d)" % \
809	(partial_index_name, max_value))
810	partial_index_value = 0
811	for index_type in index_types:
812	partial_index_value += self.process_index(index_type)
813
814	self.print_line("-" * (PAD_TEXT + PAD_VALUE + 1))
815	print pad_msg("%s INDEX (ABSOLUTE)" % partial_index_name, partial_index_value)
816	percentage = (partial_index_value * 100) / max_value
817	msg = pad_msg("%s INDEX (RELATIVE)" % partial_index_name, percentage)
818	msg += " (%d out of a maximum of %d points is %d%%)" %\
819	(partial_index_value, max_value, percentage)
820	print msg
821	return partial_index_value
822
823	def process_index(self, index_type):
824	"""
825	Compute and print index of specified type
826	"""
827	index = self.index[index_type]
828	index_method = "index_" + index_type
829	getattr(self, index_method)()
830	if not self.quiet:
831	index.print_info()
832	self.cheesecake_index += index.value
833	return index.value
834
835	def print_line(self, line):
836	"""
837	Print line of text, unless quiet flag was given
838	"""
839	if self.quiet:
840	return
841	print line
842
843
844	class CodeParser(object):
845	"""
846	Information about the structure of a Python module
847
848	* Collects classes, methods, functions and any associated docstrings
849	* Does some dumb grep-style parsing, but in the future may do some real smart parsing
850	"""
851	def __init__(self, pyfile, log=None):
852	if log:
853	self.log = log.codeparser
854	else:
855	self.log = logger.default.codeparser
856	self.classes = []
857	self.methods = []
858	self.functions = []
859	self.docstrings = {}
860	self.object_at_page = {}
861	try:
862	self.fh = open(pyfile)
863	except IOError, e:
864	print str(e)
865	return
866
867	(path, filename) = os.path.split(pyfile)
868	(self.module, ext) = os.path.splitext(filename)
869	self.log("Inspecting file: " + pyfile)
870
871	self.parse_file()
872	self.log("classes: " + ",".join(self.classes))
873	self.log("methods: " + ",".join(self.methods))
874	self.log("functions: " + ",".join(self.functions))
875
876	def parse_file(self):
877	"""
878	Parse module text and retrieve classes, methods, functions
879	and associated docstrings
880	"""
881	cls_found = method_found = func_found = 0
882	cls = method = func = ""
883	crt_line = 0
884	for line in self.fh:
885	crt_line += 1
886	if not self.object_at_page.has_key(crt_line-1):
887	s = re.search(r"\"\"\"", line)
888	if s:
889	self.docstrings[self.module] = 1
890	s = re.search(r"^class (\S+)(\(\|:)", line)
891	if s:
892	cls = s.group(1)
893	self.classes.append(cls)
894	self.object_at_page[crt_line] = cls
895	self.log("Found class " + cls)
896	continue
897	s = re.search(r"^\s+def (\S+)\(", line)
898	if s:
899	method = s.group(1)
900	if method.startswith("__"):
901	self.log("Skipping method " + method)
902	continue
903	self.methods.append(method)
904	self.object_at_page[crt_line] = method
905	self.log("Found method " + method)
906	continue
907	s = re.search(r"^def (\S+)\(", line)
908	if s:
909	func = s.group(1)
910	self.functions.append(func)
911	self.object_at_page[crt_line] = func
912	self.log("Found function " + func)
913	continue
914	s1 = re.search(r"\"\"\"", line)
915	s2 = re.search(r"\".\S+.\"", line)
916	if s1 or s2:
917	# LOok at object if any on previous line
918	obj1 = self.object_at_page.get(crt_line-1)
919	obj2 = None
920	if crt_line > 1:
921	# Look at object is any 2 lines before
922	obj2 = self.object_at_page.get(crt_line-2)
923	obj = obj1 or obj2
924	if obj:
925	self.docstrings[obj] = 1
926	self.log("Found docstring for object " + obj)
927
928	def object_count(self):
929	"""
930	Return number of objects found in this module
931
932	* module
933	* classes
934	* methods
935	* functions
936	"""
937	module_count = 1
938	cls_count = len(self.classes)
939	method_count = len(self.methods)
940	func_count = len(self.functions)
941	return module_count + cls_count + method_count + func_count
942
943	def docstring_count(self):
944	"""
945	Return number of docstrings found in this module
946	"""
947	return len(self.docstrings.keys())
948
949	class StdoutRedirector(object):
950	"""
951	Redirect stdout to a temp file
952	"""
953
954	def __init__(self, filename=None):
955	if filename:
956	self.fh = open(filename, 'w')
957	else:
958	self.fh = os.tmpfile()
959
960	def write(self, buf):
961	self.fh.write(buf)
962
963	def flush(self):
964	self.fh.flush()
965
966	def read_buffer(self):
967	"""
968	Return contents of the temp file
969	"""
970	self.fh.seek(0)
971	return self.fh.read()
972
973	### Utility functions ###
974
975	def pad_with_dots(msg, length=PAD_TEXT):
976	"""
977	Pad text with dots up to given length
978	"""
979	length = len(msg)
980	msg = msg + " "
981	for i in range(length, PAD_TEXT):
982	msg += "."
983	return msg
984
985	def pad_left_spaces(value, length=PAD_VALUE):
986	"""
987	Pad value with spaces at left up to given length
988	"""
989	msg = ""
990	diff = length - len(str(value))
991	for i in range(diff):
992	msg += " "
993	msg += str(value)
994	return msg
995
996	def pad_msg(msg, value):
997	"""
998	Pad message with dots and pad value with spaces
999	"""
1000	length = len(msg)
1001	msg = msg + " "
1002	for i in range(length, PAD_TEXT):
1003	msg += "."
1004	diff = PAD_VALUE - len(str(value))
1005	for i in range(diff):
1006	msg += " "
1007	msg += str(value)
1008	return msg
1009
1010	### End utility functions ###
1011
1012	def process_cmdline_args():
1013	"""
1014	Parse command-line options
1015	"""
1016	parser = OptionParser()
1017	parser.add_option("-n", "--name", dest="name",
1018	default="", help="package name (will be retrieved via setuptools utilities, if present)")
1019	parser.add_option("-u", "--url", dest="url",
1020	default="", help="package URL")
1021	parser.add_option("-p", "--path", dest="path",
1022	default="", help="package path on local file system")
1023	parser.add_option("-s", "--sandbox", dest="sandbox",
1024	default="/tmp/cheesecake_sandbox",
1025	help="directory where package will be unpacked (default=/tmp/cheesecake_sandbox)")
1026	parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
1027	default=False, help="verbose output (default=False)")
1028	parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
1029	default=False, help="only print Cheesecake index value (default=False)")
1030
1031	(options, args) = parser.parse_args()
1032	return options
1033
1034	def main():
1035	"""
1036	Display Cheesecake index for package specified via command-line options
1037	"""
1038	options = process_cmdline_args()
1039	name = options.name
1040	url = options.url
1041	path = options.path
1042	sandbox = options.sandbox
1043	verbose = options.verbose
1044	quiet = options.quiet
1045
1046	if not name and not url and not path:
1047	print "Error: No package name, URL or path specified (see --help)"
1048	sys.exit(1)
1049
1050	try:
1051	c = Cheesecake(name=name, url=url, path=path, sandbox=sandbox, verbose=verbose, quiet=quiet)
1052	c.compute_cheesecake_index()
1053	except CheesecakeError, e:
1054	print str(e)
1055
1056	if __name__ == "__main__":
1057	main()

Note: See TracBrowser for help on using the browser.

root/trunk/cheesecake/cheesecake_index.py

Download in other formats: