WebKitTools/Scripts/webkitpy/commands/queues.py - platform/external/webkit - Git at Google

 #!/usr/bin/env python
 # Copyright (c) 2009, Google Inc. All rights reserved.
 # Copyright (c) 2009 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
 # in the documentation and/or other materials provided with the
 # distribution.
 #     * Neither the name of Google Inc. nor the names of its
 # contributors may be used to endorse or promote products derived from
 # this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 import traceback
 import os

 from datetime import datetime
 from optparse import make_option
 from StringIO import StringIO

 from webkitpy.bugzilla import CommitterValidator
 from webkitpy.executive import ScriptError
 from webkitpy.grammar import pluralize
 from webkitpy.webkit_logging import error, log
 from webkitpy.multicommandtool import Command
 from webkitpy.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate
 from webkitpy.statusserver import StatusServer
 from webkitpy.stepsequence import StepSequenceErrorHandler
 from webkitpy.queueengine import QueueEngine, QueueEngineDelegate

 class AbstractQueue(Command, QueueEngineDelegate):
     watchers = [
         "webkit-bot-watchers@googlegroups.com",
     ]

     _pass_status = "Pass"
     _fail_status = "Fail"
     _error_status = "Error"

     def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations
         options_list = (options or []) + [
             make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),
         ]
         Command.__init__(self, "Run the %s" % self.name, options=options_list)

     def _cc_watchers(self, bug_id):
         try:
             self.tool.bugs.add_cc_to_bug(bug_id, self.watchers)
         except Exception, e:
             traceback.print_exc()
             log("Failed to CC watchers.")

     def _update_status(self, message, patch=None, results_file=None):
         self.tool.status_server.update_status(self.name, message, patch, results_file)

     def _did_pass(self, patch):
         self._update_status(self._pass_status, patch)

     def _did_fail(self, patch):
         self._update_status(self._fail_status, patch)

     def _did_error(self, patch, reason):
         message = "%s: %s" % (self._error_status, reason)
         self._update_status(message, patch)

     def queue_log_path(self):
         return "%s.log" % self.name

     def work_item_log_path(self, patch):
         return os.path.join("%s-logs" % self.name, "%s.log" % patch.bug_id())

     def begin_work_queue(self):
         log("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self.tool.scm().checkout_root))
         if self.options.confirm:
             response = self.tool.user.prompt("Are you sure?  Type \"yes\" to continue: ")
             if (response != "yes"):
                 error("User declined.")
         log("Running WebKit %s." % self.name)

     def should_continue_work_queue(self):
         return True

     def next_work_item(self):
         raise NotImplementedError, "subclasses must implement"

     def should_proceed_with_work_item(self, work_item):
         raise NotImplementedError, "subclasses must implement"

     def process_work_item(self, work_item):
         raise NotImplementedError, "subclasses must implement"

     def handle_unexpected_error(self, work_item, message):
         raise NotImplementedError, "subclasses must implement"

     def run_webkit_patch(self, args):
         webkit_patch_args = [self.tool.path()]
         # FIXME: This is a hack, we should have a more general way to pass global options.
         webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host]
         webkit_patch_args += map(str, args)
         self.tool.executive.run_and_throw_if_fail(webkit_patch_args)

     def log_progress(self, patch_ids):
         log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids))))

     def execute(self, options, args, tool, engine=QueueEngine):
         self.options = options
         self.tool = tool
         return engine(self.name, self).run()

     @classmethod
     def _update_status_for_script_error(cls, tool, state, script_error, is_error=False):
         message = script_error.message
         if is_error:
             message = "Error: %s" % message
         output = script_error.message_with_output(output_limit=5*1024*1024) # 5MB
         return tool.status_server.update_status(cls.name, message, state["patch"], StringIO(output))


 class CommitQueue(AbstractQueue, StepSequenceErrorHandler):
     name = "commit-queue"
     def __init__(self):
         AbstractQueue.__init__(self)

     # AbstractQueue methods

     def begin_work_queue(self):
         AbstractQueue.begin_work_queue(self)
         self.committer_validator = CommitterValidator(self.tool.bugs)

     def _validate_patches_in_commit_queue(self):
         # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
         bug_ids = self.tool.bugs.queries.fetch_bug_ids_from_commit_queue()
         all_patches = sum([self.tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True) for bug_id in bug_ids], [])
         return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)

     def next_work_item(self):
         patches = self._validate_patches_in_commit_queue()
         # FIXME: We could sort the patches in a specific order here, was suggested by https://bugs.webkit.org/show_bug.cgi?id=33395
         if not patches:
             self._update_status("Empty queue")
             return None
         # Only bother logging if we have patches in the queue.
         self.log_progress([patch.id() for patch in patches])
         return patches[0]

     def _can_build_and_test(self):
         try:
             self.run_webkit_patch(["build-and-test", "--force-clean", "--non-interactive", "--build-style=both", "--quiet"])
         except ScriptError, e:
             self._update_status("Unabled to successfully build and test", None)
             return False
         return True

     def _builders_are_green(self):
         red_builders_names = self.tool.buildbot.red_core_builders_names()
         if red_builders_names:
             red_builders_names = map(lambda name: "\"%s\"" % name, red_builders_names) # Add quotes around the names.
             self._update_status("Builders [%s] are red. See http://build.webkit.org" % ", ".join(red_builders_names), None)
             return False
         return True

     def should_proceed_with_work_item(self, patch):
         if not self._builders_are_green():
             return False
         if not self._can_build_and_test():
             return False
         if not self._builders_are_green():
             return False
         self._update_status("Landing patch", patch)
         return True

     def process_work_item(self, patch):
         try:
             self._cc_watchers(patch.bug_id())
             # We pass --no-update here because we've already validated
             # that the current revision actually builds and passes the tests.
             # If we update, we risk moving to a revision that doesn't!
             self.run_webkit_patch(["land-attachment", "--force-clean", "--non-interactive", "--no-update", "--parent-command=commit-queue", "--build-style=both", "--quiet", patch.id()])
             self._did_pass(patch)
         except ScriptError, e:
             self._did_fail(patch)
             raise e

     def handle_unexpected_error(self, patch, message):
         self.committer_validator.reject_patch_from_commit_queue(patch.id(), message)

     # StepSequenceErrorHandler methods

     @staticmethod
     def _error_message_for_bug(tool, status_id, script_error):
         if not script_error.output:
             return script_error.message_with_output()
         results_link = tool.status_server.results_url_for_status(status_id)
         return "%s\nFull output: %s" % (script_error.message_with_output(), results_link)

     @classmethod
     def handle_script_error(cls, tool, state, script_error):
         status_id = cls._update_status_for_script_error(tool, state, script_error)
         validator = CommitterValidator(tool.bugs)
         validator.reject_patch_from_commit_queue(state["patch"].id(), cls._error_message_for_bug(tool, status_id, script_error))


 class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler):
     def __init__(self, options=None):
         AbstractQueue.__init__(self, options)

     def _review_patch(self, patch):
         raise NotImplementedError, "subclasses must implement"

     # PersistentPatchCollectionDelegate methods

     def collection_name(self):
         return self.name

     def fetch_potential_patch_ids(self):
         return self.tool.bugs.queries.fetch_attachment_ids_from_review_queue()

     def status_server(self):
         return self.tool.status_server

     def is_terminal_status(self, status):
         return status == "Pass" or status == "Fail" or status.startswith("Error:")

     # AbstractQueue methods

     def begin_work_queue(self):
         AbstractQueue.begin_work_queue(self)
         self._patches = PersistentPatchCollection(self)

     def next_work_item(self):
         patch_id = self._patches.next()
         if patch_id:
             return self.tool.bugs.fetch_attachment(patch_id)
         self._update_status("Empty queue")

     def should_proceed_with_work_item(self, patch):
         raise NotImplementedError, "subclasses must implement"

     def process_work_item(self, patch):
         try:
             self._review_patch(patch)
             self._did_pass(patch)
         except ScriptError, e:
             if e.exit_code != QueueEngine.handled_error_code:
                 self._did_fail(patch)
             raise e

     def handle_unexpected_error(self, patch, message):
         log(message)

     # StepSequenceErrorHandler methods

     @classmethod
     def handle_script_error(cls, tool, state, script_error):
         log(script_error.message_with_output())


 class StyleQueue(AbstractReviewQueue):
     name = "style-queue"
     def __init__(self):
         AbstractReviewQueue.__init__(self)

     def should_proceed_with_work_item(self, patch):
         self._update_status("Checking style", patch)
         return True

     def _review_patch(self, patch):
         self.run_webkit_patch(["check-style", "--force-clean", "--non-interactive", "--parent-command=style-queue", patch.id()])

     @classmethod
     def handle_script_error(cls, tool, state, script_error):
         is_svn_apply = script_error.command_name() == "svn-apply"
         status_id = cls._update_status_for_script_error(tool, state, script_error, is_error=is_svn_apply)
         if is_svn_apply:
             QueueEngine.exit_after_handled_error(script_error)
         message = "Attachment %s did not pass %s:\n\n%s\n\nIf any of these errors are false positives, please file a bug against check-webkit-style." % (state["patch"].id(), cls.name, script_error.message_with_output(output_limit=3*1024))
         tool.bugs.post_comment_to_bug(state["patch"].bug_id(), message, cc=cls.watchers)
         exit(1)
	#!/usr/bin/env python
	# Copyright (c) 2009, Google Inc. All rights reserved.
	# Copyright (c) 2009 Apple Inc. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions are
	# met:
	#
	# * Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# * Redistributions in binary form must reproduce the above
	# copyright notice, this list of conditions and the following disclaimer
	# in the documentation and/or other materials provided with the
	# distribution.
	# * Neither the name of Google Inc. nor the names of its
	# contributors may be used to endorse or promote products derived from
	# this software without specific prior written permission.
	#
	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	import traceback
	import os

	from datetime import datetime
	from optparse import make_option
	from StringIO import StringIO

	from webkitpy.bugzilla import CommitterValidator
	from webkitpy.executive import ScriptError
	from webkitpy.grammar import pluralize
	from webkitpy.webkit_logging import error, log
	from webkitpy.multicommandtool import Command
	from webkitpy.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate
	from webkitpy.statusserver import StatusServer
	from webkitpy.stepsequence import StepSequenceErrorHandler
	from webkitpy.queueengine import QueueEngine, QueueEngineDelegate

	class AbstractQueue(Command, QueueEngineDelegate):
	watchers = [
	"webkit-bot-watchers@googlegroups.com",
	]

	_pass_status = "Pass"
	_fail_status = "Fail"
	_error_status = "Error"

	def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations
	options_list = (options or []) + [
	make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue. Dangerous!"),
	]
	Command.__init__(self, "Run the %s" % self.name, options=options_list)

	def _cc_watchers(self, bug_id):
	try:
	self.tool.bugs.add_cc_to_bug(bug_id, self.watchers)
	except Exception, e:
	traceback.print_exc()
	log("Failed to CC watchers.")

	def _update_status(self, message, patch=None, results_file=None):
	self.tool.status_server.update_status(self.name, message, patch, results_file)

	def _did_pass(self, patch):
	self._update_status(self._pass_status, patch)

	def _did_fail(self, patch):
	self._update_status(self._fail_status, patch)

	def _did_error(self, patch, reason):
	message = "%s: %s" % (self._error_status, reason)
	self._update_status(message, patch)

	def queue_log_path(self):
	return "%s.log" % self.name

	def work_item_log_path(self, patch):
	return os.path.join("%s-logs" % self.name, "%s.log" % patch.bug_id())

	def begin_work_queue(self):
	log("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self.tool.scm().checkout_root))
	if self.options.confirm:
	response = self.tool.user.prompt("Are you sure? Type \"yes\" to continue: ")
	if (response != "yes"):
	error("User declined.")
	log("Running WebKit %s." % self.name)

	def should_continue_work_queue(self):
	return True

	def next_work_item(self):
	raise NotImplementedError, "subclasses must implement"

	def should_proceed_with_work_item(self, work_item):
	raise NotImplementedError, "subclasses must implement"

	def process_work_item(self, work_item):
	raise NotImplementedError, "subclasses must implement"

	def handle_unexpected_error(self, work_item, message):
	raise NotImplementedError, "subclasses must implement"

	def run_webkit_patch(self, args):
	webkit_patch_args = [self.tool.path()]
	# FIXME: This is a hack, we should have a more general way to pass global options.
	webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host]
	webkit_patch_args += map(str, args)
	self.tool.executive.run_and_throw_if_fail(webkit_patch_args)

	def log_progress(self, patch_ids):
	log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids))))

	def execute(self, options, args, tool, engine=QueueEngine):
	self.options = options
	self.tool = tool
	return engine(self.name, self).run()

	@classmethod
	def _update_status_for_script_error(cls, tool, state, script_error, is_error=False):
	message = script_error.message
	if is_error:
	message = "Error: %s" % message
	output = script_error.message_with_output(output_limit=510241024) # 5MB
	return tool.status_server.update_status(cls.name, message, state["patch"], StringIO(output))


	class CommitQueue(AbstractQueue, StepSequenceErrorHandler):
	name = "commit-queue"
	def __init__(self):
	AbstractQueue.__init__(self)

	# AbstractQueue methods

	def begin_work_queue(self):
	AbstractQueue.begin_work_queue(self)
	self.committer_validator = CommitterValidator(self.tool.bugs)

	def _validate_patches_in_commit_queue(self):
	# Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
	bug_ids = self.tool.bugs.queries.fetch_bug_ids_from_commit_queue()
	all_patches = sum([self.tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True) for bug_id in bug_ids], [])
	return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)

	def next_work_item(self):
	patches = self._validate_patches_in_commit_queue()
	# FIXME: We could sort the patches in a specific order here, was suggested by https://bugs.webkit.org/show_bug.cgi?id=33395
	if not patches:
	self._update_status("Empty queue")
	return None
	# Only bother logging if we have patches in the queue.
	self.log_progress([patch.id() for patch in patches])
	return patches[0]

	def _can_build_and_test(self):
	try:
	self.run_webkit_patch(["build-and-test", "--force-clean", "--non-interactive", "--build-style=both", "--quiet"])
	except ScriptError, e:
	self._update_status("Unabled to successfully build and test", None)
	return False
	return True

	def _builders_are_green(self):
	red_builders_names = self.tool.buildbot.red_core_builders_names()
	if red_builders_names:
	red_builders_names = map(lambda name: "\"%s\"" % name, red_builders_names) # Add quotes around the names.
	self._update_status("Builders [%s] are red. See http://build.webkit.org" % ", ".join(red_builders_names), None)
	return False
	return True

	def should_proceed_with_work_item(self, patch):
	if not self._builders_are_green():
	return False
	if not self._can_build_and_test():
	return False
	if not self._builders_are_green():
	return False
	self._update_status("Landing patch", patch)
	return True

	def process_work_item(self, patch):
	try:
	self._cc_watchers(patch.bug_id())
	# We pass --no-update here because we've already validated
	# that the current revision actually builds and passes the tests.
	# If we update, we risk moving to a revision that doesn't!
	self.run_webkit_patch(["land-attachment", "--force-clean", "--non-interactive", "--no-update", "--parent-command=commit-queue", "--build-style=both", "--quiet", patch.id()])
	self._did_pass(patch)
	except ScriptError, e:
	self._did_fail(patch)
	raise e

	def handle_unexpected_error(self, patch, message):
	self.committer_validator.reject_patch_from_commit_queue(patch.id(), message)

	# StepSequenceErrorHandler methods

	@staticmethod
	def _error_message_for_bug(tool, status_id, script_error):
	if not script_error.output:
	return script_error.message_with_output()
	results_link = tool.status_server.results_url_for_status(status_id)
	return "%s\nFull output: %s" % (script_error.message_with_output(), results_link)

	@classmethod
	def handle_script_error(cls, tool, state, script_error):
	status_id = cls._update_status_for_script_error(tool, state, script_error)
	validator = CommitterValidator(tool.bugs)
	validator.reject_patch_from_commit_queue(state["patch"].id(), cls._error_message_for_bug(tool, status_id, script_error))


	class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler):
	def __init__(self, options=None):
	AbstractQueue.__init__(self, options)

	def _review_patch(self, patch):
	raise NotImplementedError, "subclasses must implement"

	# PersistentPatchCollectionDelegate methods

	def collection_name(self):
	return self.name

	def fetch_potential_patch_ids(self):
	return self.tool.bugs.queries.fetch_attachment_ids_from_review_queue()

	def status_server(self):
	return self.tool.status_server

	def is_terminal_status(self, status):
	return status == "Pass" or status == "Fail" or status.startswith("Error:")

	# AbstractQueue methods

	def begin_work_queue(self):
	AbstractQueue.begin_work_queue(self)
	self._patches = PersistentPatchCollection(self)

	def next_work_item(self):
	patch_id = self._patches.next()
	if patch_id:
	return self.tool.bugs.fetch_attachment(patch_id)
	self._update_status("Empty queue")

	def should_proceed_with_work_item(self, patch):
	raise NotImplementedError, "subclasses must implement"

	def process_work_item(self, patch):
	try:
	self._review_patch(patch)
	self._did_pass(patch)
	except ScriptError, e:
	if e.exit_code != QueueEngine.handled_error_code:
	self._did_fail(patch)
	raise e

	def handle_unexpected_error(self, patch, message):
	log(message)

	# StepSequenceErrorHandler methods

	@classmethod
	def handle_script_error(cls, tool, state, script_error):
	log(script_error.message_with_output())


	class StyleQueue(AbstractReviewQueue):
	name = "style-queue"
	def __init__(self):
	AbstractReviewQueue.__init__(self)

	def should_proceed_with_work_item(self, patch):
	self._update_status("Checking style", patch)
	return True

	def _review_patch(self, patch):
	self.run_webkit_patch(["check-style", "--force-clean", "--non-interactive", "--parent-command=style-queue", patch.id()])

	@classmethod
	def handle_script_error(cls, tool, state, script_error):
	is_svn_apply = script_error.command_name() == "svn-apply"
	status_id = cls._update_status_for_script_error(tool, state, script_error, is_error=is_svn_apply)
	if is_svn_apply:
	QueueEngine.exit_after_handled_error(script_error)
	message = "Attachment %s did not pass %s:\n\n%s\n\nIf any of these errors are false positives, please file a bug against check-webkit-style." % (state["patch"].id(), cls.name, script_error.message_with_output(output_limit=3*1024))
	tool.bugs.post_comment_to_bug(state["patch"].bug_id(), message, cc=cls.watchers)
	exit(1)