bitbake: bitbake-worker: Further IO performance tweaks
Looking further at the CPU loads on systems running large numbers of tasks, the following things helps performance: * Loop on waitpid until there are no processes still waiting * Using select to wait for the cooker pipe to be writable before writing avoiding pointless 100% cpu usage * Only reading from worker pipes that select highlights are readable (Bitbake rev: 9375349e27b08b4d1cfe4825c042d4c82120e00b) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
parent
eb937ee0a1
commit
24b668efa0
|
@ -95,6 +95,7 @@ def worker_flush(worker_queue):
|
||||||
pass
|
pass
|
||||||
while (worker_queue_int or not worker_queue.empty()):
|
while (worker_queue_int or not worker_queue.empty()):
|
||||||
try:
|
try:
|
||||||
|
(_, ready, _) = select.select([], [worker_pipe], [], 1)
|
||||||
if not worker_queue.empty():
|
if not worker_queue.empty():
|
||||||
worker_queue_int = worker_queue_int + worker_queue.get()
|
worker_queue_int = worker_queue_int + worker_queue.get()
|
||||||
written = os.write(worker_pipe, worker_queue_int)
|
written = os.write(worker_pipe, worker_queue_int)
|
||||||
|
@ -369,9 +370,11 @@ class BitbakeWorker(object):
|
||||||
self.handle_item(b"quit", self.handle_quit)
|
self.handle_item(b"quit", self.handle_quit)
|
||||||
|
|
||||||
for pipe in self.build_pipes:
|
for pipe in self.build_pipes:
|
||||||
self.build_pipes[pipe].read()
|
if self.build_pipes[pipe].input in ready:
|
||||||
|
self.build_pipes[pipe].read()
|
||||||
if len(self.build_pids):
|
if len(self.build_pids):
|
||||||
self.process_waitpid()
|
while self.process_waitpid():
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
def handle_item(self, item, func):
|
def handle_item(self, item, func):
|
||||||
|
@ -426,9 +429,9 @@ class BitbakeWorker(object):
|
||||||
try:
|
try:
|
||||||
pid, status = os.waitpid(-1, os.WNOHANG)
|
pid, status = os.waitpid(-1, os.WNOHANG)
|
||||||
if pid == 0 or os.WIFSTOPPED(status):
|
if pid == 0 or os.WIFSTOPPED(status):
|
||||||
return None
|
return False
|
||||||
except OSError:
|
except OSError:
|
||||||
return None
|
return False
|
||||||
|
|
||||||
workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
|
workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
|
||||||
|
|
||||||
|
@ -447,6 +450,8 @@ class BitbakeWorker(object):
|
||||||
|
|
||||||
worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
|
worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
def handle_finishnow(self, _):
|
def handle_finishnow(self, _):
|
||||||
if self.build_pids:
|
if self.build_pids:
|
||||||
logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
|
logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
|
||||||
|
|
Loading…
Reference in New Issue