mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-04-21 03:32:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: bitbake-worker/runqueue: Avoid unnecessary bytes object copies

Browse Source 
declaring queue=b"" creates an object of types bytes().
bytes() is an immutable object, and therefore doing "self.queue = self.queue + r"
creates a new object containing "self.queue" concatenated with "r".

On my test setup, we are passing 180MB of data of "workerdata" to the bitbake-worker,
so those copies significantly slow down the initialization of the bitbake-worker.

Rather use bytearray() which a mutable type, and use extend() to avoid copies.
In my test setup, byterray.extend() is 10.000 times faster than copying the queue,
for a queue size of 180MB.

(Bitbake rev: 9993a89e5b97dda5f3657e5a7cc3a4fa94ff7111)

Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
This commit is contained in:
Etienne Cordonnier
2023-09-22 10:51:46 +02:00
committed by Steve Sakoman
parent 222be3e3b9
commit 3a3afebf41
2 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
bitbake/bin/bitbake-worker
View File

@@ -91,19 +91,19 @@ def worker_fire_prepickled(event):
worker_thread_exit = False
def worker_flush(worker_queue):
worker_queue_int = b""
worker_queue_int = bytearray()
global worker_pipe, worker_thread_exit
while True:
try:
worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
worker_queue_int.extend(worker_queue.get(True, 1))
except queue.Empty:
pass
while (worker_queue_int or not worker_queue.empty()):
try:
(_, ready, _) = select.select([], [worker_pipe], [], 1)
if not worker_queue.empty():
worker_queue_int = worker_queue_int + worker_queue.get()
worker_queue_int.extend(worker_queue.get())
written = os.write(worker_pipe, worker_queue_int)
worker_queue_int = worker_queue_int[written:]
except (IOError, OSError) as e:
@@ -338,12 +338,12 @@ class runQueueWorkerPipe():
if pipeout:
pipeout.close()
bb.utils.nonblockingfd(self.input)
self.queue = b""
self.queue = bytearray()
def read(self):
start = len(self.queue)
try:
self.queue = self.queue + (self.input.read(102400) or b"")
self.queue.extend(self.input.read(102400) or b"")
except (OSError, IOError) as e:
if e.errno != errno.EAGAIN:
raise
@@ -371,7 +371,7 @@ class BitbakeWorker(object):
def __init__(self, din):
self.input = din
bb.utils.nonblockingfd(self.input)
self.queue = b""
self.queue = bytearray()
self.cookercfg = None
self.databuilder = None
self.data = None
@@ -405,7 +405,7 @@ class BitbakeWorker(object):
if len(r) == 0:
# EOF on pipe, server must have terminated
self.sigterm_exception(signal.SIGTERM, None)
self.queue = self.queue + r
self.queue.extend(r)
except (OSError, IOError):
pass
if len(self.queue):

4
bitbake/lib/bb/runqueue.py
View File

@@ -3101,7 +3101,7 @@ class runQueuePipe():
if pipeout:
pipeout.close()
bb.utils.nonblockingfd(self.input)
self.queue = b""
self.queue = bytearray()
self.d = d
self.rq = rq
self.rqexec = rqexec
@@ -3120,7 +3120,7 @@ class runQueuePipe():
start = len(self.queue)
try:
self.queue = self.queue + (self.input.read(102400) or b"")
self.queue.extend(self.input.read(102400) or b"")
except (OSError, IOError) as e:
if e.errno != errno.EAGAIN:
raise