mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-01-29 21:08:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: server/process: Add bitbake-server and exec() a new server process

Browse Source 
Trying to have a new python process forked off an original doesn't work
out well and ends up having race issues. To avoid this, exec() a new
bitbake server process. This starts with a fresh python interpreter
and resolves various atexit and other multiprocessing issues once
and for all.

(Bitbake rev: 9501dd6fdd7a7c25cbfa4464cf881fcf8c049ce2)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2020-08-24 17:12:30 +01:00
parent 6bab132879
commit 7002d67de0
2 changed files with 110 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
bitbake/bin/bitbake-server Executable file
View File

@@ -0,0 +1,54 @@
#!/usr/bin/env python3
#
# SPDX-License-Identifier: GPL-2.0-only
#
# Copyright (C) 2020 Richard Purdie
#
import os
import sys
import warnings
import logging
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
if sys.getfilesystemencoding() != "utf-8":
sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
# Users shouldn't be running this code directly
if len(sys.argv) != 10 or not sys.argv[1].startswith("decafbad"):
print("bitbake-server is meant for internal execution by bitbake itself, please don't use it standalone.")
sys.exit(1)
import bb.server.process
lockfd = int(sys.argv[2])
readypipeinfd = int(sys.argv[3])
logfile = sys.argv[4]
lockname = sys.argv[5]
sockname = sys.argv[6]
timeout = sys.argv[7]
xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
if xmlrpcinterface[0] == "None":
xmlrpcinterface = (None, xmlrpcinterface[1])
if timeout == "None":
timeout = None
# Replace standard fds with our own
with open('/dev/null', 'r') as si:
os.dup2(si.fileno(), sys.stdin.fileno())
so = open(logfile, 'a+')
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(so.fileno(), sys.stderr.fileno())
# Have stdout and stderr be the same so log output matches chronologically
# and there aren't two seperate buffers
sys.stderr = sys.stdout
logger = logging.getLogger("BitBake")
# Ensure logging messages get sent to the UI as events
handler = bb.event.LogHandler()
logger.addHandler(handler)
bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface)

96
bitbake/lib/bb/server/process.py
View File

@@ -38,7 +38,7 @@ class ProcessServer():
profile_filename = "profile.log"
profile_processed_filename = "profile.log.processed"
def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface):
def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
self.command_channel = False
self.command_channel_reply = False
self.quit = False
@@ -54,6 +54,7 @@ class ProcessServer():
self._idlefuns = {}
self.bitbake_lock = lock
self.bitbake_lock_name = lockname
self.sock = sock
self.sockname = sockname
@@ -259,7 +260,7 @@ class ProcessServer():
# Finally release the lockfile but warn about other processes holding it open
lock = self.bitbake_lock
lockfile = lock.name
lockfile = self.bitbake_lock_name
lock.close()
lock = None
@@ -393,9 +394,10 @@ class BitBakeProcessServerConnection(object):
self.connection.recv.close()
return
start_log_format = '--- Starting bitbake server pid %s at %s ---'
start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
class BitBakeServer(object):
start_log_format = '--- Starting bitbake server pid %s at %s ---'
start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
@@ -408,6 +410,7 @@ class BitBakeServer(object):
# Place the log in the builddirectory alongside the lock file
logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
self.logfile = logfile
startdatetime = datetime.datetime.now()
bb.daemonize.createDaemon(self._startServer, logfile)
@@ -429,7 +432,7 @@ class BitBakeServer(object):
ready.close()
bb.error("Unable to start bitbake server (%s)" % str(r))
if os.path.exists(logfile):
logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
started = False
lines = []
lastlines = []
@@ -441,7 +444,7 @@ class BitBakeServer(object):
lastlines.append(line)
res = logstart_re.match(line.rstrip())
if res:
ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
if ldatetime >= startdatetime:
started = True
lines.append(line)
@@ -462,42 +465,55 @@ class BitBakeServer(object):
ready.close()
def _startServer(self):
print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
os.close(self.readypipe)
os.set_inheritable(self.bitbake_lock.fileno(), True)
os.set_inheritable(self.readypipein, True)
serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
os.execl(serverscript, serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
import bb.cookerdata
import bb.cooker
print(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format)))
sys.stdout.flush()
try:
bitbake_lock = os.fdopen(lockfd, "w")
# Create server control socket
if os.path.exists(sockname):
os.unlink(sockname)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
try:
os.chdir(os.path.dirname(sockname))
sock.bind(os.path.basename(sockname))
finally:
os.chdir(cwd)
sock.listen(1)
server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface)
writer = ConnectionWriter(readypipeinfd)
try:
featureset = []
cooker = bb.cooker.BBCooker(featureset, server.register_idle_function)
except bb.BBHandledException:
return None
writer.send("r")
writer.close()
server.cooker = cooker
print("Started bitbake server pid %d" % os.getpid())
sys.stdout.flush()
try:
# Create server control socket
if os.path.exists(self.sockname):
os.unlink(self.sockname)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
try:
os.chdir(os.path.dirname(self.sockname))
sock.bind(os.path.basename(self.sockname))
finally:
os.chdir(cwd)
sock.listen(1)
server = ProcessServer(self.bitbake_lock, sock, self.sockname, self.server_timeout, self.xmlrpcinterface)
os.close(self.readypipe)
writer = ConnectionWriter(self.readypipein)
try:
self.cooker = bb.cooker.BBCooker(self.featureset, server.register_idle_function)
except bb.BBHandledException:
return None
writer.send("r")
writer.close()
server.cooker = self.cooker
print("Started bitbake server pid %d" % os.getpid())
sys.stdout.flush()
server.run()
finally:
# Flush any ,essages/errors to the logfile before exit
sys.stdout.flush()
sys.stderr.flush()
server.run()
finally:
# Flush any ,essages/errors to the logfile before exit
sys.stdout.flush()
sys.stderr.flush()
def connectProcessServer(sockname, featureset):
# Connect to socket