mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-01-29 21:08:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: server: Rework the server API so process and xmlrpc servers coexist

Browse Source 
This changes the way bitbake server works quite radically. Now, the
server is always a process based server with the option of starting
an XMLRPC listener on a specific inferface/port.

Behind the scenes this is done with a "bitbake.sock" file alongside
the bitbake.lock file. If we can obtain the lock, we know we need
to start a server. The server always listens on the socket and UIs
can then connect to this. UIs connect by sending a set of three file
descriptors over the domain socket, one for sending commands, one for
receiving command results and the other for receiving events.

These changes meant we can throw away all the horrid server abstraction
code, the plugable transport option to bitbake and the code becomes
much more readable and debuggable. It also likely removes a ton of
ways you could hang the UI/cooker in weird ways due to all the race
conditions that existed with previous processes.

Changes:

 * The foreground option for bitbake-server was dropped. Just tail
   the log if you really want this, the codepaths were complicated enough
   without adding one for this.
 * BBSERVER="autodetect" was dropped. The server will autostart and
   autoconnect in process mode. You have to specify an xmlrpc server
   address since that can't be autodetected. I can't see a use case
   for autodetect now.
 * The transport/servetype option to bitbake was dropped.
 * A BB_SERVER_TIMEOUT variable is added which allows the server
   to stay resident for a period of time after the last client
   disconnects before unloading. This is used if the -T/--idle-timeout
   option is not passed to bitbake.

This change is invasive and may well introduce new issues however I
believe the codebase is in a much better position for further
development and debugging.

(Bitbake rev: 72a3dbe13a23588e24c0baca6d58c35cdeba3f63)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2017-07-18 22:28:40 +01:00
parent 21a19e0e0b
commit 4602408c69
10 changed files with 797 additions and 924 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
bitbake/lib/bb/cooker.py
View File

@@ -215,19 +215,6 @@ class BBCooker:
self.configuration.server_register_idlecallback(_process_inotify_updates, [self.confignotifier, self.notifier])
# Take a lock so only one copy of bitbake can run against a given build
# directory at a time
if not self.lockBitbake():
bb.fatal("Only one copy of bitbake should be run against a build directory")
try:
self.lock.seek(0)
self.lock.truncate()
if len(configuration.interface) >= 2:
self.lock.write("%s:%s\n" % (configuration.interface[0], configuration.interface[1]));
self.lock.flush()
except:
pass
# TOSTOP must not be set or our children will hang when they output
try:
fd = sys.stdout.fileno()
@@ -1557,33 +1544,6 @@ class BBCooker:
def post_serve(self):
prserv.serv.auto_shutdown(self.data)
bb.event.fire(CookerExit(), self.data)
lockfile = self.lock.name
self.lock.close()
self.lock = None
while not self.lock:
with bb.utils.timeout(3):
self.lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
if not self.lock:
# Some systems may not have lsof available
procs = None
try:
procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
except OSError as e:
if e.errno != errno.ENOENT:
raise
if procs is None:
# Fall back to fuser if lsof is unavailable
try:
procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
except OSError as e:
if e.errno != errno.ENOENT:
raise
msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
if procs:
msg += ":\n%s" % str(procs)
print(msg)
def shutdown(self, force = False):
@@ -1605,23 +1565,7 @@ class BBCooker:
def clientComplete(self):
"""Called when the client is done using the server"""
if self.configuration.server_only:
self.finishcommand()
else:
self.shutdown(True)
def lockBitbake(self):
if not hasattr(self, 'lock'):
self.lock = None
if self.data:
lockfile = self.data.expand("${TOPDIR}/bitbake.lock")
if lockfile:
self.lock = bb.utils.lockfile(lockfile, False, False)
return self.lock
def unlockBitbake(self):
if hasattr(self, 'lock') and self.lock:
bb.utils.unlockfile(self.lock)
self.finishcommand()
def server_main(cooker, func, *args):
cooker.pre_serve()

5
bitbake/lib/bb/cookerdata.py
View File

@@ -76,7 +76,7 @@ class ConfigParameters(object):
for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp",
"verbose", "debug", "dry_run", "dump_signatures",
"debug_domains", "extra_assume_provided", "profile",
"prefile", "postfile", "tracking"]:
"prefile", "postfile", "tracking", "server_timeout"]:
options[o] = getattr(self.options, o)
ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
@@ -144,7 +144,8 @@ class CookerConfiguration(object):
self.dump_signatures = []
self.dry_run = False
self.tracking = False
self.interface = []
self.xmlrpcinterface = []
self.server_timeout = None
self.writeeventlog = False
self.server_only = False
self.limited_deps = False

230
bitbake/lib/bb/main.py
View File

@@ -28,6 +28,8 @@ import logging
import optparse
import warnings
import fcntl
import time
import traceback
import bb
from bb import event
@@ -37,6 +39,9 @@ from bb import ui
from bb import server
from bb import cookerdata
import bb.server.process
import bb.server.xmlrpcclient
logger = logging.getLogger("BitBake")
class BBMainException(Exception):
@@ -58,9 +63,6 @@ class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
if option.dest == 'ui':
valid_uis = list_extension_modules(bb.ui, 'main')
option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
elif option.dest == 'servertype':
valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
option.help = option.help.replace('@CHOICES@', present_options(valid_server_types))
return optparse.IndentedHelpFormatter.format_option(self, option)
@@ -238,11 +240,6 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
default=os.environ.get('BITBAKE_UI', 'knotty'),
help="The user interface to use (@CHOICES@ - default %default).")
# @CHOICES@ is substituted out by BitbakeHelpFormatter above
parser.add_option("-t", "--servertype", action="store", dest="servertype",
default=["process", "xmlrpc"]["BBSERVER" in os.environ],
help="Choose which server type to use (@CHOICES@ - default %default).")
parser.add_option("", "--token", action="store", dest="xmlrpctoken",
default=os.environ.get("BBTOKEN"),
help="Specify the connection token to be used when connecting "
@@ -258,14 +255,11 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
help="Run bitbake without a UI, only starting a server "
"(cooker) process.")
parser.add_option("", "--foreground", action="store_true",
help="Run bitbake server in foreground.")
parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
help="The name/address for the bitbake server to bind to.")
help="The name/address for the bitbake xmlrpc server to bind to.")
parser.add_option("-T", "--idle-timeout", type=int,
default=int(os.environ.get("BBTIMEOUT", "0")),
parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
default=float(os.environ.get("BB_SERVER_TIMEOUT", 0)) or None,
help="Set timeout to unload bitbake server due to inactivity")
parser.add_option("", "--no-setscene", action="store_true",
@@ -283,7 +277,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
parser.add_option("-m", "--kill-server", action="store_true",
dest="kill_server", default=False,
help="Terminate the remote server.")
help="Terminate the bitbake server.")
parser.add_option("", "--observe-only", action="store_true",
dest="observe_only", default=False,
@@ -322,70 +316,20 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S")
options.writeeventlog = eventlog
# if BBSERVER says to autodetect, let's do that
if options.remote_server:
port = -1
if options.remote_server != 'autostart':
host, port = options.remote_server.split(":", 2)
if options.bind:
try:
#Checking that the port is a number and is a ':' delimited value
(host, port) = options.bind.split(':')
port = int(port)
# use automatic port if port set to -1, means read it from
# the bitbake.lock file; this is a bit tricky, but we always expect
# to be in the base of the build directory if we need to have a
# chance to start the server later, anyway
if port == -1:
lock_location = "./bitbake.lock"
# we try to read the address at all times; if the server is not started,
# we'll try to start it after the first connect fails, below
try:
lf = open(lock_location, 'r')
remotedef = lf.readline()
[host, port] = remotedef.split(":")
port = int(port)
lf.close()
options.remote_server = remotedef
except Exception as e:
if options.remote_server != 'autostart':
raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))
except (ValueError,IndexError):
raise BBMainException("FATAL: Malformed host:port bind parameter")
options.xmlrpcinterface = (host, port)
else:
options.xmlrpcinterface = (None, 0)
return options, targets[1:]
def start_server(servermodule, configParams, configuration, features):
server = servermodule.BitBakeServer()
single_use = not configParams.server_only and os.getenv('BBSERVER') != 'autostart'
if configParams.bind:
(host, port) = configParams.bind.split(':')
server.initServer((host, int(port)), single_use=single_use,
idle_timeout=configParams.idle_timeout)
configuration.interface = [server.serverImpl.host, server.serverImpl.port]
else:
server.initServer(single_use=single_use)
configuration.interface = []
try:
configuration.setServerRegIdleCallback(server.getServerIdleCB())
cooker = bb.cooker.BBCooker(configuration, features)
server.addcooker(cooker)
server.saveConnectionDetails()
except Exception as e:
while hasattr(server, "event_queue"):
import queue
try:
event = server.event_queue.get(block=False)
except (queue.Empty, IOError):
break
if isinstance(event, logging.LogRecord):
logger.handle(event)
raise
if not configParams.foreground:
server.detach()
cooker.shutdown()
cooker.lock.close()
return server
def bitbake_main(configParams, configuration):
# Python multiprocessing requires /dev/shm on Linux
@@ -406,45 +350,15 @@ def bitbake_main(configParams, configuration):
configuration.setConfigParameters(configParams)
if configParams.server_only:
if configParams.servertype != "xmlrpc":
raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
"servertype as 'xmlrpc'.\n")
if not configParams.bind:
raise BBMainException("FATAL: The '--server-only' option requires a name/address "
"to bind to with the -B option.\n")
else:
try:
#Checking that the port is a number
int(configParams.bind.split(":")[1])
except (ValueError,IndexError):
raise BBMainException(
"FATAL: Malformed host:port bind parameter")
if configParams.remote_server:
if configParams.server_only and configParams.remote_server:
raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" %
("the BBSERVER environment variable" if "BBSERVER" in os.environ \
else "the '--remote-server' option"))
elif configParams.foreground:
raise BBMainException("FATAL: The '--foreground' option can only be used "
"with --server-only.\n")
if configParams.bind and configParams.servertype != "xmlrpc":
raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must "
"set the servertype as 'xmlrpc'.\n")
if configParams.remote_server and configParams.servertype != "xmlrpc":
raise BBMainException("FATAL: If '--remote-server' is defined, we must "
"set the servertype as 'xmlrpc'.\n")
if configParams.observe_only and (not configParams.remote_server or configParams.bind):
raise BBMainException("FATAL: '--observe-only' can only be used by UI clients "
"connecting to a server.\n")
if configParams.kill_server and not configParams.remote_server:
raise BBMainException("FATAL: '--kill-server' can only be used to "
"terminate a remote server")
if "BBDEBUG" in os.environ:
level = int(os.environ["BBDEBUG"])
if level > configuration.debug:
@@ -453,7 +367,7 @@ def bitbake_main(configParams, configuration):
bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
configuration.debug_domains)
server, server_connection, ui_module = setup_bitbake(configParams, configuration)
server_connection, ui_module = setup_bitbake(configParams, configuration)
if server_connection is None and configParams.kill_server:
return 0
@@ -463,16 +377,15 @@ def bitbake_main(configParams, configuration):
return 0
try:
for event in bb.event.ui_queue:
server_connection.events.queue_event(event)
bb.event.ui_queue = []
return ui_module.main(server_connection.connection, server_connection.events,
configParams)
finally:
bb.event.ui_queue = []
server_connection.terminate()
else:
print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
server.serverImpl.port))
if configParams.foreground:
server.serverImpl.serve_forever()
return 0
return 1
@@ -495,58 +408,69 @@ def setup_bitbake(configParams, configuration, extrafeatures=None, setup_logging
# Collect the feature set for the UI
featureset = getattr(ui_module, "featureSet", [])
if configParams.server_only:
for param in ('prefile', 'postfile'):
value = getattr(configParams, param)
if value:
setattr(configuration, "%s_server" % param, value)
param = "%s_server" % param
if extrafeatures:
for feature in extrafeatures:
if not feature in featureset:
featureset.append(feature)
servermodule = import_extension_module(bb.server,
configParams.servertype,
'BitBakeServer')
server_connection = None
if configParams.remote_server:
if os.getenv('BBSERVER') == 'autostart':
if configParams.remote_server == 'autostart' or \
not servermodule.check_connection(configParams.remote_server, timeout=2):
configParams.bind = 'localhost:0'
srv = start_server(servermodule, configParams, configuration, featureset)
configParams.remote_server = '%s:%d' % tuple(configuration.interface)
bb.event.ui_queue = []
# we start a stub server that is actually a XMLRPClient that connects to a real server
from bb.server.xmlrpc import BitBakeXMLRPCClient
server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
configParams.xmlrpctoken)
server.saveConnectionDetails(configParams.remote_server)
# Connect to a remote XMLRPC server
server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset,
configParams.observe_only, configParams.xmlrpctoken)
else:
# we start a server with a given configuration
server = start_server(servermodule, configParams, configuration, featureset)
retries = 8
while retries:
try:
topdir, lock = lockBitbake()
sockname = topdir + "/bitbake.sock"
if lock:
# we start a server with a given configuration
logger.info("Starting bitbake server...")
server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset)
# The server will handle any events already in the queue
bb.event.ui_queue = []
else:
logger.info("Reconnecting to bitbake server...")
if not os.path.exists(sockname):
print("Previous bitbake instance shutting down?, waiting to retry...")
time.sleep(5)
raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?")
if not configParams.server_only:
server_connection = bb.server.process.connectProcessServer(sockname, featureset)
if server_connection:
break
except (Exception, bb.server.process.ProcessTimeout) as e:
if not retries:
raise
retries -= 1
if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)):
logger.info("Retrying server connection...")
else:
logger.info("Retrying server connection... (%s)" % traceback.format_exc())
if not retries:
bb.fatal("Unable to connect to bitbake server, or start one")
if retries < 5:
time.sleep(5)
if configParams.kill_server:
server_connection.connection.terminateServer()
server_connection.terminate()
bb.event.ui_queue = []
logger.info("Terminated bitbake server.")
return None, None
if configParams.server_only:
server_connection = None
else:
try:
server_connection = server.establishConnection(featureset)
except Exception as e:
bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
# Restore the environment in case the UI needs it
for k in cleanedvars:
os.environ[k] = cleanedvars[k]
if configParams.kill_server:
server_connection.connection.terminateServer()
bb.event.ui_queue = []
return None, None, None
logger.removeHandler(handler)
server_connection.setupEventQueue()
return server_connection, ui_module
# Restore the environment in case the UI needs it
for k in cleanedvars:
os.environ[k] = cleanedvars[k]
def lockBitbake():
topdir = bb.cookerdata.findTopdir()
lockfile = topdir + "/bitbake.lock"
return topdir, bb.utils.lockfile(lockfile, False, False)
logger.removeHandler(handler)
return server, server_connection, ui_module

6
bitbake/lib/bb/server/__init__.py
View File

@@ -18,10 +18,4 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
""" Base code for Bitbake server process
Have a common base for that all Bitbake server classes ensures a consistent
approach to the interface, and minimize risks associated with code duplication.
"""

612
bitbake/lib/bb/server/process.py
View File

@@ -22,128 +22,205 @@
import bb
import bb.event
import itertools
import logging
import multiprocessing
import threading
import array
import os
import signal
import sys
import time
import select
from queue import Empty
from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
import socket
import subprocess
import errno
import bb.server.xmlrpcserver
from bb import daemonize
from multiprocessing import queues
logger = logging.getLogger('BitBake')
class ServerCommunicator():
def __init__(self, connection, event_handle, server):
self.connection = connection
self.event_handle = event_handle
self.server = server
class ProcessTimeout(SystemExit):
pass
def runCommand(self, command):
# @todo try/except
self.connection.send(command)
if not self.server.is_alive():
raise SystemExit
while True:
# don't let the user ctrl-c while we're waiting for a response
try:
for idx in range(0,4): # 0, 1, 2, 3
if self.connection.poll(5):
return self.connection.recv()
else:
bb.warn("Timeout while attempting to communicate with bitbake server")
bb.fatal("Gave up; Too many tries: timeout while attempting to communicate with bitbake server")
except KeyboardInterrupt:
pass
def getEventHandle(self):
handle, error = self.runCommand(["getUIHandlerNum"])
if error:
logger.error("Unable to get UI Handler Number: %s" % error)
raise BaseException(error)
return handle
class EventAdapter():
"""
Adapter to wrap our event queue since the caller (bb.event) expects to
call a send() method, but our actual queue only has put()
"""
def __init__(self, queue):
self.queue = queue
def send(self, event):
try:
self.queue.put(event)
except Exception as err:
print("EventAdapter puked: %s" % str(err))
class ProcessServer(Process):
class ProcessServer(multiprocessing.Process):
profile_filename = "profile.log"
profile_processed_filename = "profile.log.processed"
def __init__(self, command_channel, event_queue, featurelist):
self._idlefuns = {}
Process.__init__(self)
self.command_channel = command_channel
self.event_queue = event_queue
self.event = EventAdapter(event_queue)
self.featurelist = featurelist
def __init__(self, lock, sock, sockname):
multiprocessing.Process.__init__(self)
self.command_channel = False
self.command_channel_reply = False
self.quit = False
self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
self.next_heartbeat = time.time()
self.quitin, self.quitout = Pipe()
self.event_handle = multiprocessing.Value("i")
self.event_handle = None
self.haveui = False
self.lastui = False
self.xmlrpc = False
self._idlefuns = {}
self.bitbake_lock = lock
self.sock = sock
self.sockname = sockname
def register_idle_function(self, function, data):
"""Register a function to be called while the server is idle"""
assert hasattr(function, '__call__')
self._idlefuns[function] = data
def run(self):
for event in bb.event.ui_queue:
self.event_queue.put(event)
self.event_handle.value = bb.event.register_UIHhandler(self, True)
if self.xmlrpcinterface[0]:
self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self)
print("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port))
heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
if heartbeat_event:
try:
self.heartbeat_seconds = float(heartbeat_event)
except:
# Throwing an exception here causes bitbake to hang.
# Just warn about the invalid setting and continue
bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT')
try:
if self.timeout:
self.timeout = float(self.timeout)
except:
bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
try:
self.bitbake_lock.seek(0)
self.bitbake_lock.truncate()
if self.xmlrpcinterface[0]:
self.bitbake_lock.write("%s %s:%s\n" % (os.getpid(), configuration.interface[0], configuration.interface[1]))
else:
self.bitbake_lock.write("%s\n" % (os.getpid()))
self.bitbake_lock.flush()
except:
pass
bb.cooker.server_main(self.cooker, self.main)
def main(self):
# Ignore SIGINT within the server, as all SIGINT handling is done by
# the UI and communicated to us
self.quitin.close()
signal.signal(signal.SIGINT, signal.SIG_IGN)
bb.utils.set_process_name("Cooker")
ready = []
self.controllersock = False
fds = [self.sock]
if self.xmlrpc:
fds.append(self.xmlrpc)
while not self.quit:
try:
if self.command_channel.poll():
command = self.command_channel.recv()
self.runCommand(command)
if self.quitout.poll():
self.quitout.recv()
if self.command_channel in ready:
command = self.command_channel.get()
if command[0] == "terminateServer":
self.quit = True
continue
try:
print("Running command %s" % command)
self.command_channel_reply.send(self.cooker.command.runCommand(command))
except Exception as e:
logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
if self.xmlrpc in ready:
self.xmlrpc.handle_requests()
if self.sock in ready:
self.controllersock, address = self.sock.accept()
if self.haveui:
print("Dropping connection attempt as we have a UI %s" % (str(ready)))
self.controllersock.close()
else:
print("Accepting %s" % (str(ready)))
fds.append(self.controllersock)
if self.controllersock in ready:
try:
print("Connecting Client")
ui_fds = recvfds(self.controllersock, 3)
# Where to write events to
writer = ConnectionWriter(ui_fds[0])
self.event_handle = bb.event.register_UIHhandler(writer, True)
self.event_writer = writer
# Where to read commands from
reader = ConnectionReader(ui_fds[1])
fds.append(reader)
self.command_channel = reader
# Where to send command return values to
writer = ConnectionWriter(ui_fds[2])
self.command_channel_reply = writer
self.haveui = True
except EOFError:
print("Disconnecting Client")
fds.remove(self.controllersock)
fds.remove(self.command_channel)
bb.event.unregister_UIHhandler(self.event_handle, True)
self.command_channel_reply.writer.close()
self.event_writer.writer.close()
del self.event_writer
self.controllersock.close()
self.haveui = False
self.lastui = time.time()
self.cooker.clientComplete()
if self.timeout is None:
print("No timeout, exiting.")
self.quit = True
if not self.haveui and self.lastui and self.timeout and (self.lastui + self.timeout) < time.time():
print("Server timeout, exiting.")
self.quit = True
ready = self.idle_commands(.1, fds)
print("Exiting")
try:
self.cooker.shutdown(True)
except:
pass
# Remove the socket file so we don't get any more connections to avoid races
os.unlink(self.sockname)
self.sock.close()
# Finally release the lockfile but warn about other processes holding it open
lock = self.bitbake_lock
lockfile = lock.name
lock.close()
lock = None
while not lock:
with bb.utils.timeout(3):
lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
if not lock:
# Some systems may not have lsof available
procs = None
try:
self.runCommand(["stateForceShutdown"])
except:
pass
procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
except OSError as e:
if e.errno != errno.ENOENT:
raise
if procs is None:
# Fall back to fuser if lsof is unavailable
try:
procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
except OSError as e:
if e.errno != errno.ENOENT:
raise
self.idle_commands(.1, [self.command_channel, self.quitout])
except Exception:
logger.exception('Running command %s', command)
self.event_queue.close()
bb.event.unregister_UIHhandler(self.event_handle.value, True)
self.command_channel.close()
self.cooker.shutdown(True)
self.quitout.close()
msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
if procs:
msg += ":\n%s" % str(procs)
print(msg)
return
# We hold the lock so we can remove the file (hide stale pid data)
bb.utils.remove(lockfile)
bb.utils.unlockfile(lock)
def idle_commands(self, delay, fds=None):
nextsleep = delay
@@ -189,140 +266,253 @@ class ProcessServer(Process):
nextsleep = self.next_heartbeat - now
if nextsleep is not None:
if self.xmlrpc:
nextsleep = self.xmlrpc.get_timeout(nextsleep)
try:
select.select(fds,[],[],nextsleep)
return select.select(fds,[],[],nextsleep)[0]
except InterruptedError:
# ignore EINTR error, nextsleep only used for wait
# certain time
pass
# Ignore EINTR
return []
else:
return []
class ServerCommunicator():
def __init__(self, connection, recv):
self.connection = connection
self.recv = recv
def runCommand(self, command):
"""
Run a cooker command on the server
"""
self.command_channel.send(self.cooker.command.runCommand(command))
def stop(self):
self.quitin.send("quit")
self.quitin.close()
self.connection.send(command)
while True:
# don't let the user ctrl-c while we're waiting for a response
try:
for idx in range(0,4): # 0, 1, 2, 3
if self.recv.poll(1):
return self.recv.get()
else:
bb.warn("Timeout while attempting to communicate with bitbake server")
raise ProcessTimeout("Gave up; Too many tries: timeout while attempting to communicate with bitbake server")
except KeyboardInterrupt:
pass
def addcooker(self, cooker):
self.cooker = cooker
def register_idle_function(self, function, data):
"""Register a function to be called while the server is idle"""
assert hasattr(function, '__call__')
self._idlefuns[function] = data
class BitBakeProcessServerConnection(object):
def __init__(self, serverImpl, ui_channel, event_queue):
self.procserver = serverImpl
self.ui_channel = ui_channel
self.event_queue = event_queue
self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver)
self.events = self.event_queue
self.terminated = False
def sigterm_terminate(self):
bb.error("UI received SIGTERM")
self.terminate()
def terminate(self):
if self.terminated:
return
self.terminated = True
def flushevents():
while True:
try:
event = self.event_queue.get(block=False)
except (Empty, IOError):
break
if isinstance(event, logging.LogRecord):
logger.handle(event)
self.procserver.stop()
while self.procserver.is_alive():
flushevents()
self.procserver.join(0.1)
self.ui_channel.close()
self.event_queue.close()
self.event_queue.setexit()
# XXX: Call explicity close in _writer to avoid
# fd leakage because isn't called on Queue.close()
self.event_queue._writer.close()
def setupEventQueue(self):
pass
# Wrap Queue to provide API which isn't server implementation specific
class ProcessEventQueue(multiprocessing.queues.Queue):
def __init__(self, maxsize):
multiprocessing.queues.Queue.__init__(self, maxsize, ctx=multiprocessing.get_context())
self.exit = False
bb.utils.set_process_name("ProcessEQueue")
def setexit(self):
self.exit = True
def waitEvent(self, timeout):
if self.exit:
return self.getEvent()
try:
if not self.server.is_alive():
return self.getEvent()
if timeout == 0:
return self.get(False)
return self.get(True, timeout)
except Empty:
return None
def getEvent(self):
try:
if not self.server.is_alive():
self.setexit()
return self.get(False)
except Empty:
if self.exit:
sys.exit(1)
return None
class BitBakeServer(object):
def initServer(self, single_use=True):
# establish communication channels. We use bidirectional pipes for
# ui <--> server command/response pairs
# and a queue for server -> ui event notifications
#
self.ui_channel, self.server_channel = Pipe()
self.event_queue = ProcessEventQueue(0)
self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None)
self.event_queue.server = self.serverImpl
def detach(self):
self.serverImpl.start()
return
def establishConnection(self, featureset):
self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
_, error = self.connection.connection.runCommand(["setFeatures", featureset])
def updateFeatureSet(self, featureset):
_, error = self.runCommand(["setFeatures", featureset])
if error:
logger.error("Unable to set the cooker to the correct featureset: %s" % error)
raise BaseException(error)
signal.signal(signal.SIGTERM, lambda i, s: self.connection.sigterm_terminate())
return self.connection
def addcooker(self, cooker):
self.cooker = cooker
self.serverImpl.addcooker(cooker)
def getEventHandle(self):
handle, error = self.runCommand(["getUIHandlerNum"])
if error:
logger.error("Unable to get UI Handler Number: %s" % error)
raise BaseException(error)
def getServerIdleCB(self):
return self.serverImpl.register_idle_function
return handle
def saveConnectionDetails(self):
def terminateServer(self):
self.connection.send(['terminateServer'])
return
def endSession(self):
self.connection.terminate()
class BitBakeProcessServerConnection(object):
def __init__(self, ui_channel, recv, eq):
self.connection = ServerCommunicator(ui_channel, recv)
self.events = eq
def terminate(self):
self.socket_connection.close()
return
class BitBakeServer(object):
def __init__(self, lock, sockname, configuration, featureset):
self.configuration = configuration
self.featureset = featureset
self.sockname = sockname
self.bitbake_lock = lock
# Create server control socket
if os.path.exists(sockname):
os.unlink(sockname)
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
try:
os.chdir(os.path.dirname(sockname))
self.sock.bind(os.path.basename(sockname))
finally:
os.chdir(cwd)
self.sock.listen(1)
os.set_inheritable(self.sock.fileno(), True)
bb.daemonize.createDaemon(self._startServer, "bitbake-cookerdaemon.log")
self.sock.close()
self.bitbake_lock.close()
def _startServer(self):
server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
self.configuration.setServerRegIdleCallback(server.register_idle_function)
# Copy prefile and postfile to _server variants
for param in ('prefile', 'postfile'):
value = getattr(self.configuration, param)
if value:
setattr(self.configuration, "%s_server" % param, value)
self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
server.cooker = self.cooker
server.server_timeout = self.configuration.server_timeout
server.xmlrpcinterface = self.configuration.xmlrpcinterface
print("Started bitbake server pid %d" % os.getpid())
server.start()
def connectProcessServer(sockname, featureset):
# Connect to socket
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
try:
os.chdir(os.path.dirname(sockname))
sock.connect(os.path.basename(sockname))
finally:
os.chdir(cwd)
try:
# Send an fd for the remote to write events to
readfd, writefd = os.pipe()
eq = BBUIEventQueue(readfd)
# Send an fd for the remote to recieve commands from
readfd1, writefd1 = os.pipe()
command_chan = ConnectionWriter(writefd1)
# Send an fd for the remote to write commands results to
readfd2, writefd2 = os.pipe()
command_chan_recv = ConnectionReader(readfd2)
sendfds(sock, [writefd, readfd1, writefd2])
server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq)
server_connection.connection.updateFeatureSet(featureset)
# Save sock so it doesn't get gc'd for the life of our connection
server_connection.socket_connection = sock
except:
sock.close()
raise
return server_connection
def sendfds(sock, fds):
'''Send an array of fds over an AF_UNIX socket.'''
fds = array.array('i', fds)
msg = bytes([len(fds) % 256])
sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
def recvfds(sock, size):
'''Receive an array of fds over an AF_UNIX socket.'''
a = array.array('i')
bytes_size = a.itemsize * size
msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))
if not msg and not ancdata:
raise EOFError
try:
if len(ancdata) != 1:
raise RuntimeError('received %d items of ancdata' %
len(ancdata))
cmsg_level, cmsg_type, cmsg_data = ancdata[0]
if (cmsg_level == socket.SOL_SOCKET and
cmsg_type == socket.SCM_RIGHTS):
if len(cmsg_data) % a.itemsize != 0:
raise ValueError
a.frombytes(cmsg_data)
assert len(a) % 256 == msg[0]
return list(a)
except (ValueError, IndexError):
pass
raise RuntimeError('Invalid data received')
class BBUIEventQueue:
def __init__(self, readfd):
self.eventQueue = []
self.eventQueueLock = threading.Lock()
self.eventQueueNotify = threading.Event()
self.reader = ConnectionReader(readfd)
self.t = threading.Thread()
self.t.setDaemon(True)
self.t.run = self.startCallbackHandler
self.t.start()
def getEvent(self):
self.eventQueueLock.acquire()
if len(self.eventQueue) == 0:
self.eventQueueLock.release()
return None
item = self.eventQueue.pop(0)
if len(self.eventQueue) == 0:
self.eventQueueNotify.clear()
self.eventQueueLock.release()
return item
def waitEvent(self, delay):
self.eventQueueNotify.wait(delay)
return self.getEvent()
def queue_event(self, event):
self.eventQueueLock.acquire()
self.eventQueue.append(event)
self.eventQueueNotify.set()
self.eventQueueLock.release()
def send_event(self, event):
self.queue_event(pickle.loads(event))
def startCallbackHandler(self):
bb.utils.set_process_name("UIEventQueue")
while True:
self.reader.wait()
event = self.reader.get()
self.queue_event(event)
class ConnectionReader(object):
def __init__(self, fd):
self.reader = multiprocessing.connection.Connection(fd, writable=False)
self.rlock = multiprocessing.Lock()
def wait(self, timeout=None):
return multiprocessing.connection.wait([self.reader], timeout)
def poll(self, timeout=None):
return self.reader.poll(timeout)
def get(self):
with self.rlock:
res = self.reader.recv_bytes()
return multiprocessing.reduction.ForkingPickler.loads(res)
def fileno(self):
return self.reader.fileno()
class ConnectionWriter(object):
def __init__(self, fd):
self.writer = multiprocessing.connection.Connection(fd, readable=False)
self.wlock = multiprocessing.Lock()
# Why bb.event needs this I have no idea
self.event = self
def send(self, obj):
obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
with self.wlock:
self.writer.send_bytes(obj)

492
bitbake/lib/bb/server/xmlrpc.py
View File

@@ -1,492 +0,0 @@
#
# BitBake XMLRPC Server
#
# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer
# Copyright (C) 2006 - 2008 Richard Purdie
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
This module implements an xmlrpc server for BitBake.
Use this by deriving a class from BitBakeXMLRPCServer and then adding
methods which you want to "export" via XMLRPC. If the methods have the
prefix xmlrpc_, then registering those function will happen automatically,
if not, you need to call register_function.
Use register_idle_function() to add a function which the xmlrpc server
calls from within server_forever when no requests are pending. Make sure
that those functions are non-blocking or else you will introduce latency
in the server's main loop.
"""
import os
import sys
import hashlib
import time
import socket
import signal
import threading
import pickle
import inspect
import select
import http.client
import xmlrpc.client
from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
import bb
from bb import daemonize
from bb.ui import uievent
DEBUG = False
class BBTransport(xmlrpc.client.Transport):
def __init__(self, timeout):
self.timeout = timeout
self.connection_token = None
xmlrpc.client.Transport.__init__(self)
# Modified from default to pass timeout to HTTPConnection
def make_connection(self, host):
#return an existing connection if possible. This allows
#HTTP/1.1 keep-alive.
if self._connection and host == self._connection[0]:
return self._connection[1]
# create a HTTP connection object from a host descriptor
chost, self._extra_headers, x509 = self.get_host_info(host)
#store the host argument along with the connection object
self._connection = host, http.client.HTTPConnection(chost, timeout=self.timeout)
return self._connection[1]
def set_connection_token(self, token):
self.connection_token = token
def send_content(self, h, body):
if self.connection_token:
h.putheader("Bitbake-token", self.connection_token)
xmlrpc.client.Transport.send_content(self, h, body)
def _create_server(host, port, timeout = 60):
t = BBTransport(timeout)
s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True)
return s, t
def check_connection(remote, timeout):
try:
host, port = remote.split(":")
port = int(port)
except Exception as e:
bb.warn("Failed to read remote definition (%s)" % str(e))
raise e
server, _transport = _create_server(host, port, timeout)
try:
ret, err = server.runCommand(['getVariable', 'TOPDIR'])
if err or not ret:
return False
except ConnectionError:
return False
return True
class BitBakeServerCommands():
def __init__(self, server):
self.server = server
self.has_client = False
def registerEventHandler(self, host, port):
"""
Register a remote UI Event Handler
"""
s, t = _create_server(host, port)
# we don't allow connections if the cooker is running
if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.cooker.state)
self.event_handle = bb.event.register_UIHhandler(s, True)
return self.event_handle, 'OK'
def unregisterEventHandler(self, handlerNum):
"""
Unregister a remote UI Event Handler
"""
return bb.event.unregister_UIHhandler(handlerNum, True)
def runCommand(self, command):
"""
Run a cooker command on the server
"""
return self.cooker.command.runCommand(command, self.server.readonly)
def getEventHandle(self):
return self.event_handle
def terminateServer(self):
"""
Trigger the server to quit
"""
self.server.quit = True
print("Server (cooker) exiting")
return
def addClient(self):
if self.has_client:
return None
token = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest()
self.server.set_connection_token(token)
self.has_client = True
return token
def removeClient(self):
if self.has_client:
self.server.set_connection_token(None)
self.has_client = False
if self.server.single_use:
self.server.quit = True
# This request handler checks if the request has a "Bitbake-token" header
# field (this comes from the client side) and compares it with its internal
# "Bitbake-token" field (this comes from the server). If the two are not
# equal, it is assumed that a client is trying to connect to the server
# while another client is connected to the server. In this case, a 503 error
# ("service unavailable") is returned to the client.
class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
def __init__(self, request, client_address, server):
self.server = server
SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server)
def do_POST(self):
try:
remote_token = self.headers["Bitbake-token"]
except:
remote_token = None
if remote_token != self.server.connection_token and remote_token != "observer":
self.report_503()
else:
if remote_token == "observer":
self.server.readonly = True
else:
self.server.readonly = False
SimpleXMLRPCRequestHandler.do_POST(self)
def report_503(self):
self.send_response(503)
response = 'No more client allowed'
self.send_header("Content-type", "text/plain")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(bytes(response, 'utf-8'))
class XMLRPCProxyServer(object):
""" not a real working server, but a stub for a proxy server connection
"""
def __init__(self, host, port, use_builtin_types=True):
self.host = host
self.port = port
self._idlefuns = {}
def addcooker(self, cooker):
self.cooker = cooker
def register_idle_function(self, function, data):
"""Register a function to be called while the server is idle"""
assert hasattr(function, '__call__')
self._idlefuns[function] = data
class XMLRPCServer(SimpleXMLRPCServer):
# remove this when you're done with debugging
# allow_reuse_address = True
def __init__(self, interface, single_use=False, idle_timeout=0):
"""
Constructor
"""
self._idlefuns = {}
self.single_use = single_use
# Use auto port configuration
if (interface[1] == -1):
interface = (interface[0], 0)
SimpleXMLRPCServer.__init__(self, interface,
requestHandler=BitBakeXMLRPCRequestHandler,
logRequests=False, allow_none=True)
self.host, self.port = self.socket.getsockname()
self.connection_token = None
#self.register_introspection_functions()
self.commands = BitBakeServerCommands(self)
self.autoregister_all_functions(self.commands, "")
self.interface = interface
self.time = time.time()
self.idle_timeout = idle_timeout
if idle_timeout:
self.register_idle_function(self.handle_idle_timeout, self)
self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
self.next_heartbeat = time.time()
def addcooker(self, cooker):
self.cooker = cooker
self.commands.cooker = cooker
def autoregister_all_functions(self, context, prefix):
"""
Convenience method for registering all functions in the scope
of this class that start with a common prefix
"""
methodlist = inspect.getmembers(context, inspect.ismethod)
for name, method in methodlist:
if name.startswith(prefix):
self.register_function(method, name[len(prefix):])
def handle_idle_timeout(self, server, data, abort):
if not abort:
if time.time() - server.time > server.idle_timeout:
server.quit = True
print("Server idle timeout expired")
return []
def serve_forever(self):
heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
if heartbeat_event:
try:
self.heartbeat_seconds = float(heartbeat_event)
except:
# Throwing an exception here causes bitbake to hang.
# Just warn about the invalid setting and continue
bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
# Start the actual XMLRPC server
bb.cooker.server_main(self.cooker, self._serve_forever)
def _serve_forever(self):
"""
Serve Requests. Overloaded to honor a quit command
"""
self.quit = False
while not self.quit:
fds = [self]
nextsleep = 0.1
for function, data in list(self._idlefuns.items()):
retval = None
try:
retval = function(self, data, False)
if retval is False:
del self._idlefuns[function]
elif retval is True:
nextsleep = 0
elif isinstance(retval, float):
if (retval < nextsleep):
nextsleep = retval
else:
fds = fds + retval
except SystemExit:
raise
except:
import traceback
traceback.print_exc()
if retval == None:
# the function execute failed; delete it
del self._idlefuns[function]
pass
socktimeout = self.socket.gettimeout() or nextsleep
socktimeout = min(socktimeout, nextsleep)
# Mirror what BaseServer handle_request would do
try:
fd_sets = select.select(fds, [], [], socktimeout)
if fd_sets[0] and self in fd_sets[0]:
if self.idle_timeout:
self.time = time.time()
self._handle_request_noblock()
except IOError:
# we ignore interrupted calls
pass
# Create new heartbeat event?
now = time.time()
if now >= self.next_heartbeat:
# We might have missed heartbeats. Just trigger once in
# that case and continue after the usual delay.
self.next_heartbeat += self.heartbeat_seconds
if self.next_heartbeat <= now:
self.next_heartbeat = now + self.heartbeat_seconds
heartbeat = bb.event.HeartbeatEvent(now)
bb.event.fire(heartbeat, self.cooker.data)
if nextsleep and now + nextsleep > self.next_heartbeat:
# Shorten timeout so that we we wake up in time for
# the heartbeat.
nextsleep = self.next_heartbeat - now
# Tell idle functions we're exiting
for function, data in list(self._idlefuns.items()):
try:
retval = function(self, data, True)
except:
pass
self.server_close()
return
def set_connection_token(self, token):
self.connection_token = token
def register_idle_function(self, function, data):
"""Register a function to be called while the server is idle"""
assert hasattr(function, '__call__')
self._idlefuns[function] = data
class BitBakeXMLRPCServerConnection(object):
def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None):
self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
self.clientinfo = clientinfo
self.serverImpl = serverImpl
self.observer_only = observer_only
if featureset:
self.featureset = featureset
else:
self.featureset = []
def connect(self, token = None):
if token is None:
if self.observer_only:
token = "observer"
else:
token = self.connection.addClient()
if token is None:
return None
self.transport.set_connection_token(token)
return self
def setupEventQueue(self):
self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
for event in bb.event.ui_queue:
self.events.queue_event(event)
_, error = self.connection.runCommand(["setFeatures", self.featureset])
if error:
# disconnect the client, we can't make the setFeature work
self.connection.removeClient()
# no need to log it here, the error shall be sent to the client
raise BaseException(error)
def removeClient(self):
if not self.observer_only:
self.connection.removeClient()
def terminate(self):
# Don't wait for server indefinitely
import socket
socket.setdefaulttimeout(2)
try:
self.events.system_quit()
except:
pass
try:
self.connection.removeClient()
except:
pass
class BitBakeServer(object):
def initServer(self, interface = ("localhost", 0),
single_use = False, idle_timeout=0):
self.interface = interface
self.serverImpl = XMLRPCServer(interface, single_use, idle_timeout)
def detach(self):
daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
del self.cooker
def establishConnection(self, featureset):
self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
return self.connection.connect()
def set_connection_token(self, token):
self.connection.transport.set_connection_token(token)
def addcooker(self, cooker):
self.cooker = cooker
self.serverImpl.addcooker(cooker)
def getServerIdleCB(self):
return self.serverImpl.register_idle_function
def saveConnectionDetails(self):
return
def endSession(self):
self.connection.terminate()
class BitBakeXMLRPCClient(object):
def __init__(self, observer_only = False, token = None):
self.token = token
self.observer_only = observer_only
# if we need extra caches, just tell the server to load them all
pass
def saveConnectionDetails(self, remote):
self.remote = remote
def establishConnection(self, featureset):
# The format of "remote" must be "server:port"
try:
[host, port] = self.remote.split(":")
port = int(port)
except Exception as e:
bb.warn("Failed to read remote definition (%s)" % str(e))
raise e
# We need our IP for the server connection. We get the IP
# by trying to connect with the server
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect((host, port))
ip = s.getsockname()[0]
s.close()
except Exception as e:
bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
raise e
try:
self.serverImpl = XMLRPCProxyServer(host, port, use_builtin_types=True)
self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
return self.connection.connect(self.token)
except Exception as e:
bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
raise e
def endSession(self):
self.connection.removeClient()
def initServer(self):
self.serverImpl = None
self.connection = None
return
def addcooker(self, cooker):
self.cooker = cooker
self.serverImpl.addcooker(cooker)
def getServerIdleCB(self):
return self.serverImpl.register_idle_function
def detach(self):
return

154
bitbake/lib/bb/server/xmlrpcclient.py Normal file
View File

@@ -0,0 +1,154 @@
#
# BitBake XMLRPC Client Interface
#
# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer
# Copyright (C) 2006 - 2008 Richard Purdie
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import socket
import http.client
import xmlrpc.client
import bb
from bb.ui import uievent
class BBTransport(xmlrpc.client.Transport):
def __init__(self, timeout):
self.timeout = timeout
self.connection_token = None
xmlrpc.client.Transport.__init__(self)
# Modified from default to pass timeout to HTTPConnection
def make_connection(self, host):
#return an existing connection if possible. This allows
#HTTP/1.1 keep-alive.
if self._connection and host == self._connection[0]:
return self._connection[1]
# create a HTTP connection object from a host descriptor
chost, self._extra_headers, x509 = self.get_host_info(host)
#store the host argument along with the connection object
self._connection = host, http.client.HTTPConnection(chost, timeout=self.timeout)
return self._connection[1]
def set_connection_token(self, token):
self.connection_token = token
def send_content(self, h, body):
if self.connection_token:
h.putheader("Bitbake-token", self.connection_token)
xmlrpc.client.Transport.send_content(self, h, body)
def _create_server(host, port, timeout = 60):
t = BBTransport(timeout)
s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True)
return s, t
def check_connection(remote, timeout):
try:
host, port = remote.split(":")
port = int(port)
except Exception as e:
bb.warn("Failed to read remote definition (%s)" % str(e))
raise e
server, _transport = _create_server(host, port, timeout)
try:
ret, err = server.runCommand(['getVariable', 'TOPDIR'])
if err or not ret:
return False
except ConnectionError:
return False
return True
class BitBakeXMLRPCServerConnection(object):
def __init__(self, host, port, clientinfo=("localhost", 0), observer_only = False, featureset = None):
self.connection, self.transport = _create_server(host, port)
self.clientinfo = clientinfo
self.observer_only = observer_only
if featureset:
self.featureset = featureset
else:
self.featureset = []
self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
_, error = self.connection.runCommand(["setFeatures", self.featureset])
if error:
# disconnect the client, we can't make the setFeature work
self.connection.removeClient()
# no need to log it here, the error shall be sent to the client
raise BaseException(error)
def connect(self, token = None):
if token is None:
if self.observer_only:
token = "observer"
else:
token = self.connection.addClient()
if token is None:
return None
self.transport.set_connection_token(token)
return self
def removeClient(self):
if not self.observer_only:
self.connection.removeClient()
def terminate(self):
# Don't wait for server indefinitely
socket.setdefaulttimeout(2)
try:
self.events.system_quit()
except:
pass
try:
self.connection.removeClient()
except:
pass
def connectXMLRPC(remote, featureset, observer_only = False, token = None):
# The format of "remote" must be "server:port"
try:
[host, port] = remote.split(":")
port = int(port)
except Exception as e:
bb.warn("Failed to parse remote definition %s (%s)" % (remote, str(e)))
raise e
# We need our IP for the server connection. We get the IP
# by trying to connect with the server
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect((host, port))
ip = s.getsockname()[0]
s.close()
except Exception as e:
bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
raise e
try:
connection = BitBakeXMLRPCServerConnection(host, port, (ip, 0), observer_only, featureset)
return connection.connect(token)
except Exception as e:
bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
raise e

158
bitbake/lib/bb/server/xmlrpcserver.py Normal file
View File

@@ -0,0 +1,158 @@
#
# BitBake XMLRPC Server Interface
#
# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer
# Copyright (C) 2006 - 2008 Richard Purdie
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import hashlib
import time
import inspect
from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
import bb
# This request handler checks if the request has a "Bitbake-token" header
# field (this comes from the client side) and compares it with its internal
# "Bitbake-token" field (this comes from the server). If the two are not
# equal, it is assumed that a client is trying to connect to the server
# while another client is connected to the server. In this case, a 503 error
# ("service unavailable") is returned to the client.
class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
def __init__(self, request, client_address, server):
self.server = server
SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server)
def do_POST(self):
try:
remote_token = self.headers["Bitbake-token"]
except:
remote_token = None
if 0 and remote_token != self.server.connection_token and remote_token != "observer":
self.report_503()
else:
if remote_token == "observer":
self.server.readonly = True
else:
self.server.readonly = False
SimpleXMLRPCRequestHandler.do_POST(self)
def report_503(self):
self.send_response(503)
response = 'No more client allowed'
self.send_header("Content-type", "text/plain")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(bytes(response, 'utf-8'))
class BitBakeXMLRPCServer(SimpleXMLRPCServer):
# remove this when you're done with debugging
# allow_reuse_address = True
def __init__(self, interface, cooker, parent):
# Use auto port configuration
if (interface[1] == -1):
interface = (interface[0], 0)
SimpleXMLRPCServer.__init__(self, interface,
requestHandler=BitBakeXMLRPCRequestHandler,
logRequests=False, allow_none=True)
self.host, self.port = self.socket.getsockname()
self.interface = interface
self.connection_token = None
self.commands = BitBakeXMLRPCServerCommands(self)
self.register_functions(self.commands, "")
self.cooker = cooker
self.parent = parent
def register_functions(self, context, prefix):
"""
Convenience method for registering all functions in the scope
of this class that start with a common prefix
"""
methodlist = inspect.getmembers(context, inspect.ismethod)
for name, method in methodlist:
if name.startswith(prefix):
self.register_function(method, name[len(prefix):])
def get_timeout(self, delay):
socktimeout = self.socket.gettimeout() or delay
return min(socktimeout, delay)
def handle_requests(self):
self._handle_request_noblock()
class BitBakeXMLRPCServerCommands():
def __init__(self, server):
self.server = server
self.has_client = False
def registerEventHandler(self, host, port):
"""
Register a remote UI Event Handler
"""
s, t = bb.server.xmlrpcclient._create_server(host, port)
# we don't allow connections if the cooker is running
if (self.server.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.server.cooker.state)
self.event_handle = bb.event.register_UIHhandler(s, True)
return self.event_handle, 'OK'
def unregisterEventHandler(self, handlerNum):
"""
Unregister a remote UI Event Handler
"""
ret = bb.event.unregister_UIHhandler(handlerNum, True)
self.event_handle = None
return ret
def runCommand(self, command):
"""
Run a cooker command on the server
"""
return self.server.cooker.command.runCommand(command, self.server.readonly)
def getEventHandle(self):
return self.event_handle
def terminateServer(self):
"""
Trigger the server to quit
"""
self.server.parent.quit = True
print("XMLRPC Server triggering exit")
return
def addClient(self):
if self.server.parent.haveui:
return None
token = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest()
self.server.connection_token = token
self.server.parent.haveui = True
return token
def removeClient(self):
if self.server.parent.haveui:
self.server.connection_token = None
self.server.parent.haveui = False

2
bitbake/lib/bb/tinfoil.py
View File

@@ -243,7 +243,7 @@ class Tinfoil:
cookerconfig = CookerConfiguration()
cookerconfig.setConfigParameters(config_params)
server, self.server_connection, ui_module = setup_bitbake(config_params,
self.server_connection, ui_module = setup_bitbake(config_params,
cookerconfig,
extrafeatures,
setup_logging=False)

4
bitbake/lib/prserv/serv.py
View File

@@ -6,7 +6,7 @@ import queue
import socket
import io
import sqlite3
import bb.server.xmlrpc
import bb.server.xmlrpcclient
import prserv
import prserv.db
import errno
@@ -300,7 +300,7 @@ class PRServerConnection(object):
host, port = singleton.getinfo()
self.host = host
self.port = port
self.connection, self.transport = bb.server.xmlrpc._create_server(self.host, self.port)
self.connection, self.transport = bb.server.xmlrpcclient._create_server(self.host, self.port)
def terminate(self):
try: