bitbake: BBHandler: use with instead of open/close

This is more pythonic and can handle unclosed file warnings better
than the previous code structure.

(Bitbake rev: 50633012a64a3b5f0662145e29ff426374fb7683)

Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

README.hardware

@@ -317,7 +317,7 @@ Steps:
 
  2. Flash the image.
 
-    # dd core-image-minimal-edgerouter.wic of=/dev/sdb
+    # dd if=core-image-minimal-edgerouter.wic of=/dev/sdb
 
  3. Insert USB disk into the edgerouter and boot it.
 

bitbake/bin/bitbake-layers

@@ -89,7 +89,7 @@ def main():
     tinfoil = tinfoil_init(False)
     try:
         for path in ([topdir] +
-                    tinfoil.config_data.getVar('BBPATH', True).split(':')):
+                    tinfoil.config_data.getVar('BBPATH').split(':')):
             pluginpath = os.path.join(path, 'lib', 'bblayers')
             bb.utils.load_plugins(logger, plugins, pluginpath)
 

bitbake/bin/bitbake-worker

@@ -11,7 +11,10 @@ import select
 import errno
 import signal
 import pickle
+import traceback
+import queue
 from multiprocessing import Lock
+from threading import Thread
 
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
@@ -63,7 +66,7 @@ if 0:
     consolelog.setFormatter(conlogformat)
     logger.addHandler(consolelog)
 
-worker_queue = b""
+worker_queue = queue.Queue()
 
 def worker_fire(event, d):
     data = b"<event>" + pickle.dumps(event) + b"</event>"
@@ -72,21 +75,39 @@ def worker_fire(event, d):
 def worker_fire_prepickled(event):
     global worker_queue
 
-    worker_queue = worker_queue + event
-    worker_flush()
+    worker_queue.put(event)
 
-def worker_flush():
-    global worker_queue, worker_pipe
+#
+# We can end up with write contention with the cooker, it can be trying to send commands
+# and we can be trying to send event data back. Therefore use a separate thread for writing 
+# back data to cooker.
+#
+worker_thread_exit = False
 
-    if not worker_queue:
-        return
+def worker_flush(worker_queue):
+    worker_queue_int = b""
+    global worker_pipe, worker_thread_exit
 
-    try:
-        written = os.write(worker_pipe, worker_queue)
-        worker_queue = worker_queue[written:]
-    except (IOError, OSError) as e:
-        if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
-            raise
+    while True:
+        try:
+            worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
+        except queue.Empty:
+            pass
+        while (worker_queue_int or not worker_queue.empty()):
+            try:
+                (_, ready, _) = select.select([], [worker_pipe], [], 1)
+                if not worker_queue.empty():
+                    worker_queue_int = worker_queue_int + worker_queue.get()
+                written = os.write(worker_pipe, worker_queue_int)
+                worker_queue_int = worker_queue_int[written:]
+            except (IOError, OSError) as e:
+                if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
+                    raise
+        if worker_thread_exit and worker_queue.empty() and not worker_queue_int:
+            return
+
+worker_thread = Thread(target=worker_flush, args=(worker_queue,))
+worker_thread.start()
 
 def worker_child_fire(event, d):
     global worker_pipe
@@ -115,7 +136,7 @@ def sigterm_handler(signum, frame):
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False):
+def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, extraconfigdata, quieterrors=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -202,13 +223,16 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
                 the_data.setVar("BUILDNAME", workerdata["buildname"])
                 the_data.setVar("DATE", workerdata["date"])
                 the_data.setVar("TIME", workerdata["time"])
+                for varname, value in extraconfigdata.items():
+                    the_data.setVar(varname, value)
+
                 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
                 ret = 0
 
                 the_data = bb_cache.loadDataFull(fn, appends)
                 the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task])
 
-                bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN", True), taskname.replace("do_", "")))
+                bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
 
                 # exported_vars() returns a generator which *cannot* be passed to os.environ.update() 
                 # successfully. We also need to unset anything from the environment which shouldn't be there 
@@ -223,20 +247,20 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
                     the_data.setVar(e, fakeenv[e])
                     the_data.setVarFlag(e, 'export', "1")
 
-                task_exports = the_data.getVarFlag(taskname, 'exports', True)
+                task_exports = the_data.getVarFlag(taskname, 'exports')
                 if task_exports:
                     for e in task_exports.split():
                         the_data.setVarFlag(e, 'export', '1')
-                        v = the_data.getVar(e, True)
+                        v = the_data.getVar(e)
                         if v is not None:
                             os.environ[e] = v
 
                 if quieterrors:
                     the_data.setVarFlag(taskname, "quieterrors", "1")
 
-            except Exception as exc:
+            except Exception:
                 if not quieterrors:
-                    logger.critical(str(exc))
+                    logger.critical(traceback.format_exc())
                 os._exit(1)
             try:
                 if cfg.dry_run:
@@ -308,6 +332,7 @@ class BitbakeWorker(object):
         self.cookercfg = None
         self.databuilder = None
         self.data = None
+        self.extraconfigdata = None
         self.build_pids = {}
         self.build_pipes = {}
     
@@ -342,6 +367,7 @@ class BitbakeWorker(object):
                     pass
             if len(self.queue):
                 self.handle_item(b"cookerconfig", self.handle_cookercfg)
+                self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
                 self.handle_item(b"workerdata", self.handle_workerdata)
                 self.handle_item(b"runtask", self.handle_runtask)
                 self.handle_item(b"finishnow", self.handle_finishnow)
@@ -349,10 +375,11 @@ class BitbakeWorker(object):
                 self.handle_item(b"quit", self.handle_quit)
 
             for pipe in self.build_pipes:
-                self.build_pipes[pipe].read()
+                if self.build_pipes[pipe].input in ready:
+                    self.build_pipes[pipe].read()
             if len(self.build_pids):
-                self.process_waitpid()
-            worker_flush()
+                while self.process_waitpid():
+                    continue
 
 
     def handle_item(self, item, func):
@@ -369,6 +396,9 @@ class BitbakeWorker(object):
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
 
+    def handle_extraconfigdata(self, data):
+        self.extraconfigdata = pickle.loads(data)
+
     def handle_workerdata(self, data):
         self.workerdata = pickle.loads(data)
         bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"]
@@ -394,7 +424,7 @@ class BitbakeWorker(object):
         fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, self.extraconfigdata, quieterrors)
 
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
@@ -407,9 +437,9 @@ class BitbakeWorker(object):
         try:
             pid, status = os.waitpid(-1, os.WNOHANG)
             if pid == 0 or os.WIFSTOPPED(status):
-                return None
+                return False
         except OSError:
-            return None
+            return False
 
         workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
 
@@ -428,6 +458,8 @@ class BitbakeWorker(object):
 
         worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
 
+        return True
+
     def handle_finishnow(self, _):
         if self.build_pids:
             logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
@@ -457,8 +489,10 @@ except BaseException as e:
         import traceback
         sys.stderr.write(traceback.format_exc())
         sys.stderr.write(str(e))
-while len(worker_queue):
-    worker_flush()
+
+worker_thread_exit = True
+worker_thread.join()
+
 workerlog_write("exitting")
 sys.exit(0)
 

bitbake/bin/toaster

@@ -111,7 +111,7 @@ verify_prereq() {
     if ! sed -n "$exp" $reqfile | python3 - ; then
         req=`grep ^Django $reqfile`
         echo "This program needs $req"
-        echo "Please install with pip install -r $reqfile"
+        echo "Please install with pip3 install -r $reqfile"
         return 2
     fi
 
@@ -254,7 +254,7 @@ case $CMD in
             return 4
         fi
         export BITBAKE_UI='toasterui'
-        $MANAGE runbuilds & echo $! >${BUILDDIR}/.runbuilds.pid
+        $MANAGE runbuilds &
         # set fail safe stop system on terminal exit
         trap stop_system SIGHUP
         echo "Successful ${CMD}."

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -38,7 +38,7 @@
             The code to execute the first part of this process, a fetch,
             looks something like the following:
             <literallayout class='monospaced'>
-     src_uri = (d.getVar('SRC_URI', True) or "").split()
+     src_uri = (d.getVar('SRC_URI') or "").split()
      fetcher = bb.fetch2.Fetch(src_uri, d)
      fetcher.download()
             </literallayout>
@@ -52,7 +52,7 @@
         <para>
             The instantiation of the fetch class is usually followed by:
             <literallayout class='monospaced'>
-     rootdir = l.getVar('WORKDIR', True)
+     rootdir = l.getVar('WORKDIR')
      fetcher.unpack(rootdir)
             </literallayout>
             This code unpacks the downloaded files to the

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -529,8 +529,8 @@
        -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS
                              Show debug logging for the specified logging domains
        -P, --profile         Profile the command and save reports.
-       -u UI, --ui=UI        The user interface to use (depexp, goggle, hob, knotty
-                             or ncurses - default knotty).
+       -u UI, --ui=UI        The user interface to use (depexp, goggle, knotty or
+                             ncurses - default knotty).
        -t SERVERTYPE, --servertype=SERVERTYPE
                              Choose which server type to use (process or xmlrpc -
                              default process).
@@ -543,6 +543,7 @@
        -B BIND, --bind=BIND  The name/address for the bitbake server to bind to.
        --no-setscene         Do not run any setscene tasks. sstate will be ignored
                              and everything needed, built.
+       --setscene-only       Only run setscene tasks, don't run any real tasks.
        --remote-server=REMOTE_SERVER
                              Connect to the specified server.
        -m, --kill-server     Terminate the remote server.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -61,6 +61,48 @@
             </para>
         </section>
 
+        <section id='line-joining'>
+            <title>Line Joining</title>
+
+            <para>
+                Outside of
+                <link linkend='functions'>functions</link>, BitBake joins
+                any line ending in a backslash character ("\")
+                with the following line before parsing statements.
+                The most common use for the "\" character is to split variable
+                assignments over multiple lines, as in the following example:
+                <literallayout class='monospaced'>
+     FOO = "bar \
+            baz \
+            qaz"
+                </literallayout>
+                Both the "\" character and the newline character
+                that follow it are removed when joining lines.
+                Thus, no newline characters end up in the value of
+                <filename>FOO</filename>.
+            </para>
+
+            <para>
+                Consider this additional example where the two
+                assignments both assign "barbaz" to
+                <filename>FOO</filename>:
+                <literallayout class='monospaced'>
+     FOO = "barbaz"
+
+     FOO = "bar\
+     baz"
+                </literallayout>
+                <note>
+                    BitBake does not interpret escape sequences like
+                    "\n" in variable values.
+                    For these to have an effect, the value must be passed
+                    to some utility that interprets escape sequences,
+                    such as <filename>printf</filename> or
+                    <filename>echo -n</filename>.
+                </note>
+            </para>
+        </section>
+
         <section id='variable-expansion'>
             <title>Variable Expansion</title>
 
@@ -1165,7 +1207,7 @@
                 <literallayout class='monospaced'>
      python some_python_function () {
          d.setVar("TEXT", "Hello World")
-         print d.getVar("TEXT", True)
+         print d.getVar("TEXT")
      }
                 </literallayout>
                 Because the Python "bb" and "os" modules are already
@@ -1180,7 +1222,7 @@
                     to freely set variable values to expandable expressions
                     without having them expanded prematurely.
                     If you do wish to expand a variable within a Python
-                    function, use <filename>d.getVar("X", True)</filename>.
+                    function, use <filename>d.getVar("X")</filename>.
                     Or, for more complicated expressions, use
                     <filename>d.expand()</filename>.
                 </note>
@@ -1232,7 +1274,7 @@
                 Here is an example:
                 <literallayout class='monospaced'>
      def get_depends(d):
-         if d.getVar('SOMECONDITION', True):
+         if d.getVar('SOMECONDITION'):
              return "dependencywithcond"
          else:
              return "dependency"
@@ -1367,7 +1409,7 @@
                 based on the value of  another variable:
                 <literallayout class='monospaced'>
      python () {
-         if d.getVar('SOMEVAR', True) == 'value':
+         if d.getVar('SOMEVAR') == 'value':
              d.setVar('ANOTHERVAR', 'value2')
      }
                 </literallayout>

bitbake/lib/bb/COW.py

@@ -213,11 +213,11 @@ if __name__ == "__main__":
     print()
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
-    for x in b.items():
+    for x in b.iteritems():
         print(x)
     print()
 
@@ -225,11 +225,11 @@ if __name__ == "__main__":
     b['a'] = 'c'
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
-    for x in b.items():
+    for x in b.iteritems():
         print(x)
     print()
 
@@ -244,22 +244,22 @@ if __name__ == "__main__":
     a['set'].add("o2")
 
     print("a", a)
-    for x in a['set'].values():
+    for x in a['set'].itervalues():
         print(x)
     print("--")
     print("b", b)
-    for x in b['set'].values():
+    for x in b['set'].itervalues():
         print(x)
     print()
 
     b['set'].add('o3')
 
     print("a", a)
-    for x in a['set'].values():
+    for x in a['set'].itervalues():
         print(x)
     print("--")
     print("b", b)
-    for x in b['set'].values():
+    for x in b['set'].itervalues():
         print(x)
     print()
 
@@ -269,7 +269,7 @@ if __name__ == "__main__":
     a['set2'].add("o2")
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
@@ -289,7 +289,7 @@ if __name__ == "__main__":
         print("Yay - has_key with delete works!")
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
@@ -300,7 +300,7 @@ if __name__ == "__main__":
     b.__revertitem__('b')
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
@@ -310,7 +310,7 @@ if __name__ == "__main__":
 
     b.__revertitem__('dict')
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)

bitbake/lib/bb/build.py

@@ -91,13 +91,13 @@ class TaskBase(event.Event):
 
     def __init__(self, t, logfile, d):
         self._task = t
-        self._package = d.getVar("PF", True)
-        self.taskfile = d.getVar("FILE", True)
+        self._package = d.getVar("PF")
+        self.taskfile = d.getVar("FILE")
         self.taskname = self._task
         self.logfile = logfile
         self.time = time.time()
         event.Event.__init__(self)
-        self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
+        self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName())
 
     def getTask(self):
         return self._task
@@ -226,17 +226,17 @@ def exec_func(func, d, dirs = None, pythonexception=False):
     else:
         lockfiles = None
 
-    tempdir = d.getVar('T', True)
+    tempdir = d.getVar('T')
 
     # or func allows items to be executed outside of the normal
     # task set, such as buildhistory
-    task = d.getVar('BB_RUNTASK', True) or func
+    task = d.getVar('BB_RUNTASK') or func
     if task == func:
         taskfunc = task
     else:
         taskfunc = "%s.%s" % (task, func)
 
-    runfmt = d.getVar('BB_RUNFMT', True) or "run.{func}.{pid}"
+    runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}"
     runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid())
     runfile = os.path.join(tempdir, runfn)
     bb.utils.mkdirhier(os.path.dirname(runfile))
@@ -368,7 +368,7 @@ exit $ret
 
     cmd = runfile
     if d.getVarFlag(func, 'fakeroot', False):
-        fakerootcmd = d.getVar('FAKEROOT', True)
+        fakerootcmd = d.getVar('FAKEROOT')
         if fakerootcmd:
             cmd = [fakerootcmd, runfile]
 
@@ -377,7 +377,7 @@ exit $ret
     else:
         logfile = sys.stdout
 
-    progress = d.getVarFlag(func, 'progress', True)
+    progress = d.getVarFlag(func, 'progress')
     if progress:
         if progress == 'percent':
             # Use default regex
@@ -429,7 +429,7 @@ exit $ret
             else:
                 break
 
-    tempdir = d.getVar('T', True)
+    tempdir = d.getVar('T')
     fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
     if os.path.exists(fifopath):
         os.unlink(fifopath)
@@ -442,7 +442,7 @@ exit $ret
                 with open(os.devnull, 'r+') as stdin:
                     bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
             except bb.process.CmdError:
-                logfn = d.getVar('BB_LOGFILE', True)
+                logfn = d.getVar('BB_LOGFILE')
                 raise FuncFailed(func, logfn)
         finally:
             os.unlink(fifopath)
@@ -473,18 +473,18 @@ def _exec_task(fn, task, d, quieterr):
     logger.debug(1, "Executing task %s", task)
 
     localdata = _task_data(fn, task, d)
-    tempdir = localdata.getVar('T', True)
+    tempdir = localdata.getVar('T')
     if not tempdir:
         bb.fatal("T variable not set, unable to build")
 
     # Change nice level if we're asked to
-    nice = localdata.getVar("BB_TASK_NICE_LEVEL", True)
+    nice = localdata.getVar("BB_TASK_NICE_LEVEL")
     if nice:
         curnice = os.nice(0)
         nice = int(nice) - curnice
         newnice = os.nice(nice)
         logger.debug(1, "Renice to %s " % newnice)
-    ionice = localdata.getVar("BB_TASK_IONICE_LEVEL", True)
+    ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
     if ionice:
         try:
             cls, prio = ionice.split(".", 1)
@@ -495,7 +495,7 @@ def _exec_task(fn, task, d, quieterr):
     bb.utils.mkdirhier(tempdir)
 
     # Determine the logfile to generate
-    logfmt = localdata.getVar('BB_LOGFMT', True) or 'log.{task}.{pid}'
+    logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}'
     logbase = logfmt.format(task=task, pid=os.getpid())
 
     # Document the order of the tasks...
@@ -562,6 +562,7 @@ def _exec_task(fn, task, d, quieterr):
 
     localdata.setVar('BB_LOGFILE', logfn)
     localdata.setVar('BB_RUNTASK', task)
+    localdata.setVar('BB_TASK_LOGGER', bblogger)
 
     flags = localdata.getVarFlags(task)
 
@@ -627,7 +628,7 @@ def exec_task(fn, task, d, profile = False):
             quieterr = True
 
         if profile:
-            profname = "profile-%s.log" % (d.getVar("PN", True) + "-" + task)
+            profname = "profile-%s.log" % (d.getVar("PN") + "-" + task)
             try:
                 import cProfile as profile
             except:
@@ -667,9 +668,9 @@ def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False):
         stamp = d.stamp[file_name]
         extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
     else:
-        stamp = d.getVar('STAMP', True)
-        file_name = d.getVar('BB_FILENAME', True)
-        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
+        stamp = d.getVar('STAMP')
+        file_name = d.getVar('BB_FILENAME')
+        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
 
     if baseonly:
         return stamp
@@ -703,9 +704,9 @@ def stamp_cleanmask_internal(taskname, d, file_name):
         stamp = d.stampclean[file_name]
         extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
     else:
-        stamp = d.getVar('STAMPCLEAN', True)
-        file_name = d.getVar('BB_FILENAME', True)
-        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
+        stamp = d.getVar('STAMPCLEAN')
+        file_name = d.getVar('BB_FILENAME')
+        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
 
     if not stamp:
         return []
@@ -723,7 +724,7 @@ def make_stamp(task, d, file_name = None):
     for mask in cleanmask:
         for name in glob.glob(mask):
             # Preserve sigdata files in the stamps directory
-            if "sigdata" in name:
+            if "sigdata" in name or "sigbasedata" in name:
                 continue
             # Preserve taint files in the stamps directory
             if name.endswith('.taint'):
@@ -741,7 +742,7 @@ def make_stamp(task, d, file_name = None):
     # as it completes
     if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
         stampbase = stamp_internal(task, d, None, True)
-        file_name = d.getVar('BB_FILENAME', True)
+        file_name = d.getVar('BB_FILENAME')
         bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
 
 def del_stamp(task, d, file_name = None):
@@ -763,7 +764,7 @@ def write_taint(task, d, file_name = None):
     if file_name:
         taintfn = d.stamp[file_name] + '.' + task + '.taint'
     else:
-        taintfn = d.getVar('STAMP', True) + '.' + task + '.taint'
+        taintfn = d.getVar('STAMP') + '.' + task + '.taint'
     bb.utils.mkdirhier(os.path.dirname(taintfn))
     # The specific content of the taint file is not really important,
     # we just need it to be random, so a random UUID is used

bitbake/lib/bb/cache.py

@@ -71,7 +71,7 @@ class RecipeInfoCommon(object):
 
     @classmethod
     def flaglist(cls, flag, varlist, metadata, squash=False):
-        out_dict = dict((var, metadata.getVarFlag(var, flag, True))
+        out_dict = dict((var, metadata.getVarFlag(var, flag))
                     for var in varlist)
         if squash:
             return dict((k,v) for (k,v) in out_dict.items() if v)
@@ -296,7 +296,7 @@ def parse_recipe(bb_data, bbfile, appends, mc=''):
     bb_data.setVar("__BBMULTICONFIG", mc)
 
     # expand tmpdir to include this topdir
-    bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR', True) or "")
+    bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR') or "")
     bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
     oldpath = os.path.abspath(os.getcwd())
     bb.parse.cached_mtime_noerror(bbfile_loc)
@@ -378,7 +378,7 @@ class Cache(NoCache):
         # It will be used later for deciding whether we 
         # need extra cache file dump/load support 
         self.caches_array = caches_array
-        self.cachedir = data.getVar("CACHE", True)
+        self.cachedir = data.getVar("CACHE")
         self.clean = set()
         self.checked = set()
         self.depends_cache = {}
@@ -792,8 +792,8 @@ class MultiProcessCache(object):
         self.cachedata_extras = self.create_cachedata()
 
     def init_cache(self, d, cache_file_name=None):
-        cachedir = (d.getVar("PERSISTENT_DIR", True) or
-                    d.getVar("CACHE", True))
+        cachedir = (d.getVar("PERSISTENT_DIR") or
+                    d.getVar("CACHE"))
         if cachedir in [None, '']:
             return
         bb.utils.mkdirhier(cachedir)

bitbake/lib/bb/codeparser.py

@@ -342,8 +342,7 @@ class ShellParser():
         except pyshlex.NeedMore:
             raise sherrors.ShellSyntaxError("Unexpected EOF")
 
-        for token in tokens:
-            self.process_tokens(token)
+        self.process_tokens(tokens)
 
     def process_tokens(self, tokens):
         """Process a supplied portion of the syntax tree as returned by
@@ -389,18 +388,24 @@ class ShellParser():
             "case_clause": case_clause,
         }
 
-        for token in tokens:
-            name, value = token
-            try:
-                more_tokens, words = token_handlers[name](value)
-            except KeyError:
-                raise NotImplementedError("Unsupported token type " + name)
+        def process_token_list(tokens):
+            for token in tokens:
+                if isinstance(token, list):
+                    process_token_list(token)
+                    continue
+                name, value = token
+                try:
+                    more_tokens, words = token_handlers[name](value)
+                except KeyError:
+                    raise NotImplementedError("Unsupported token type " + name)
 
-            if more_tokens:
-                self.process_tokens(more_tokens)
+                if more_tokens:
+                    self.process_tokens(more_tokens)
 
-            if words:
-                self.process_words(words)
+                if words:
+                    self.process_words(words)
+
+        process_token_list(tokens)
 
     def process_words(self, words):
         """Process a set of 'words' in pyshyacc parlance, which includes

bitbake/lib/bb/command.py

@@ -28,8 +28,15 @@ and must not trigger events, directly or indirectly.
 Commands are queued in a CommandQueue
 """
 
+from collections import OrderedDict, defaultdict
+
 import bb.event
 import bb.cooker
+import bb.remotedata
+
+class DataStoreConnectionHandle(object):
+    def __init__(self, dsindex=0):
+        self.dsindex = dsindex
 
 class CommandCompleted(bb.event.Event):
     pass
@@ -55,6 +62,7 @@ class Command:
         self.cooker = cooker
         self.cmds_sync = CommandsSync()
         self.cmds_async = CommandsAsync()
+        self.remotedatastores = bb.remotedata.RemoteDatastores(cooker)
 
         # FIXME Add lock for this
         self.currentAsyncCommand = None
@@ -125,11 +133,11 @@ class Command:
 
     def finishAsyncCommand(self, msg=None, code=None):
         if msg or msg == "":
-            bb.event.fire(CommandFailed(msg), self.cooker.expanded_data)
+            bb.event.fire(CommandFailed(msg), self.cooker.data)
         elif code:
-            bb.event.fire(CommandExit(code), self.cooker.expanded_data)
+            bb.event.fire(CommandExit(code), self.cooker.data)
         else:
-            bb.event.fire(CommandCompleted(), self.cooker.expanded_data)
+            bb.event.fire(CommandCompleted(), self.cooker.data)
         self.currentAsyncCommand = None
         self.cooker.finishcommand()
 
@@ -179,6 +187,7 @@ class CommandsSync:
         """
         varname = params[0]
         value = str(params[1])
+        command.cooker.extraconfigdata[varname] = value
         command.cooker.data.setVar(varname, value)
 
     def getSetVariable(self, command, params):
@@ -298,6 +307,236 @@ class CommandsSync:
         command.cooker.updateConfigOpts(options, environment)
     updateConfig.needconfig = False
 
+    def parseConfiguration(self, command, params):
+        """Instruct bitbake to parse its configuration
+        NOTE: it is only necessary to call this if you aren't calling any normal action
+        (otherwise parsing is taken care of automatically)
+        """
+        command.cooker.parseConfiguration()
+    parseConfiguration.needconfig = False
+
+    def getLayerPriorities(self, command, params):
+        ret = []
+        # regex objects cannot be marshalled by xmlrpc
+        for collection, pattern, regex, pri in command.cooker.bbfile_config_priorities:
+            ret.append((collection, pattern, regex.pattern, pri))
+        return ret
+    getLayerPriorities.readonly = True
+
+    def getRecipes(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return list(command.cooker.recipecaches[mc].pkg_pn.items())
+    getRecipes.readonly = True
+
+    def getRecipeDepends(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return list(command.cooker.recipecaches[mc].deps.items())
+    getRecipeDepends.readonly = True
+
+    def getRecipeVersions(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].pkg_pepvpr
+    getRecipeVersions.readonly = True
+
+    def getRuntimeDepends(self, command, params):
+        ret = []
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        rundeps = command.cooker.recipecaches[mc].rundeps
+        for key, value in rundeps.items():
+            if isinstance(value, defaultdict):
+                value = dict(value)
+            ret.append((key, value))
+        return ret
+    getRuntimeDepends.readonly = True
+
+    def getRuntimeRecommends(self, command, params):
+        ret = []
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        runrecs = command.cooker.recipecaches[mc].runrecs
+        for key, value in runrecs.items():
+            if isinstance(value, defaultdict):
+                value = dict(value)
+            ret.append((key, value))
+        return ret
+    getRuntimeRecommends.readonly = True
+
+    def getRecipeInherits(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].inherits
+    getRecipeInherits.readonly = True
+
+    def getBbFilePriority(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].bbfile_priority
+    getBbFilePriority.readonly = True
+
+    def getDefaultPreference(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].pkg_dp
+    getDefaultPreference.readonly = True
+
+    def getSkippedRecipes(self, command, params):
+        # Return list sorted by reverse priority order
+        import bb.cache
+        skipdict = OrderedDict(sorted(command.cooker.skiplist.items(),
+                                      key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0])))
+        return list(skipdict.items())
+    getSkippedRecipes.readonly = True
+
+    def getOverlayedRecipes(self, command, params):
+        return list(command.cooker.collection.overlayed.items())
+    getOverlayedRecipes.readonly = True
+
+    def getFileAppends(self, command, params):
+        fn = params[0]
+        return command.cooker.collection.get_file_appends(fn)
+    getFileAppends.readonly = True
+
+    def getAllAppends(self, command, params):
+        return command.cooker.collection.bbappends
+    getAllAppends.readonly = True
+
+    def findProviders(self, command, params):
+        return command.cooker.findProviders()
+    findProviders.readonly = True
+
+    def findBestProvider(self, command, params):
+        pn = params[0]
+        return command.cooker.findBestProvider(pn)
+    findBestProvider.readonly = True
+
+    def allProviders(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return list(bb.providers.allProviders(command.cooker.recipecaches[mc]).items())
+    allProviders.readonly = True
+
+    def getRuntimeProviders(self, command, params):
+        rprovide = params[0]
+        try:
+            mc = params[1]
+        except IndexError:
+            mc = ''
+        all_p = bb.providers.getRuntimeProviders(command.cooker.recipecaches[mc], rprovide)
+        if all_p:
+            best = bb.providers.filterProvidersRunTime(all_p, rprovide,
+                            command.cooker.data,
+                            command.cooker.recipecaches[mc])[0][0]
+        else:
+            best = None
+        return all_p, best
+    getRuntimeProviders.readonly = True
+
+    def dataStoreConnectorFindVar(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        datastore = command.remotedatastores[dsindex]
+        value = datastore._findVar(name)
+
+        if value:
+            content = value.get('_content', None)
+            if isinstance(content, bb.data_smart.DataSmart):
+                # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
+                idx = command.remotedatastores.check_store(content, True)
+                return {'_content': DataStoreConnectionHandle(idx), '_connector_origtype': 'DataStoreConnectionHandle'}
+            elif isinstance(content, set):
+                return {'_content': list(content), '_connector_origtype': 'set'}
+        return value
+    dataStoreConnectorFindVar.readonly = True
+
+    def dataStoreConnectorGetKeys(self, command, params):
+        dsindex = params[0]
+        datastore = command.remotedatastores[dsindex]
+        return list(datastore.keys())
+    dataStoreConnectorGetKeys.readonly = True
+
+    def dataStoreConnectorGetVarHistory(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        datastore = command.remotedatastores[dsindex]
+        return datastore.varhistory.variable(name)
+    dataStoreConnectorGetVarHistory.readonly = True
+
+    def dataStoreConnectorExpandPythonRef(self, command, params):
+        config_data_dict = params[0]
+        varname = params[1]
+        expr = params[2]
+
+        config_data = command.remotedatastores.receive_datastore(config_data_dict)
+
+        varparse = bb.data_smart.VariableParse(varname, config_data)
+        return varparse.python_sub(expr)
+
+    def dataStoreConnectorRelease(self, command, params):
+        dsindex = params[0]
+        if dsindex <= 0:
+            raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex)
+        command.remotedatastores.release(dsindex)
+
+    def parseRecipeFile(self, command, params):
+        """
+        Parse the specified recipe file (with or without bbappends)
+        and return a datastore object representing the environment
+        for the recipe.
+        """
+        fn = params[0]
+        appends = params[1]
+        appendlist = params[2]
+        if len(params) > 3:
+            config_data_dict = params[3]
+            config_data = command.remotedatastores.receive_datastore(config_data_dict)
+        else:
+            config_data = None
+
+        if appends:
+            if appendlist is not None:
+                appendfiles = appendlist
+            else:
+                appendfiles = command.cooker.collection.get_file_appends(fn)
+        else:
+            appendfiles = []
+        # We are calling bb.cache locally here rather than on the server,
+        # but that's OK because it doesn't actually need anything from
+        # the server barring the global datastore (which we have a remote
+        # version of)
+        if config_data:
+            # We have to use a different function here if we're passing in a datastore
+            # NOTE: we took a copy above, so we don't do it here again
+            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)['']
+        else:
+            # Use the standard path
+            parser = bb.cache.NoCache(command.cooker.databuilder)
+            envdata = parser.loadDataFull(fn, appendfiles)
+        idx = command.remotedatastores.store(envdata)
+        return DataStoreConnectionHandle(idx)
+    parseRecipeFile.readonly = True
+
 class CommandsAsync:
     """
     A class of asynchronous commands
@@ -311,8 +550,12 @@ class CommandsAsync:
         """
         bfile = params[0]
         task = params[1]
+        if len(params) > 2:
+            hidewarning = params[2]
+        else:
+            hidewarning = False
 
-        command.cooker.buildFile(bfile, task)
+        command.cooker.buildFile(bfile, task, hidewarning)
     buildFile.needcache = False
 
     def buildTargets(self, command, params):
@@ -472,3 +715,11 @@ class CommandsAsync:
         command.finishAsyncCommand()
     resetCooker.needcache = False
 
+    def clientComplete(self, command, params):
+        """
+        Do the right thing when the controlling client exits
+        """
+        command.cooker.clientComplete()
+        command.finishAsyncCommand()
+    clientComplete.needcache = False
+

bitbake/lib/bb/cooker.py

@@ -252,6 +252,10 @@ class BBCooker:
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
     def config_notifications(self, event):
+        if event.maskname == "IN_Q_OVERFLOW":
+            bb.warn("inotify event queue overflowed, invalidating caches.")
+            self.baseconfig_valid = False
+            return
         if not event.pathname in self.configwatcher.bbwatchedfiles:
             return
         if not event.pathname in self.inotify_modified_files:
@@ -259,6 +263,10 @@ class BBCooker:
         self.baseconfig_valid = False
 
     def notifications(self, event):
+        if event.maskname == "IN_Q_OVERFLOW":
+            bb.warn("inotify event queue overflowed, invalidating caches.")
+            self.parsecache_valid = False
+            return
         if not event.pathname in self.inotify_modified_files:
             self.inotify_modified_files.append(event.pathname)
         self.parsecache_valid = False
@@ -323,7 +331,7 @@ class BBCooker:
         # Need to preserve BB_CONSOLELOG over resets
         consolelog = None
         if hasattr(self, "data"):
-            consolelog = self.data.getVar("BB_CONSOLELOG", True)
+            consolelog = self.data.getVar("BB_CONSOLELOG")
 
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.enableDataTracking()
@@ -350,6 +358,7 @@ class BBCooker:
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
         self.data_hash = self.databuilder.data_hash
+        self.extraconfigdata = {}
 
         if consolelog:
             self.data.setVar("BB_CONSOLELOG", consolelog)
@@ -358,9 +367,7 @@ class BBCooker:
         # Copy of the data store which has been expanded.
         # Used for firing events and accessing variables where expansion needs to be accounted for
         #
-        self.expanded_data = bb.data.createCopy(self.data)
-        bb.data.update_data(self.expanded_data)
-        bb.parse.init_parser(self.expanded_data)
+        bb.parse.init_parser(self.data)
 
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.disableDataTracking()
@@ -518,7 +525,7 @@ class BBCooker:
             bb.msg.loggerVerboseLogs = True
 
         # Change nice level if we're asked to
-        nice = self.data.getVar("BB_NICE_LEVEL", True)
+        nice = self.data.getVar("BB_NICE_LEVEL")
         if nice:
             curnice = os.nice(0)
             nice = int(nice) - curnice
@@ -531,7 +538,7 @@ class BBCooker:
         for mc in self.multiconfigs:
             self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
 
-        self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS", True))
+        self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
 
     def updateConfigOpts(self, options, environment):
         clean = True
@@ -575,13 +582,12 @@ class BBCooker:
 
     def showVersions(self):
 
-        pkg_pn = self.recipecaches[''].pkg_pn
-        (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecaches[''], pkg_pn)
+        (latest_versions, preferred_versions) = self.findProviders()
 
         logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
         logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
 
-        for p in sorted(pkg_pn):
+        for p in sorted(self.recipecaches[''].pkg_pn):
             pref = preferred_versions[p]
             latest = latest_versions[p]
 
@@ -611,7 +617,7 @@ class BBCooker:
             fn = self.matchFile(fn)
             fn = bb.cache.realfn2virtual(fn, cls, mc)
         elif len(pkgs_to_build) == 1:
-            ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or ""
+            ignore = self.data.getVar("ASSUME_PROVIDED") or ""
             if pkgs_to_build[0] in set(ignore.split()):
                 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
@@ -657,6 +663,8 @@ class BBCooker:
         # A task of None means use the default task
         if task is None:
             task = self.configuration.cmd
+        if not task.startswith("do_"):
+            task = "do_%s" % task
 
         fulltargetlist = self.checkPackages(pkgs_to_build, task)
         taskdata = {}
@@ -715,6 +723,9 @@ class BBCooker:
         Create a dependency graph of pkgs_to_build including reverse dependency
         information.
         """
+        if not task.startswith("do_"):
+            task = "do_%s" % task
+
         runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
         rq.rqdata.prepare()
@@ -818,6 +829,9 @@ class BBCooker:
         """
         Create a dependency tree of pkgs_to_build, returning the data.
         """
+        if not task.startswith("do_"):
+            task = "do_%s" % task
+
         _, taskdata = self.prepareTreeData(pkgs_to_build, task)
 
         seen_fns = []
@@ -995,7 +1009,7 @@ class BBCooker:
             bb.data.expandKeys(localdata)
 
             # Handle PREFERRED_PROVIDERS
-            for p in (localdata.getVar('PREFERRED_PROVIDERS', True) or "").split():
+            for p in (localdata.getVar('PREFERRED_PROVIDERS') or "").split():
                 try:
                     (providee, provider) = p.split(':')
                 except:
@@ -1006,7 +1020,7 @@ class BBCooker:
                 self.recipecaches[mc].preferred[providee] = provider
 
     def findCoreBaseFiles(self, subdir, configfile):
-        corebase = self.data.getVar('COREBASE', True) or ""
+        corebase = self.data.getVar('COREBASE') or ""
         paths = []
         for root, dirs, files in os.walk(corebase + '/' + subdir):
             for d in dirs:
@@ -1056,7 +1070,7 @@ class BBCooker:
         """
 
         matches = []
-        bbpaths = self.data.getVar('BBPATH', True).split(':')
+        bbpaths = self.data.getVar('BBPATH').split(':')
         for path in bbpaths:
             dirpath = os.path.join(path, directory)
             if os.path.exists(dirpath):
@@ -1068,6 +1082,20 @@ class BBCooker:
         if matches:
             bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
 
+    def findProviders(self, mc=''):
+        return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+
+    def findBestProvider(self, pn, mc=''):
+        if pn in self.recipecaches[mc].providers:
+            filenames = self.recipecaches[mc].providers[pn]
+            eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.data, self.recipecaches[mc])
+            filename = eligible[0]
+            return None, None, None, filename
+        elif pn in self.recipecaches[mc].pkg_pn:
+            return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+        else:
+            return None, None, None, None
+
     def findConfigFiles(self, varname):
         """
         Find config files which are appropriate values for varname.
@@ -1078,7 +1106,7 @@ class BBCooker:
 
         data = self.data
         # iterate configs
-        bbpaths = data.getVar('BBPATH', True).split(':')
+        bbpaths = data.getVar('BBPATH').split(':')
         for path in bbpaths:
             confpath = os.path.join(path, "conf", var)
             if os.path.exists(confpath):
@@ -1147,7 +1175,7 @@ class BBCooker:
                 bb.debug(1,'Processing %s in collection list' % (c))
 
                 # Get collection priority if defined explicitly
-                priority = self.data.getVar("BBFILE_PRIORITY_%s" % c, True)
+                priority = self.data.getVar("BBFILE_PRIORITY_%s" % c)
                 if priority:
                     try:
                         prio = int(priority)
@@ -1161,7 +1189,7 @@ class BBCooker:
                     collection_priorities[c] = None
 
                 # Check dependencies and store information for priority calculation
-                deps = self.data.getVar("LAYERDEPENDS_%s" % c, True)
+                deps = self.data.getVar("LAYERDEPENDS_%s" % c)
                 if deps:
                     try:
                         depDict = bb.utils.explode_dep_versions2(deps)
@@ -1170,7 +1198,7 @@ class BBCooker:
                     for dep, oplist in list(depDict.items()):
                         if dep in collection_list:
                             for opstr in oplist:
-                                layerver = self.data.getVar("LAYERVERSION_%s" % dep, True)
+                                layerver = self.data.getVar("LAYERVERSION_%s" % dep)
                                 (op, depver) = opstr.split()
                                 if layerver:
                                     try:
@@ -1191,7 +1219,7 @@ class BBCooker:
                     collection_depends[c] = []
 
                 # Check recommends and store information for priority calculation
-                recs = self.data.getVar("LAYERRECOMMENDS_%s" % c, True)
+                recs = self.data.getVar("LAYERRECOMMENDS_%s" % c)
                 if recs:
                     try:
                         recDict = bb.utils.explode_dep_versions2(recs)
@@ -1201,7 +1229,7 @@ class BBCooker:
                         if rec in collection_list:
                             if oplist:
                                 opstr = oplist[0]
-                                layerver = self.data.getVar("LAYERVERSION_%s" % rec, True)
+                                layerver = self.data.getVar("LAYERVERSION_%s" % rec)
                                 if layerver:
                                     (op, recver) = opstr.split()
                                     try:
@@ -1235,11 +1263,15 @@ class BBCooker:
             # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
             for c in collection_list:
                 calc_layer_priority(c)
-                regex = self.data.getVar("BBFILE_PATTERN_%s" % c, True)
+                regex = self.data.getVar("BBFILE_PATTERN_%s" % c)
                 if regex == None:
                     parselog.error("BBFILE_PATTERN_%s not defined" % c)
                     errors = True
                     continue
+                elif regex == "":
+                    parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c)
+                    errors = False
+                    continue
                 try:
                     cre = re.compile(regex)
                 except re.error:
@@ -1270,7 +1302,7 @@ class BBCooker:
             bf = os.path.abspath(bf)
 
         self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-        filelist, masked = self.collection.collect_bbfiles(self.data, self.expanded_data)
+        filelist, masked = self.collection.collect_bbfiles(self.data, self.data)
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1301,15 +1333,16 @@ class BBCooker:
             raise NoSpecificMatch
         return matches[0]
 
-    def buildFile(self, buildfile, task):
+    def buildFile(self, buildfile, task, hidewarning=False):
         """
         Build the file matching regexp buildfile
         """
-        bb.event.fire(bb.event.BuildInit(), self.expanded_data)
+        bb.event.fire(bb.event.BuildInit(), self.data)
 
-        # Too many people use -b because they think it's how you normally
-        # specify a target to be built, so show a warning
-        bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
+        if not hidewarning:
+            # Too many people use -b because they think it's how you normally
+            # specify a target to be built, so show a warning
+            bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
 
         # Parse the configuration here. We need to do it explicitly here since
         # buildFile() doesn't use the cache
@@ -1318,6 +1351,8 @@ class BBCooker:
         # If we are told to do the None task then query the default task
         if (task == None):
             task = self.configuration.cmd
+        if not task.startswith("do_"):
+            task = "do_%s" % task
 
         fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
         fn = self.matchFile(fn)
@@ -1354,8 +1389,6 @@ class BBCooker:
         # Invalidate task for target if force mode active
         if self.configuration.force:
             logger.verbose("Invalidate task %s, %s", task, fn)
-            if not task.startswith("do_"):
-                task = "do_%s" % task
             bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn)
 
         # Setup taskdata structure
@@ -1363,12 +1396,10 @@ class BBCooker:
         taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort)
         taskdata[mc].add_provider(self.data, self.recipecaches[mc], item)
 
-        buildname = self.data.getVar("BUILDNAME", True)
-        bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.expanded_data)
+        buildname = self.data.getVar("BUILDNAME")
+        bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.data)
 
         # Execute the runqueue
-        if not task.startswith("do_"):
-            task = "do_%s" % task
         runlist = [[mc, item, task, fn]]
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
@@ -1396,7 +1427,7 @@ class BBCooker:
                 return False
 
             if not retval:
-                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.expanded_data)
+                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.data)
                 self.command.finishAsyncCommand(msg)
                 return False
             if retval is True:
@@ -1451,7 +1482,7 @@ class BBCooker:
 
         packages = [target if ':' in target else '%s:%s' % (target, task) for target in targets]
 
-        bb.event.fire(bb.event.BuildInit(packages), self.expanded_data)
+        bb.event.fire(bb.event.BuildInit(packages), self.data)
 
         taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort)
 
@@ -1484,7 +1515,7 @@ class BBCooker:
                 v = self.data.getVar(k, expand)
                 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
                     dump[k] = {
-    'v' : v ,
+    'v' : str(v) ,
     'history' : self.data.varhistory.variable(k),
                     }
                     for d in flaglist:
@@ -1582,14 +1613,14 @@ class BBCooker:
                 bb.event.fire(bb.event.SanityCheck(False), self.data)
 
             for mc in self.multiconfigs:
-                ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED", True) or ""
+                ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED") or ""
                 self.recipecaches[mc].ignored_dependencies = set(ignore.split())
 
                 for dep in self.configuration.extra_assume_provided:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
             self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-            (filelist, masked) = self.collection.collect_bbfiles(self.data, self.expanded_data)
+            (filelist, masked) = self.collection.collect_bbfiles(self.data, self.data)
 
             self.parser = CookerParser(self, filelist, masked)
             self.parsecache_valid = True
@@ -1623,7 +1654,7 @@ class BBCooker:
         if len(pkgs_to_build) == 0:
             raise NothingToBuild
 
-        ignore = (self.expanded_data.getVar("ASSUME_PROVIDED", True) or "").split()
+        ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
         for pkg in pkgs_to_build:
             if pkg in ignore:
                 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
@@ -1656,13 +1687,13 @@ class BBCooker:
         try:
             self.prhost = prserv.serv.auto_start(self.data)
         except prserv.serv.PRServiceConfigError:
-            bb.event.fire(CookerExit(), self.expanded_data)
+            bb.event.fire(CookerExit(), self.data)
             self.state = state.error
         return
 
     def post_serve(self):
         prserv.serv.auto_shutdown(self.data)
-        bb.event.fire(CookerExit(), self.expanded_data)
+        bb.event.fire(CookerExit(), self.data)
         lockfile = self.lock.name
         self.lock.close()
         self.lock = None
@@ -1707,6 +1738,13 @@ class BBCooker:
     def reset(self):
         self.initConfigurationData()
 
+    def clientComplete(self):
+        """Called when the client is done using the server"""
+        if self.configuration.server_only:
+            self.finishcommand()
+        else:
+            self.shutdown(True)
+
     def lockBitbake(self):
         if not hasattr(self, 'lock'):
             self.lock = None
@@ -1793,7 +1831,7 @@ class CookerCollectFiles(object):
 
         collectlog.debug(1, "collecting .bb files")
 
-        files = (config.getVar( "BBFILES", True) or "").split()
+        files = (config.getVar( "BBFILES") or "").split()
         config.setVar("BBFILES", " ".join(files))
 
         # Sort files by priority
@@ -1823,7 +1861,7 @@ class CookerCollectFiles(object):
                     if g not in newfiles:
                         newfiles.append(g)
 
-        bbmask = config.getVar('BBMASK', True)
+        bbmask = config.getVar('BBMASK')
 
         if bbmask:
             # First validate the individual regular expressions and ignore any
@@ -1919,7 +1957,7 @@ class CookerCollectFiles(object):
 
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
             if regex in unmatched:
-                if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection, True) != '1':
+                if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
                     collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
 
         return priorities
@@ -2076,7 +2114,7 @@ class CookerParser(object):
         self.toparse = self.total - len(self.fromcache)
         self.progress_chunk = int(max(self.toparse / 100, 1))
 
-        self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
+        self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
                                  multiprocessing.cpu_count()), len(self.willparse))
 
         self.start()

bitbake/lib/bb/cookerdata.py

@@ -146,6 +146,7 @@ class CookerConfiguration(object):
         self.tracking = False
         self.interface = []
         self.writeeventlog = False
+        self.server_only = False
 
         self.env = {}
 
@@ -212,7 +213,7 @@ def _inherit(bbclass, data):
 
 def findConfigFile(configfile, data):
     search = []
-    bbpath = data.getVar("BBPATH", True)
+    bbpath = data.getVar("BBPATH")
     if bbpath:
         for i in bbpath.split(":"):
             search.append(os.path.join(i, "conf", configfile))
@@ -286,7 +287,7 @@ class CookerDataBuilder(object):
             self.data_hash = self.data.get_hash()
             self.mcdata[''] = self.data
 
-            multiconfig = (self.data.getVar("BBMULTICONFIG", True) or "").split()
+            multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
             for config in multiconfig:
                 mcdata = self.parseConfigurationFiles(['conf/multiconfig/%s.conf' % config] + self.prefiles, self.postfiles)
                 bb.event.fire(bb.event.ConfigParsed(), mcdata)
@@ -319,7 +320,7 @@ class CookerDataBuilder(object):
             data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
             data = parse_config_file(layerconf, data)
 
-            layers = (data.getVar('BBLAYERS', True) or "").split()
+            layers = (data.getVar('BBLAYERS') or "").split()
 
             data = bb.data.createCopy(data)
             approved = bb.utils.approved_variables()
@@ -342,7 +343,7 @@ class CookerDataBuilder(object):
             data.delVar('LAYERDIR_RE')
             data.delVar('LAYERDIR')
 
-        if not data.getVar("BBPATH", True):
+        if not data.getVar("BBPATH"):
             msg = "The BBPATH variable is not set"
             if not layerconf:
                 msg += (" and bitbake did not find a conf/bblayers.conf file in"
@@ -357,7 +358,7 @@ class CookerDataBuilder(object):
             data = parse_config_file(p, data)
 
         # Handle any INHERITs and inherit the base class
-        bbclasses  = ["base"] + (data.getVar('INHERIT', True) or "").split()
+        bbclasses  = ["base"] + (data.getVar('INHERIT') or "").split()
         for bbclass in bbclasses:
             data = _inherit(bbclass, data)
 
@@ -369,7 +370,7 @@ class CookerDataBuilder(object):
                 parselog.critical("Undefined event handler function '%s'" % var)
                 sys.exit(1)
             handlerln = int(data.getVarFlag(var, "lineno", False))
-            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
+            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
 
         data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
 

bitbake/lib/bb/data.py

@@ -78,59 +78,6 @@ def initVar(var, d):
     """Non-destructive var init for data structure"""
     d.initVar(var)
 
-
-def setVar(var, value, d):
-    """Set a variable to a given value"""
-    d.setVar(var, value)
-
-
-def getVar(var, d, exp = False):
-    """Gets the value of a variable"""
-    return d.getVar(var, exp)
-
-
-def renameVar(key, newkey, d):
-    """Renames a variable from key to newkey"""
-    d.renameVar(key, newkey)
-
-def delVar(var, d):
-    """Removes a variable from the data set"""
-    d.delVar(var)
-
-def appendVar(var, value, d):
-    """Append additional value to a variable"""
-    d.appendVar(var, value)
-
-def setVarFlag(var, flag, flagvalue, d):
-    """Set a flag for a given variable to a given value"""
-    d.setVarFlag(var, flag, flagvalue)
-
-def getVarFlag(var, flag, d):
-    """Gets given flag from given var"""
-    return d.getVarFlag(var, flag, False)
-
-def delVarFlag(var, flag, d):
-    """Removes a given flag from the variable's flags"""
-    d.delVarFlag(var, flag)
-
-def setVarFlags(var, flags, d):
-    """Set the flags for a given variable
-
-    Note:
-        setVarFlags will not clear previous
-        flags. Think of this method as
-        addVarFlags
-    """
-    d.setVarFlags(var, flags)
-
-def getVarFlags(var, d):
-    """Gets a variable's flags"""
-    return d.getVarFlags(var)
-
-def delVarFlags(var, d):
-    """Removes a variable's flags"""
-    d.delVarFlags(var)
-
 def keys(d):
     """Return a list of keys in d"""
     return d.keys()
@@ -174,7 +121,7 @@ def inheritFromOS(d, savedenv, permitted):
     for s in savedenv.keys():
         if s in permitted:
             try:
-                d.setVar(s, savedenv.getVar(s, True), op = 'from env')
+                d.setVar(s, savedenv.getVar(s), op = 'from env')
                 if s in exportlist:
                     d.setVarFlag(s, "export", True, op = 'auto env export')
             except TypeError:
@@ -194,7 +141,7 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
     try:
         if all:
             oval = d.getVar(var, False)
-        val = d.getVar(var, True)
+        val = d.getVar(var)
     except (KeyboardInterrupt, bb.build.FuncFailed):
         raise
     except Exception as exc:
@@ -258,11 +205,13 @@ def exported_keys(d):
                                       not d.getVarFlag(key, 'unexport', False))
 
 def exported_vars(d):
-    for key in exported_keys(d):
+    k = list(exported_keys(d))
+    for key in k:
         try:
-            value = d.getVar(key, True)
-        except Exception:
-            pass
+            value = d.getVar(key)
+        except Exception as err:
+            bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE"), key, err))
+            continue
 
         if value is not None:
             yield key, str(value)
@@ -276,8 +225,8 @@ def emit_func(func, o=sys.__stdout__, d = init()):
 
     o.write('\n')
     emit_var(func, o, d, False) and o.write('\n')
-    newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
-    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
+    newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
     seen = set()
     while newdeps:
         deps = newdeps
@@ -286,8 +235,8 @@ def emit_func(func, o=sys.__stdout__, d = init()):
         for dep in deps:
             if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False):
                emit_var(dep, o, d, False) and o.write('\n')
-               newdeps |=  bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep, True))
-               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+               newdeps |=  bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep))
+               newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
         newdeps -= seen
 
 _functionfmt = """
@@ -310,7 +259,7 @@ def emit_func_python(func, o=sys.__stdout__, d = init()):
     pp = bb.codeparser.PythonParser(func, logger)
     pp.parse_python(d.getVar(func, False))
     newdeps = pp.execs
-    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
     seen = set()
     while newdeps:
         deps = newdeps
@@ -322,7 +271,7 @@ def emit_func_python(func, o=sys.__stdout__, d = init()):
                pp = bb.codeparser.PythonParser(dep, logger)
                pp.parse_python(d.getVar(dep, False))
                newdeps |= pp.execs
-               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+               newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
         newdeps -= seen
 
 def update_data(d):
@@ -346,7 +295,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
         def handle_contains(value, contains, d):
             newvalue = ""
             for k in sorted(contains):
-                l = (d.getVar(k, True) or "").split()
+                l = (d.getVar(k) or "").split()
                 for word in sorted(contains[k]):
                     if word in l:
                         newvalue += "\n%s{%s} = Set" %  (k, word)
@@ -364,7 +313,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
             if varflags.get("python"):
                 parser = bb.codeparser.PythonParser(key, logger)
                 if value and "\t" in value:
-                    logger.warning("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True)))
+                    logger.warning("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE")))
                 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
                 deps = deps | parser.references
                 deps = deps | (keys & parser.execs)
@@ -419,7 +368,7 @@ def generate_dependencies(d):
 
     keys = set(key for key in d if not key.startswith("__"))
     shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
-    varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True)
+    varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')
 
     deps = {}
     values = {}

bitbake/lib/bb/data_smart.py

@@ -108,7 +108,7 @@ class VariableParse:
                 varparse = self.d.expand_cache[key]
                 var = varparse.value
             else:
-                var = self.d.getVarFlag(key, "_content", True)
+                var = self.d.getVarFlag(key, "_content")
             self.references.add(key)
             if var is not None:
                 return var
@@ -116,13 +116,21 @@ class VariableParse:
                 return match.group()
 
     def python_sub(self, match):
-            code = match.group()[3:-1]
+            if isinstance(match, str):
+                code = match
+            else:
+                code = match.group()[3:-1]
+
+            if "_remote_data" in self.d:
+                connector = self.d["_remote_data"]
+                return connector.expandPythonRef(self.varname, code, self.d)
+
             codeobj = compile(code.strip(), self.varname or "<expansion>", "eval")
 
             parser = bb.codeparser.PythonParser(self.varname, logger)
             parser.parse_python(code)
             if self.varname:
-                vardeps = self.d.getVarFlag(self.varname, "vardeps", True)
+                vardeps = self.d.getVarFlag(self.varname, "vardeps")
                 if vardeps is None:
                     parser.log.flush()
             else:
@@ -146,7 +154,7 @@ class DataContext(dict):
         self['d'] = metadata
 
     def __missing__(self, key):
-        value = self.metadata.getVar(key, True)
+        value = self.metadata.getVar(key)
         if value is None or self.metadata.getVarFlag(key, 'func', False):
             raise KeyError(key)
         else:
@@ -222,6 +230,19 @@ class VariableHistory(object):
         new.variables = self.variables.copy()
         return new
 
+    def __getstate__(self):
+        vardict = {}
+        for k, v in self.variables.iteritems():
+            vardict[k] = v
+        return {'dataroot': self.dataroot,
+                'variables': vardict}
+
+    def __setstate__(self, state):
+        self.dataroot = state['dataroot']
+        self.variables = COWDictBase.copy()
+        for k, v in state['variables'].items():
+            self.variables[k] = v
+
     def record(self, *kwonly, **loginfo):
         if not self.dataroot._tracking:
             return
@@ -247,10 +268,15 @@ class VariableHistory(object):
         self.variables[var].append(loginfo.copy())
 
     def variable(self, var):
-        if var in self.variables:
-            return self.variables[var]
+        remote_connector = self.dataroot.getVar('_remote_data', False)
+        if remote_connector:
+            varhistory = remote_connector.getVarHistory(var)
         else:
-            return []
+            varhistory = []
+
+        if var in self.variables:
+            varhistory.extend(self.variables[var])
+        return varhistory
 
     def emit(self, var, oval, val, o, d):
         history = self.variable(var)
@@ -318,7 +344,7 @@ class VariableHistory(object):
         the files in which they were added.
         """
         history = self.variable(var)
-        finalitems = (d.getVar(var, True) or '').split()
+        finalitems = (d.getVar(var) or '').split()
         filemap = {}
         isset = False
         for event in history:
@@ -426,11 +452,11 @@ class DataSmart(MutableMapping):
             # Can end up here recursively so setup dummy values
             self.overrides = []
             self.overridesset = set()
-            self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+            self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
             self.overridesset = set(self.overrides)
             self.inoverride = False
             self.expand_cache = {}
-            newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+            newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
             if newoverrides == self.overrides:
                 break
             self.overrides = newoverrides
@@ -449,6 +475,10 @@ class DataSmart(MutableMapping):
             if var in dest:
                 return dest[var]
 
+            if "_remote_data" in dest:
+                connector = dest["_remote_data"]["_content"]
+                return connector.getVar(var)
+
             if "_data" not in dest:
                 break
             dest = dest["_data"]
@@ -471,6 +501,12 @@ class DataSmart(MutableMapping):
         if 'parsing' in loginfo:
             parsing=True
 
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.setVar(var, value)
+            if not res:
+                return
+
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         self.expand_cache = {}
@@ -541,7 +577,7 @@ class DataSmart(MutableMapping):
             nextnew = set()
             self.overridevars.update(new)
             for i in new:
-                vardata = self.expandWithRefs(self.getVar(i, True), i)
+                vardata = self.expandWithRefs(self.getVar(i), i)
                 nextnew.update(vardata.references)
                 nextnew.update(vardata.contains.keys())
             new = nextnew
@@ -565,7 +601,7 @@ class DataSmart(MutableMapping):
                 if len(shortvar) == 0:
                     override = None
 
-    def getVar(self, var, expand, noweakdefault=False, parsing=False):
+    def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
         return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
 
     def renameVar(self, key, newkey, **loginfo):
@@ -662,7 +698,7 @@ class DataSmart(MutableMapping):
                 self.dict["__exportlist"]["_content"] = set()
             self.dict["__exportlist"]["_content"].add(var)
 
-    def getVarFlag(self, var, flag, expand, noweakdefault=False, parsing=False):
+    def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False):
         local_var = self._findVar(var)
         value = None
         if flag == "_content" and var in self.overridedata and not parsing:
@@ -844,7 +880,7 @@ class DataSmart(MutableMapping):
         data = DataSmart()
         data.dict["_data"] = self.dict
         data.varhistory = self.varhistory.copy()
-        data.varhistory.datasmart = data
+        data.varhistory.dataroot = data
         data.inchistory = self.inchistory.copy()
 
         data._tracking = self._tracking
@@ -875,7 +911,7 @@ class DataSmart(MutableMapping):
 
     def localkeys(self):
         for key in self.dict:
-            if key != '_data':
+            if key not in ['_data', '_remote_data']:
                 yield key
 
     def __iter__(self):
@@ -884,7 +920,7 @@ class DataSmart(MutableMapping):
         def keylist(d):        
             klist = set()
             for key in d:
-                if key == "_data":
+                if key in ["_data", "_remote_data"]:
                     continue
                 if key in deleted:
                     continue
@@ -898,6 +934,10 @@ class DataSmart(MutableMapping):
             if "_data" in d:
                 klist |= keylist(d["_data"])
 
+            if "_remote_data" in d:
+                connector = d["_remote_data"]["_content"]
+                klist |= connector.getKeys()
+
             return klist
 
         self.need_overrides()
@@ -937,7 +977,7 @@ class DataSmart(MutableMapping):
         bb.data.expandKeys(d)
         bb.data.update_data(d)
 
-        config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split())
+        config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
         keys = set(key for key in iter(d) if not key.startswith("__"))
         for key in keys:
             if key in config_whitelist:
@@ -956,7 +996,6 @@ class DataSmart(MutableMapping):
 
         for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
             bb_list = d.getVar(key, False) or []
-            bb_list.sort()
             data.update({key:str(bb_list)})
 
             if key == "__BBANONFUNCS":

bitbake/lib/bb/event.py

@@ -48,6 +48,16 @@ class Event(object):
     def __init__(self):
         self.pid = worker_pid
 
+
+class HeartbeatEvent(Event):
+    """Triggered at regular time intervals of 10 seconds. Other events can fire much more often
+       (runQueueTaskStarted when there are many short tasks) or not at all for long periods
+       of time (again runQueueTaskStarted, when there is just one long-running task), so this
+       event is more suitable for doing some task-independent work occassionally."""
+    def __init__(self, time):
+        Event.__init__(self)
+        self.time = time
+
 Registered        = 10
 AlreadyRegistered = 14
 
@@ -441,6 +451,23 @@ class DiskFull(Event):
         self._free = freespace
         self._mountpoint = mountpoint
 
+class DiskUsageSample:
+    def __init__(self, available_bytes, free_bytes, total_bytes):
+        # Number of bytes available to non-root processes.
+        self.available_bytes = available_bytes
+        # Number of bytes available to root processes.
+        self.free_bytes = free_bytes
+        # Total capacity of the volume.
+        self.total_bytes = total_bytes
+
+class MonitorDiskEvent(Event):
+    """If BB_DISKMON_DIRS is set, then this event gets triggered each time disk space is checked.
+       Provides information about devices that are getting monitored."""
+    def __init__(self, disk_usage):
+        Event.__init__(self)
+        # hash of device root path -> DiskUsageSample
+        self.disk_usage = disk_usage
+
 class NoProvider(Event):
     """No Provider for an Event"""
 

bitbake/lib/bb/fetch2/__init__.py

@@ -491,7 +491,7 @@ def fetcher_init(d):
     Calls before this must not hit the cache.
     """
     # When to drop SCM head revisions controlled by user policy
-    srcrev_policy = d.getVar('BB_SRCREV_POLICY', True) or "clear"
+    srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
     if srcrev_policy == "cache":
         logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
     elif srcrev_policy == "clear":
@@ -572,7 +572,7 @@ def verify_checksum(ud, d, precomputed={}):
 
     if ud.method.recommends_checksum(ud) and not ud.md5_expected and not ud.sha256_expected:
         # If strict checking enabled and neither sum defined, raise error
-        strict = d.getVar("BB_STRICT_CHECKSUM", True) or "0"
+        strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
         if strict == "1":
             logger.error('No checksum specified for %s, please add at least one to the recipe:\n'
                              'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
@@ -718,7 +718,7 @@ def subprocess_setup():
 
 def get_autorev(d):
     #  only not cache src rev in autorev case
-    if d.getVar('BB_SRCREV_POLICY', True) != "cache":
+    if d.getVar('BB_SRCREV_POLICY') != "cache":
         d.setVar('BB_DONT_CACHE', '1')
     return "AUTOINC"
 
@@ -737,7 +737,7 @@ def get_srcrev(d, method_name='sortable_revision'):
     """
 
     scms = []
-    fetcher = Fetch(d.getVar('SRC_URI', True).split(), d)
+    fetcher = Fetch(d.getVar('SRC_URI').split(), d)
     urldata = fetcher.ud
     for u in urldata:
         if urldata[u].method.supports_srcrev():
@@ -757,7 +757,7 @@ def get_srcrev(d, method_name='sortable_revision'):
     #
     # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
     #
-    format = d.getVar('SRCREV_FORMAT', True)
+    format = d.getVar('SRCREV_FORMAT')
     if not format:
         raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
 
@@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
 
     origenv = d.getVar("BB_ORIGENV", False)
     for var in exportvars:
-        val = d.getVar(var, True) or (origenv and origenv.getVar(var, True))
+        val = d.getVar(var) or (origenv and origenv.getVar(var))
         if val:
             cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
 
@@ -856,12 +856,15 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
 
     return output
 
-def check_network_access(d, info = "", url = None):
+def check_network_access(d, info, url):
     """
-    log remote network access, and error if BB_NO_NETWORK is set
+    log remote network access, and error if BB_NO_NETWORK is set or the given
+    URI is untrusted
     """
-    if d.getVar("BB_NO_NETWORK", True) == "1":
+    if d.getVar("BB_NO_NETWORK") == "1":
         raise NetworkAccess(url, info)
+    elif not trusted_network(d, url):
+        raise UntrustedUrl(url, info)
     else:
         logger.debug(1, "Fetcher accessed the network with the command %s" % info)
 
@@ -958,7 +961,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
 
         # We may be obtaining a mirror tarball which needs further processing by the real fetcher
         # If that tarball is a local file:// we need to provide a symlink to it
-        dldir = ld.getVar("DL_DIR", True)
+        dldir = ld.getVar("DL_DIR")
         if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
                 and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
             # Create donestamp in old format to avoid triggering a re-download
@@ -1032,14 +1035,14 @@ def trusted_network(d, url):
     BB_ALLOWED_NETWORKS is set globally or for a specific recipe.
     Note: modifies SRC_URI & mirrors.
     """
-    if d.getVar('BB_NO_NETWORK', True) == "1":
+    if d.getVar('BB_NO_NETWORK') == "1":
         return True
 
     pkgname = d.expand(d.getVar('PN', False))
     trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
 
     if not trusted_hosts:
-        trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS', True)
+        trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS')
 
     # Not enabled.
     if not trusted_hosts:
@@ -1071,7 +1074,7 @@ def srcrev_internal_helper(ud, d, name):
     """
 
     srcrev = None
-    pn = d.getVar("PN", True)
+    pn = d.getVar("PN")
     attempts = []
     if name != '' and pn:
         attempts.append("SRCREV_%s_pn-%s" % (name, pn))
@@ -1082,7 +1085,7 @@ def srcrev_internal_helper(ud, d, name):
     attempts.append("SRCREV")
 
     for a in attempts:
-        srcrev = d.getVar(a, True)              
+        srcrev = d.getVar(a)              
         if srcrev and srcrev != "INVALID":
             break
 
@@ -1115,7 +1118,7 @@ def get_checksum_file_list(d):
     """
     fetch = Fetch([], d, cache = False, localonly = True)
 
-    dl_dir = d.getVar('DL_DIR', True)
+    dl_dir = d.getVar('DL_DIR')
     filelist = []
     for u in fetch.urls:
         ud = fetch.ud[u]
@@ -1129,9 +1132,9 @@ def get_checksum_file_list(d):
                 if f.startswith(dl_dir):
                     # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
                     if os.path.exists(f):
-                        bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN', True), os.path.basename(f)))
+                        bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f)))
                     else:
-                        bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN', True), os.path.basename(f)))
+                        bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f)))
                 filelist.append(f + ":" + str(os.path.exists(f)))
 
     return " ".join(filelist)
@@ -1180,13 +1183,13 @@ class FetchData(object):
         elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
             self.md5_expected = None
         else:
-            self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name, True)
+            self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
         if self.sha256_name in self.parm:
             self.sha256_expected = self.parm[self.sha256_name]
         elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
             self.sha256_expected = None
         else:
-            self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name, True)
+            self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
         self.ignore_checksums = False
 
         self.names = self.parm.get("name",'default').split(',')
@@ -1204,7 +1207,7 @@ class FetchData(object):
             raise NonLocalMethod()
 
         if self.parm.get("proto", None) and "protocol" not in self.parm:
-            logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN', True))
+            logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN'))
             self.parm["protocol"] = self.parm.get("proto", None)
 
         if hasattr(self.method, "urldata_init"):
@@ -1217,7 +1220,7 @@ class FetchData(object):
         elif self.localfile:
             self.localpath = self.method.localpath(self, d)
 
-        dldir = d.getVar("DL_DIR", True)
+        dldir = d.getVar("DL_DIR")
 
         if not self.needdonestamp:
             return
@@ -1257,12 +1260,12 @@ class FetchData(object):
         if "srcdate" in self.parm:
             return self.parm['srcdate']
 
-        pn = d.getVar("PN", True)
+        pn = d.getVar("PN")
 
         if pn:
-            return d.getVar("SRCDATE_%s" % pn, True) or d.getVar("SRCDATE", True) or d.getVar("DATE", True)
+            return d.getVar("SRCDATE_%s" % pn) or d.getVar("SRCDATE") or d.getVar("DATE")
 
-        return d.getVar("SRCDATE", True) or d.getVar("DATE", True)
+        return d.getVar("SRCDATE") or d.getVar("DATE")
 
 class FetchMethod(object):
     """Base class for 'fetch'ing data"""
@@ -1282,7 +1285,7 @@ class FetchMethod(object):
         Can also setup variables in urldata for use in go (saving code duplication
         and duplicate code execution)
         """
-        return os.path.join(data.getVar("DL_DIR", d, True), urldata.localfile)
+        return os.path.join(d.getVar("DL_DIR"), urldata.localfile)
 
     def supports_checksum(self, urldata):
         """
@@ -1450,7 +1453,7 @@ class FetchMethod(object):
         if not cmd:
             return
 
-        path = data.getVar('PATH', True)
+        path = data.getVar('PATH')
         if path:
             cmd = "PATH=\"%s\" %s" % (path, cmd)
         bb.note("Unpacking %s to %s/" % (file, unpackdir))
@@ -1507,7 +1510,7 @@ class FetchMethod(object):
 
     def generate_revision_key(self, ud, d, name):
         key = self._revision_key(ud, d, name)
-        return "%s-%s" % (key, d.getVar("PN", True) or "")
+        return "%s-%s" % (key, d.getVar("PN") or "")
 
 class Fetch(object):
     def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
@@ -1515,14 +1518,14 @@ class Fetch(object):
             raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
 
         if len(urls) == 0:
-            urls = d.getVar("SRC_URI", True).split()
+            urls = d.getVar("SRC_URI").split()
         self.urls = urls
         self.d = d
         self.ud = {}
         self.connection_cache = connection_cache
 
-        fn = d.getVar('FILE', True)
-        mc = d.getVar('__BBMULTICONFIG', True) or ""
+        fn = d.getVar('FILE')
+        mc = d.getVar('__BBMULTICONFIG') or ""
         if cache and fn and mc + fn in urldata_cache:
             self.ud = urldata_cache[mc + fn]
 
@@ -1565,8 +1568,8 @@ class Fetch(object):
         if not urls:
             urls = self.urls
 
-        network = self.d.getVar("BB_NO_NETWORK", True)
-        premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY", True) == "1")
+        network = self.d.getVar("BB_NO_NETWORK")
+        premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY") == "1")
 
         for u in urls:
             ud = self.ud[u]
@@ -1584,7 +1587,7 @@ class Fetch(object):
                     localpath = ud.localpath
                 elif m.try_premirror(ud, self.d):
                     logger.debug(1, "Trying PREMIRRORS")
-                    mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True))
+                    mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
                     localpath = try_mirrors(self, self.d, ud, mirrors, False)
 
                 if premirroronly:
@@ -1624,7 +1627,7 @@ class Fetch(object):
                         if not verified_stamp:
                             m.clean(ud, self.d)
                         logger.debug(1, "Trying MIRRORS")
-                        mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
+                        mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                         localpath = try_mirrors(self, self.d, ud, mirrors)
 
                 if not localpath or ((not os.path.exists(localpath)) and localpath.find("*") == -1):
@@ -1657,7 +1660,7 @@ class Fetch(object):
             m = ud.method
             logger.debug(1, "Testing URL %s", u)
             # First try checking uri, u, from PREMIRRORS
-            mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True))
+            mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
             ret = try_mirrors(self, self.d, ud, mirrors, True)
             if not ret:
                 # Next try checking from the original uri, u
@@ -1665,7 +1668,7 @@ class Fetch(object):
                     ret = m.checkstatus(self, ud, self.d)
                 except:
                     # Finally, try checking uri, u, from MIRRORS
-                    mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
+                    mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                     ret = try_mirrors(self, self.d, ud, mirrors, True)
 
             if not ret:

bitbake/lib/bb/fetch2/clearcase.py

@@ -108,13 +108,13 @@ class ClearCase(FetchMethod):
         else:
             ud.module = ""
 
-        ud.basecmd = d.getVar("FETCHCMD_ccrc", True) or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
+        ud.basecmd = d.getVar("FETCHCMD_ccrc") or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
 
-        if data.getVar("SRCREV", d, True) == "INVALID":
+        if d.getVar("SRCREV") == "INVALID":
           raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")
 
         ud.label = d.getVar("SRCREV", False)
-        ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", True)
+        ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC")
 
         ud.server     = "%s://%s%s" % (ud.proto, ud.host, ud.path)
 
@@ -124,7 +124,7 @@ class ClearCase(FetchMethod):
 
         ud.viewname         = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True))
         ud.csname           = "%s-config-spec" % (ud.identifier)
-        ud.ccasedir         = os.path.join(data.getVar("DL_DIR", d, True), ud.type)
+        ud.ccasedir         = os.path.join(d.getVar("DL_DIR"), ud.type)
         ud.viewdir          = os.path.join(ud.ccasedir, ud.viewname)
         ud.configspecfile   = os.path.join(ud.ccasedir, ud.csname)
         ud.localfile        = "%s.tar.gz" % (ud.identifier)
@@ -144,7 +144,7 @@ class ClearCase(FetchMethod):
         self.debug("configspecfile  = %s" % ud.configspecfile)
         self.debug("localfile       = %s" % ud.localfile)
 
-        ud.localfile = os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
+        ud.localfile = os.path.join(d.getVar("DL_DIR"), ud.localfile)
 
     def _build_ccase_command(self, ud, command):
         """

bitbake/lib/bb/fetch2/cvs.py

@@ -87,10 +87,10 @@ class Cvs(FetchMethod):
             cvsroot = ud.path
         else:
             cvsroot = ":" + method
-            cvsproxyhost = d.getVar('CVS_PROXY_HOST', True)
+            cvsproxyhost = d.getVar('CVS_PROXY_HOST')
             if cvsproxyhost:
                 cvsroot += ";proxy=" + cvsproxyhost
-            cvsproxyport = d.getVar('CVS_PROXY_PORT', True)
+            cvsproxyport = d.getVar('CVS_PROXY_PORT')
             if cvsproxyport:
                 cvsroot += ";proxyport=" + cvsproxyport
             cvsroot += ":" + ud.user
@@ -110,7 +110,7 @@ class Cvs(FetchMethod):
         if ud.tag:
             options.append("-r %s" % ud.tag)
 
-        cvsbasecmd = d.getVar("FETCHCMD_cvs", True)
+        cvsbasecmd = d.getVar("FETCHCMD_cvs")
         cvscmd = cvsbasecmd + " '-d" + cvsroot + "' co " + " ".join(options) + " " + ud.module
         cvsupdatecmd = cvsbasecmd + " '-d" + cvsroot + "' update -d -P " + " ".join(options)
 
@@ -120,8 +120,8 @@ class Cvs(FetchMethod):
 
         # create module directory
         logger.debug(2, "Fetch: checking for module directory")
-        pkg = d.getVar('PN', True)
-        pkgdir = os.path.join(d.getVar('CVSDIR', True), pkg)
+        pkg = d.getVar('PN')
+        pkgdir = os.path.join(d.getVar('CVSDIR'), pkg)
         moddir = os.path.join(pkgdir, localdir)
         workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
@@ -164,8 +164,8 @@ class Cvs(FetchMethod):
     def clean(self, ud, d):
         """ Clean CVS Files and tarballs """
 
-        pkg = d.getVar('PN', True)
-        pkgdir = os.path.join(d.getVar("CVSDIR", True), pkg)
+        pkg = d.getVar('PN')
+        pkgdir = os.path.join(d.getVar("CVSDIR"), pkg)
 
         bb.utils.remove(pkgdir, True)
         bb.utils.remove(ud.localpath)

bitbake/lib/bb/fetch2/git.py

@@ -182,9 +182,9 @@ class Git(FetchMethod):
         if ud.usehead:
             ud.unresolvedrev['default'] = 'HEAD'
 
-        ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git -c core.fsyncobjectfiles=0"
+        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
 
-        ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable
+        ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") or ud.rebaseable
 
         ud.setup_revisons(d)
 
@@ -207,8 +207,8 @@ class Git(FetchMethod):
             for name in ud.names:
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
         ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
-        gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/")
+        ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
+        gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
         ud.clonedir = os.path.join(gitdir, gitsrcname)
 
         ud.localfile = ud.clonedir
@@ -229,7 +229,7 @@ class Git(FetchMethod):
     def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
-        if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
+        if d.getVar("BB_FETCH_PREMIRRORONLY") is not None:
             return True
         if os.path.exists(ud.clonedir):
             return False
@@ -252,7 +252,7 @@ class Git(FetchMethod):
                 repourl = repourl[7:]
             clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
             if ud.proto.lower() != 'file':
-                bb.fetch2.check_network_access(d, clone_cmd)
+                bb.fetch2.check_network_access(d, clone_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
             runfetchcmd(clone_cmd, d, log=progresshandler)
 
@@ -384,7 +384,7 @@ class Git(FetchMethod):
         cmd = "%s ls-remote %s %s" % \
               (ud.basecmd, repourl, search)
         if ud.proto.lower() != 'file':
-            bb.fetch2.check_network_access(d, cmd)
+            bb.fetch2.check_network_access(d, cmd, repourl)
         output = runfetchcmd(cmd, d, True)
         if not output:
             raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
@@ -418,7 +418,7 @@ class Git(FetchMethod):
         """
         pupver = ('', '')
 
-        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)")
+        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or "(?P<pver>([0-9][\.|_]?)+)")
         try:
             output = self._lsremote(ud, d, "refs/tags/*")
         except bb.fetch2.FetchError or bb.fetch2.NetworkAccess:

bitbake/lib/bb/fetch2/hg.py

@@ -78,15 +78,15 @@ class Hg(FetchMethod):
         hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
                             ud.host, ud.path.replace('/', '.'))
         ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
+        ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
 
-        hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + "/hg/")
+        hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/")
         ud.pkgdir = os.path.join(hgdir, hgsrcname)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
         ud.localfile = ud.moddir
-        ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg"
+        ud.basecmd = d.getVar("FETCHCMD_hg") or "/usr/bin/env hg"
 
-        ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True)
+        ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS")
 
     def need_update(self, ud, d):
         revTag = ud.parm.get('rev', 'tip')
@@ -99,7 +99,7 @@ class Hg(FetchMethod):
     def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
-        if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
+        if d.getVar("BB_FETCH_PREMIRRORONLY") is not None:
             return True
         if os.path.exists(ud.moddir):
             return False
@@ -221,7 +221,7 @@ class Hg(FetchMethod):
         """
         Compute tip revision for the url
         """
-        bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info"))
+        bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info"), ud.url)
         output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d)
         return output.strip()
 

bitbake/lib/bb/fetch2/local.py

@@ -63,13 +63,13 @@ class Local(FetchMethod):
         newpath = path
         if path[0] == "/":
             return [path]
-        filespath = data.getVar('FILESPATH', d, True)
+        filespath = d.getVar('FILESPATH')
         if filespath:
             logger.debug(2, "Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
             searched.extend(hist)
         if not newpath:
-            filesdir = data.getVar('FILESDIR', d, True)
+            filesdir = d.getVar('FILESDIR')
             if filesdir:
                 logger.debug(2, "Searching for %s in path: %s" % (path, filesdir))
                 newpath = os.path.join(filesdir, path)
@@ -81,7 +81,7 @@ class Local(FetchMethod):
             logger.debug(2, "Searching for %s in path: %s" % (path, newpath))
             return searched
         if not os.path.exists(newpath):
-            dldirfile = os.path.join(d.getVar("DL_DIR", True), path)
+            dldirfile = os.path.join(d.getVar("DL_DIR"), path)
             logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path))
             bb.utils.mkdirhier(os.path.dirname(dldirfile))
             searched.append(dldirfile)
@@ -100,13 +100,13 @@ class Local(FetchMethod):
         # no need to fetch local files, we'll deal with them in place.
         if self.supports_checksum(urldata) and not os.path.exists(urldata.localpath):
             locations = []
-            filespath = data.getVar('FILESPATH', d, True)
+            filespath = d.getVar('FILESPATH')
             if filespath:
                 locations = filespath.split(":")
-            filesdir = data.getVar('FILESDIR', d, True)
+            filesdir = d.getVar('FILESDIR')
             if filesdir:
                 locations.append(filesdir)
-            locations.append(d.getVar("DL_DIR", True))
+            locations.append(d.getVar("DL_DIR"))
 
             msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n    " + "\n    ".join(locations)
             raise FetchError(msg)

bitbake/lib/bb/fetch2/npm.py

@@ -87,12 +87,12 @@ class Npm(FetchMethod):
             bb.utils.mkdirhier(ud.pkgdatadir)
         ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
 
-        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
         ud.prefixdir = prefixdir
 
-        ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0")
+        ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
         ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
+        ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
 
     def need_update(self, ud, d):
         if os.path.exists(ud.localpath):
@@ -101,8 +101,8 @@ class Npm(FetchMethod):
 
     def _runwget(self, ud, d, command, quiet):
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
-        bb.fetch2.check_network_access(d, command)
-        dldir = d.getVar("DL_DIR", True)
+        bb.fetch2.check_network_access(d, command, ud.url)
+        dldir = d.getVar("DL_DIR")
         runfetchcmd(command, d, quiet, workdir=dldir)
 
     def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
@@ -116,7 +116,7 @@ class Npm(FetchMethod):
         # Change to subdir before executing command
         if not os.path.exists(destdir):
             os.makedirs(destdir)
-        path = d.getVar('PATH', True)
+        path = d.getVar('PATH')
         if path:
             cmd = "PATH=\"%s\" %s" % (path, cmd)
         bb.note("Unpacking %s to %s/" % (file, destdir))
@@ -132,7 +132,7 @@ class Npm(FetchMethod):
 
 
     def unpack(self, ud, destdir, d):
-        dldir = d.getVar("DL_DIR", True)
+        dldir = d.getVar("DL_DIR")
         depdumpfile = "%s-%s.deps.json" % (ud.pkgname, ud.version)
         with open("%s/npm/%s" % (dldir, depdumpfile)) as datafile:
             workobj = json.load(datafile)
@@ -251,24 +251,30 @@ class Npm(FetchMethod):
         lockdown = {}
 
         if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror):
-            dest = d.getVar("DL_DIR", True)
+            dest = d.getVar("DL_DIR")
             bb.utils.mkdirhier(dest)
             runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest)
             return
 
-        shwrf = d.getVar('NPM_SHRINKWRAP', True)
+        shwrf = d.getVar('NPM_SHRINKWRAP')
         logger.debug(2, "NPM shrinkwrap file is %s" % shwrf)
-        try:
-            with open(shwrf) as datafile:
-                shrinkobj = json.load(datafile)
-        except:
+        if shwrf:
+            try:
+                with open(shwrf) as datafile:
+                    shrinkobj = json.load(datafile)
+            except Exception as e:
+                raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e)))
+        elif not ud.ignore_checksums:
             logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname)
-        lckdf = d.getVar('NPM_LOCKDOWN', True)
+        lckdf = d.getVar('NPM_LOCKDOWN')
         logger.debug(2, "NPM lockdown file is %s" % lckdf)
-        try:
-            with open(lckdf) as datafile:
-                lockdown = json.load(datafile)
-        except:
+        if lckdf:
+            try:
+                with open(lckdf) as datafile:
+                    lockdown = json.load(datafile)
+            except Exception as e:
+                raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e)))
+        elif not ud.ignore_checksums:
             logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname)
 
         if ('name' not in shrinkobj):
@@ -286,7 +292,7 @@ class Npm(FetchMethod):
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
 
-            dldir = d.getVar("DL_DIR", True)
+            dldir = d.getVar("DL_DIR")
             logger.info("Creating tarball of npm data")
             runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d,
                         workdir=dldir)

bitbake/lib/bb/fetch2/osc.py

@@ -34,7 +34,7 @@ class Osc(FetchMethod):
 
         # Create paths to osc checkouts
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.getVar('OSCDIR', True), ud.host)
+        ud.pkgdir = os.path.join(d.getVar('OSCDIR'), ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
         if 'rev' in ud.parm:
@@ -84,7 +84,7 @@ class Osc(FetchMethod):
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(os.path.join(d.getVar('OSCDIR', True), ud.path, ud.module), os.R_OK):
+        if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
@@ -112,7 +112,7 @@ class Osc(FetchMethod):
         Generate a .oscrc to be used for this run.
         """
 
-        config_path = os.path.join(d.getVar('OSCDIR', True), "oscrc")
+        config_path = os.path.join(d.getVar('OSCDIR'), "oscrc")
         if (os.path.exists(config_path)):
             os.remove(config_path)
 
@@ -121,8 +121,8 @@ class Osc(FetchMethod):
         f.write("apisrv = %s\n" % ud.host)
         f.write("scheme = http\n")
         f.write("su-wrapper = su -c\n")
-        f.write("build-root = %s\n" % d.getVar('WORKDIR', True))
-        f.write("urllist = %s\n" % d.getVar("OSCURLLIST", True))
+        f.write("build-root = %s\n" % d.getVar('WORKDIR'))
+        f.write("urllist = %s\n" % d.getVar("OSCURLLIST"))
         f.write("extra-pkgs = gzip\n")
         f.write("\n")
         f.write("[%s]\n" % ud.host)

bitbake/lib/bb/fetch2/perforce.py

@@ -44,13 +44,13 @@ class Perforce(FetchMethod):
         provided by the env, use it.  If P4PORT is specified by the recipe, use
         its values, which may override the settings in P4CONFIG.
         """
-        ud.basecmd = d.getVar('FETCHCMD_p4', True)
+        ud.basecmd = d.getVar('FETCHCMD_p4')
         if not ud.basecmd:
             ud.basecmd = "/usr/bin/env p4"
 
-        ud.dldir = d.getVar('P4DIR', True)
+        ud.dldir = d.getVar('P4DIR')
         if not ud.dldir:
-            ud.dldir = '%s/%s' % (d.getVar('DL_DIR', True), 'p4')
+            ud.dldir = '%s/%s' % (d.getVar('DL_DIR'), 'p4')
 
         path = ud.url.split('://')[1]
         path = path.split(';')[0]
@@ -62,7 +62,7 @@ class Perforce(FetchMethod):
             ud.path = path
 
         ud.usingp4config = False
-        p4port = d.getVar('P4PORT', True)
+        p4port = d.getVar('P4PORT')
 
         if p4port:
             logger.debug(1, 'Using recipe provided P4PORT: %s' % p4port)
@@ -71,7 +71,7 @@ class Perforce(FetchMethod):
             logger.debug(1, 'Trying to use P4CONFIG to automatically set P4PORT...')
             ud.usingp4config = True
             p4cmd = '%s info | grep "Server address"' % ud.basecmd
-            bb.fetch2.check_network_access(d, p4cmd)
+            bb.fetch2.check_network_access(d, p4cmd, ud.url)
             ud.host = runfetchcmd(p4cmd, d, True)
             ud.host = ud.host.split(': ')[1].strip()
             logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
@@ -140,7 +140,7 @@ class Perforce(FetchMethod):
         'p4 files' command, including trailing '#rev' file revision indicator
         """
         p4cmd = self._buildp4command(ud, d, 'files')
-        bb.fetch2.check_network_access(d, p4cmd)
+        bb.fetch2.check_network_access(d, p4cmd, ud.url)
         p4fileslist = runfetchcmd(p4cmd, d, True)
         p4fileslist = [f.rstrip() for f in p4fileslist.splitlines()]
 
@@ -171,7 +171,7 @@ class Perforce(FetchMethod):
 
         for afile in filelist:
             p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
-            bb.fetch2.check_network_access(d, p4fetchcmd)
+            bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
             runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir)
 
         runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
@@ -191,7 +191,7 @@ class Perforce(FetchMethod):
     def _latest_revision(self, ud, d, name):
         """ Return the latest upstream scm revision number """
         p4cmd = self._buildp4command(ud, d, "changes")
-        bb.fetch2.check_network_access(d, p4cmd)
+        bb.fetch2.check_network_access(d, p4cmd, ud.url)
         tip = runfetchcmd(p4cmd, d, True)
 
         if not tip:

bitbake/lib/bb/fetch2/repo.py

@@ -51,17 +51,17 @@ class Repo(FetchMethod):
         if not ud.manifest.endswith('.xml'):
             ud.manifest += '.xml'
 
-        ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d)
+        ud.localfile = d.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch))
 
     def download(self, ud, d):
         """Fetch url"""
 
-        if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
+        if os.access(os.path.join(d.getVar("DL_DIR"), ud.localfile), os.R_OK):
             logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
             return
 
         gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
-        repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
+        repodir = d.getVar("REPODIR") or os.path.join(d.getVar("DL_DIR"), "repo")
         codir = os.path.join(repodir, gitsrcname, ud.manifest)
 
         if ud.user:

bitbake/lib/bb/fetch2/sftp.py

@@ -92,7 +92,7 @@ class SFTP(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = data.expand(urllib.parse.unquote(ud.basename), d)
+        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
 
     def download(self, ud, d):
         """Fetch urls"""
@@ -104,7 +104,7 @@ class SFTP(FetchMethod):
             port = '-P %d' % urlo.port
             urlo.port = None
 
-        dldir = data.getVar('DL_DIR', d, True)
+        dldir = d.getVar('DL_DIR')
         lpath = os.path.join(dldir, ud.localfile)
 
         user = ''

bitbake/lib/bb/fetch2/ssh.py

@@ -87,11 +87,11 @@ class SSH(FetchMethod):
         m = __pattern__.match(urldata.url)
         path = m.group('path')
         host = m.group('host')
-        urldata.localpath = os.path.join(d.getVar('DL_DIR', True),
+        urldata.localpath = os.path.join(d.getVar('DL_DIR'),
                 os.path.basename(os.path.normpath(path)))
 
     def download(self, urldata, d):
-        dldir = d.getVar('DL_DIR', True)
+        dldir = d.getVar('DL_DIR')
 
         m = __pattern__.match(urldata.url)
         path = m.group('path')

bitbake/lib/bb/fetch2/svn.py

@@ -50,7 +50,7 @@ class Svn(FetchMethod):
         if not "module" in ud.parm:
             raise MissingParameterError('module', ud.url)
 
-        ud.basecmd = d.getVar('FETCHCMD_svn', True)
+        ud.basecmd = d.getVar('FETCHCMD_svn')
 
         ud.module = ud.parm["module"]
 
@@ -173,7 +173,7 @@ class Svn(FetchMethod):
         """
         Return the latest upstream revision number
         """
-        bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "log1"))
+        bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "log1"), ud.url)
 
         output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "log1"), d, True)
 

bitbake/lib/bb/fetch2/wget.py

@@ -88,14 +88,14 @@ class Wget(FetchMethod):
         if not ud.localfile:
             ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
 
-        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
 
     def _runwget(self, ud, d, command, quiet):
 
         progresshandler = WgetProgressHandler(d)
 
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
-        bb.fetch2.check_network_access(d, command)
+        bb.fetch2.check_network_access(d, command, ud.url)
         runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
 
     def download(self, ud, d):
@@ -104,7 +104,7 @@ class Wget(FetchMethod):
         fetchcmd = self.basecmd
 
         if 'downloadfilename' in ud.parm:
-            dldir = d.getVar("DL_DIR", True)
+            dldir = d.getVar("DL_DIR")
             bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile))
             fetchcmd += " -O " + dldir + os.sep + ud.localfile
 
@@ -543,7 +543,7 @@ class Wget(FetchMethod):
         self.suffix_regex_comp = re.compile(psuffix_regex)
 
         # compile regex, can be specific by package or generic regex
-        pn_regex = d.getVar('UPSTREAM_CHECK_REGEX', True)
+        pn_regex = d.getVar('UPSTREAM_CHECK_REGEX')
         if pn_regex:
             package_custom_regex_comp = re.compile(pn_regex)
         else:
@@ -564,7 +564,7 @@ class Wget(FetchMethod):
         sanity check to ensure same name and type.
         """
         package = ud.path.split("/")[-1]
-        current_version = ['', d.getVar('PV', True), '']
+        current_version = ['', d.getVar('PV'), '']
 
         """possible to have no version in pkg name, such as spectrum-fw"""
         if not re.search("\d+", package):
@@ -579,7 +579,7 @@ class Wget(FetchMethod):
         bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
 
         uri = ""
-        regex_uri = d.getVar("UPSTREAM_CHECK_URI", True)
+        regex_uri = d.getVar("UPSTREAM_CHECK_URI")
         if not regex_uri:
             path = ud.path.split(package)[0]
 
@@ -588,7 +588,7 @@ class Wget(FetchMethod):
             dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
             m = dirver_regex.search(path)
             if m:
-                pn = d.getVar('PN', True)
+                pn = d.getVar('PN')
                 dirver = m.group('dirver')
 
                 dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn)))

bitbake/lib/bb/main.py

@@ -179,8 +179,8 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
                           help="Increase the debug level. You can specify this more than once.")
 
-        parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False,
-                          help="Output less log message data to the terminal.")
+        parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
+                          help="Output less log message data to the terminal. You can specify this more than once.")
 
         parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
                           help="Don't execute, just go through the motions.")
@@ -389,12 +389,8 @@ def bitbake_main(configParams, configuration):
     except:
         pass
 
-
     configuration.setConfigParameters(configParams)
 
-    ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
-    servermodule = import_extension_module(bb.server, configParams.servertype, 'BitBakeServer')
-
     if configParams.server_only:
         if configParams.servertype != "xmlrpc":
             raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
@@ -442,66 +438,11 @@ def bitbake_main(configParams, configuration):
     bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
                           configuration.debug_domains)
 
-    # Ensure logging messages get sent to the UI as events
-    handler = bb.event.LogHandler()
-    if not configParams.status_only:
-        # In status only mode there are no logs and no UI
-        logger.addHandler(handler)
-
-    # Clear away any spurious environment variables while we stoke up the cooker
-    cleanedvars = bb.utils.clean_environment()
-
-    featureset = []
-    if not configParams.server_only:
-        # Collect the feature set for the UI
-        featureset = getattr(ui_module, "featureSet", [])
-
-    if configParams.server_only:
-        for param in ('prefile', 'postfile'):
-            value = getattr(configParams, param)
-            if value:
-                setattr(configuration, "%s_server" % param, value)
-                param = "%s_server" % param
-
-    if not configParams.remote_server:
-        # we start a server with a given configuration
-        server = start_server(servermodule, configParams, configuration, featureset)
-        bb.event.ui_queue = []
-    else:
-        if os.getenv('BBSERVER') == 'autostart':
-            if configParams.remote_server == 'autostart' or \
-               not servermodule.check_connection(configParams.remote_server, timeout=2):
-                configParams.bind = 'localhost:0'
-                srv = start_server(servermodule, configParams, configuration, featureset)
-                configParams.remote_server = '%s:%d' % tuple(configuration.interface)
-                bb.event.ui_queue = []
-
-        # we start a stub server that is actually a XMLRPClient that connects to a real server
-        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
-                                                  configParams.xmlrpctoken)
-        server.saveConnectionDetails(configParams.remote_server)
-
+    server, server_connection, ui_module = setup_bitbake(configParams, configuration)
+    if server_connection is None and configParams.kill_server:
+        return 0
 
     if not configParams.server_only:
-        try:
-            server_connection = server.establishConnection(featureset)
-        except Exception as e:
-            bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
-
-        if configParams.kill_server:
-            server_connection.connection.terminateServer()
-            bb.event.ui_queue = []
-            return 0
-
-        server_connection.setupEventQueue()
-
-        # Restore the environment in case the UI needs it
-        for k in cleanedvars:
-            os.environ[k] = cleanedvars[k]
-
-        logger.removeHandler(handler)
-
-
         if configParams.status_only:
             server_connection.terminate()
             return 0
@@ -520,3 +461,77 @@ def bitbake_main(configParams, configuration):
         return 0
 
     return 1
+
+def setup_bitbake(configParams, configuration, extrafeatures=None):
+    # Ensure logging messages get sent to the UI as events
+    handler = bb.event.LogHandler()
+    if not configParams.status_only:
+        # In status only mode there are no logs and no UI
+        logger.addHandler(handler)
+
+    # Clear away any spurious environment variables while we stoke up the cooker
+    cleanedvars = bb.utils.clean_environment()
+
+    if configParams.server_only:
+        featureset = []
+        ui_module = None
+    else:
+        ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
+        # Collect the feature set for the UI
+        featureset = getattr(ui_module, "featureSet", [])
+
+    if configParams.server_only:
+        for param in ('prefile', 'postfile'):
+            value = getattr(configParams, param)
+            if value:
+                setattr(configuration, "%s_server" % param, value)
+                param = "%s_server" % param
+
+    if extrafeatures:
+        for feature in extrafeatures:
+            if not feature in featureset:
+                featureset.append(feature)
+
+    servermodule = import_extension_module(bb.server,
+                                            configParams.servertype,
+                                            'BitBakeServer')
+    if configParams.remote_server:
+        if os.getenv('BBSERVER') == 'autostart':
+            if configParams.remote_server == 'autostart' or \
+               not servermodule.check_connection(configParams.remote_server, timeout=2):
+                configParams.bind = 'localhost:0'
+                srv = start_server(servermodule, configParams, configuration, featureset)
+                configParams.remote_server = '%s:%d' % tuple(configuration.interface)
+                bb.event.ui_queue = []
+        # we start a stub server that is actually a XMLRPClient that connects to a real server
+        from bb.server.xmlrpc import BitBakeXMLRPCClient
+        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
+                                                  configParams.xmlrpctoken)
+        server.saveConnectionDetails(configParams.remote_server)
+    else:
+        # we start a server with a given configuration
+        server = start_server(servermodule, configParams, configuration, featureset)
+        bb.event.ui_queue = []
+
+    if configParams.server_only:
+        server_connection = None
+    else:
+        try:
+            server_connection = server.establishConnection(featureset)
+        except Exception as e:
+            bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
+
+        if configParams.kill_server:
+            server_connection.connection.terminateServer()
+            bb.event.ui_queue = []
+            return None, None, None
+
+        server_connection.setupEventQueue()
+
+        # Restore the environment in case the UI needs it
+        for k in cleanedvars:
+            os.environ[k] = cleanedvars[k]
+
+        logger.removeHandler(handler)
+
+    return server, server_connection, ui_module

bitbake/lib/bb/monitordisk.py

@@ -129,7 +129,7 @@ def getDiskData(BBDirs, configuration):
             bb.utils.mkdirhier(path)
         dev = getMountedDev(path)
         # Use path/action as the key
-        devDict[os.path.join(path, action)] = [dev, minSpace, minInode]
+        devDict[(path, action)] = [dev, minSpace, minInode]
 
     return devDict
 
@@ -141,7 +141,7 @@ def getInterval(configuration):
     spaceDefault = 50 * 1024 * 1024
     inodeDefault = 5 * 1024
 
-    interval = configuration.getVar("BB_DISKMON_WARNINTERVAL", True)
+    interval = configuration.getVar("BB_DISKMON_WARNINTERVAL")
     if not interval:
         return spaceDefault, inodeDefault
     else:
@@ -179,7 +179,7 @@ class diskMonitor:
         self.enableMonitor = False
         self.configuration = configuration
 
-        BBDirs = configuration.getVar("BB_DISKMON_DIRS", True) or None
+        BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
         if BBDirs:
             self.devDict = getDiskData(BBDirs, configuration)
             if self.devDict:
@@ -205,18 +205,21 @@ class diskMonitor:
         """ Take action for the monitor """
 
         if self.enableMonitor:
-            for k in self.devDict:
-                path = os.path.dirname(k)
-                action = os.path.basename(k)
-                dev = self.devDict[k][0]
-                minSpace = self.devDict[k][1]
-                minInode = self.devDict[k][2]
+            diskUsage = {}
+            for k, attributes in self.devDict.items():
+                path, action = k
+                dev, minSpace, minInode = attributes
 
                 st = os.statvfs(path)
 
-                # The free space, float point number
+                # The available free space, integer number
                 freeSpace = st.f_bavail * st.f_frsize
 
+                # Send all relevant information in the event.
+                freeSpaceRoot = st.f_bfree * st.f_frsize
+                totalSpace = st.f_blocks * st.f_frsize
+                diskUsage[dev] = bb.event.DiskUsageSample(freeSpace, freeSpaceRoot, totalSpace)
+
                 if minSpace and freeSpace < minSpace:
                     # Always show warning, the self.checked would always be False if the action is WARN
                     if self.preFreeS[k] == 0 or self.preFreeS[k] - freeSpace > self.spaceInterval and not self.checked[k]:
@@ -235,7 +238,7 @@ class diskMonitor:
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
 
-                # The free inodes, float point number
+                # The free inodes, integer number
                 freeInode = st.f_favail
 
                 if minInode and freeInode < minInode:
@@ -260,4 +263,6 @@ class diskMonitor:
                         self.checked[k] = True
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
+
+            bb.event.fire(bb.event.MonitorDiskEvent(diskUsage), self.configuration)
         return

bitbake/lib/bb/parse/__init__.py

@@ -123,7 +123,7 @@ def init_parser(d):
 
 def resolve_file(fn, d):
     if not os.path.isabs(fn):
-        bbpath = d.getVar("BBPATH", True)
+        bbpath = d.getVar("BBPATH")
         newfn, attempts = bb.utils.which(bbpath, fn, history=True)
         for af in attempts:
             mark_dependency(d, af)

bitbake/lib/bb/parse/ast.py

@@ -30,8 +30,6 @@ import itertools
 from bb import methodpool
 from bb.parse import logger
 
-_bbversions_re = re.compile(r"\[(?P<from>[0-9]+)-(?P<to>[0-9]+)\]")
-
 class StatementGroup(list):
     def eval(self, data):
         for statement in self:
@@ -347,7 +345,7 @@ def finalize(fn, d, variant = None):
         if not handlerfn:
             bb.fatal("Undefined event handler function '%s'" % var)
         handlerln = int(d.getVarFlag(var, "lineno", False))
-        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
+        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
 
     bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
@@ -385,29 +383,8 @@ def _create_variants(datastores, names, function, onlyfinalise):
             else:
                 create_variant("%s-%s" % (variant, name), datastores[variant], name)
 
-def _expand_versions(versions):
-    def expand_one(version, start, end):
-        for i in range(start, end + 1):
-            ver = _bbversions_re.sub(str(i), version, 1)
-            yield ver
-
-    versions = iter(versions)
-    while True:
-        try:
-            version = next(versions)
-        except StopIteration:
-            break
-
-        range_ver = _bbversions_re.search(version)
-        if not range_ver:
-            yield version
-        else:
-            newversions = expand_one(version, int(range_ver.group("from")),
-                                     int(range_ver.group("to")))
-            versions = itertools.chain(newversions, versions)
-
 def multi_finalize(fn, d):
-    appends = (d.getVar("__BBAPPEND", True) or "").split()
+    appends = (d.getVar("__BBAPPEND") or "").split()
     for append in appends:
         logger.debug(1, "Appending .bbappend file %s to %s", append, fn)
         bb.parse.BBHandler.handle(append, d, True)
@@ -422,51 +399,7 @@ def multi_finalize(fn, d):
         d.setVar("__SKIPPED", e.args[0])
     datastores = {"": safe_d}
 
-    versions = (d.getVar("BBVERSIONS", True) or "").split()
-    if versions:
-        pv = orig_pv = d.getVar("PV", True)
-        baseversions = {}
-
-        def verfunc(ver, d, pv_d = None):
-            if pv_d is None:
-                pv_d = d
-
-            overrides = d.getVar("OVERRIDES", True).split(":")
-            pv_d.setVar("PV", ver)
-            overrides.append(ver)
-            bpv = baseversions.get(ver) or orig_pv
-            pv_d.setVar("BPV", bpv)
-            overrides.append(bpv)
-            d.setVar("OVERRIDES", ":".join(overrides))
-
-        versions = list(_expand_versions(versions))
-        for pos, version in enumerate(list(versions)):
-            try:
-                pv, bpv = version.split(":", 2)
-            except ValueError:
-                pass
-            else:
-                versions[pos] = pv
-                baseversions[pv] = bpv
-
-        if pv in versions and not baseversions.get(pv):
-            versions.remove(pv)
-        else:
-            pv = versions.pop()
-
-            # This is necessary because our existing main datastore
-            # has already been finalized with the old PV, we need one
-            # that's been finalized with the new PV.
-            d = bb.data.createCopy(safe_d)
-            verfunc(pv, d, safe_d)
-            try:
-                finalize(fn, d)
-            except bb.parse.SkipRecipe as e:
-                d.setVar("__SKIPPED", e.args[0])
-
-        _create_variants(datastores, versions, verfunc, onlyfinalise)
-
-    extended = d.getVar("BBCLASSEXTEND", True) or ""
+    extended = d.getVar("BBCLASSEXTEND") or ""
     if extended:
         # the following is to support bbextends with arguments, for e.g. multilib
         # an example is as follows:
@@ -484,7 +417,7 @@ def multi_finalize(fn, d):
             else:
                 extendedmap[ext] = ext
 
-        pn = d.getVar("PN", True)
+        pn = d.getVar("PN")
         def extendfunc(name, d):
             if name != extendedmap[name]:
                 d.setVar("BBEXTENDCURR", extendedmap[name])

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -66,7 +66,7 @@ def inherit(files, fn, lineno, d):
             file = os.path.join('classes', '%s.bbclass' % file)
 
         if not os.path.isabs(file):
-            bbpath = d.getVar("BBPATH", True)
+            bbpath = d.getVar("BBPATH")
             abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
             for af in attempts:
                 if af != abs_fn:
@@ -87,17 +87,17 @@ def get_statements(filename, absolute_filename, base_name):
     try:
         return cached_statements[absolute_filename]
     except KeyError:
-        file = open(absolute_filename, 'r')
-        statements = ast.StatementGroup()
+        with open(absolute_filename, 'r') as f:
+            statements = ast.StatementGroup()
+
+            lineno = 0
+            while True:
+                lineno = lineno + 1
+                s = f.readline()
+                if not s: break
+                s = s.rstrip()
+                feeder(lineno, s, filename, base_name, statements)
 
-        lineno = 0
-        while True:
-            lineno = lineno + 1
-            s = file.readline()
-            if not s: break
-            s = s.rstrip()
-            feeder(lineno, s, filename, base_name, statements)
-        file.close()
         if __inpython__:
             # add a blank line to close out any python definition
             feeder(lineno, "", filename, base_name, statements, eof=True)

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -83,16 +83,16 @@ def include(parentfn, fn, lineno, data, error_out):
 
     if not os.path.isabs(fn):
         dname = os.path.dirname(parentfn)
-        bbpath = "%s:%s" % (dname, data.getVar("BBPATH", True))
+        bbpath = "%s:%s" % (dname, data.getVar("BBPATH"))
         abs_fn, attempts = bb.utils.which(bbpath, fn, history=True)
         if abs_fn and bb.parse.check_dependency(data, abs_fn):
-            logger.warning("Duplicate inclusion for %s in %s" % (abs_fn, data.getVar('FILE', True)))
+            logger.warning("Duplicate inclusion for %s in %s" % (abs_fn, data.getVar('FILE')))
         for af in attempts:
             bb.parse.mark_dependency(data, af)
         if abs_fn:
             fn = abs_fn
     elif bb.parse.check_dependency(data, fn):
-        logger.warning("Duplicate inclusion for %s in %s" % (fn, data.getVar('FILE', True)))
+        logger.warning("Duplicate inclusion for %s in %s" % (fn, data.getVar('FILE')))
 
     try:
         bb.parse.handle(fn, data, True)

bitbake/lib/bb/persist_data.py

@@ -207,8 +207,8 @@ def connect(database):
 def persist(domain, d):
     """Convenience factory for SQLTable objects based upon metadata"""
     import bb.utils
-    cachedir = (d.getVar("PERSISTENT_DIR", True) or
-                d.getVar("CACHE", True))
+    cachedir = (d.getVar("PERSISTENT_DIR") or
+                d.getVar("CACHE"))
     if not cachedir:
         logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
         sys.exit(1)

bitbake/lib/bb/providers.py

@@ -123,11 +123,11 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
 
     # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
     # hence we do this manually rather than use OVERRIDES
-    preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn, True)
+    preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
     if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn, True)
+        preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
     if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION", True)
+        preferred_v = cfgData.getVar("PREFERRED_VERSION")
 
     if preferred_v:
         m = re.match('(\d+:)*(.*)(_.*)*', preferred_v)
@@ -289,7 +289,7 @@ def filterProviders(providers, item, cfgData, dataCache):
 
     eligible = _filterProviders(providers, item, cfgData, dataCache)
 
-    prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item, True)
+    prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item)
     if prefervar:
         dataCache.preferred[item] = prefervar
 
@@ -318,7 +318,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
     eligible = _filterProviders(providers, item, cfgData, dataCache)
 
     # First try and match any PREFERRED_RPROVIDER entry
-    prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item, True)
+    prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item)
     foundUnique = False
     if prefervar:
         for p in eligible:
@@ -345,7 +345,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
             pn = dataCache.pkg_fn[p]
             provides = dataCache.pn_provides[pn]
             for provide in provides:
-                prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide, True)
+                prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
                 #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
                 if prefervar in pns and pns[prefervar] not in preferred:
                     var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)

bitbake/lib/bb/remotedata.py

@@ -0,0 +1,116 @@
+"""
+BitBake 'remotedata' module
+
+Provides support for using a datastore from the bitbake client
+"""
+
+# Copyright (C) 2016  Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import bb.data
+
+class RemoteDatastores:
+    """Used on the server side to manage references to server-side datastores"""
+    def __init__(self, cooker):
+        self.cooker = cooker
+        self.datastores = {}
+        self.locked = []
+        self.nextindex = 1
+
+    def __len__(self):
+        return len(self.datastores)
+
+    def __getitem__(self, key):
+        if key is None:
+            return self.cooker.data
+        else:
+            return self.datastores[key]
+
+    def items(self):
+        return self.datastores.items()
+
+    def store(self, d, locked=False):
+        """
+        Put a datastore into the collection. If locked=True then the datastore
+        is understood to be managed externally and cannot be released by calling
+        release().
+        """
+        idx = self.nextindex
+        self.datastores[idx] = d
+        if locked:
+            self.locked.append(idx)
+        self.nextindex += 1
+        return idx
+
+    def check_store(self, d, locked=False):
+        """
+        Put a datastore into the collection if it's not already in there;
+        in either case return the index
+        """
+        for key, val in self.datastores.items():
+            if val is d:
+                idx = key
+                break
+        else:
+            idx = self.store(d, locked)
+        return idx
+
+    def release(self, idx):
+        """Discard a datastore in the collection"""
+        if idx in self.locked:
+            raise Exception('Tried to release locked datastore %d' % idx)
+        del self.datastores[idx]
+
+    def receive_datastore(self, remote_data):
+        """Receive a datastore object sent from the client (as prepared by transmit_datastore())"""
+        dct = dict(remote_data)
+        d = bb.data_smart.DataSmart()
+        d.dict = dct
+        while True:
+            if '_remote_data' in dct:
+                dsindex = dct['_remote_data']['_content']
+                del dct['_remote_data']
+                if dsindex is None:
+                    dct['_data'] = self.cooker.data.dict
+                else:
+                    dct['_data'] = self.datastores[dsindex].dict
+                break
+            elif '_data' in dct:
+                idct = dict(dct['_data'])
+                dct['_data'] = idct
+                dct = idct
+            else:
+                break
+        return d
+
+    @staticmethod
+    def transmit_datastore(d):
+        """Prepare a datastore object for sending over IPC from the client end"""
+        # FIXME content might be a dict, need to turn that into a list as well
+        def copy_dicts(dct):
+            if '_remote_data' in dct:
+                dsindex = dct['_remote_data']['_content'].dsindex
+                newdct = dct.copy()
+                newdct['_remote_data'] = {'_content': dsindex}
+                return list(newdct.items())
+            elif '_data' in dct:
+                newdct = dct.copy()
+                newdata = copy_dicts(dct['_data'])
+                if newdata:
+                    newdct['_data'] = newdata
+                return list(newdct.items())
+            return None
+        main_dict = copy_dicts(d.dict)
+        return main_dict

bitbake/lib/bb/runqueue.py

@@ -262,8 +262,8 @@ class RunQueueData:
         self.rq = rq
         self.warn_multi_bb = False
 
-        self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
-        self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
+        self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST") or ""
+        self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split()
         self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
         self.setscenewhitelist_checked = False
         self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
@@ -976,16 +976,22 @@ class RunQueue:
         self.cfgData = cfgData
         self.rqdata = RunQueueData(self, cooker, cfgData, dataCaches, taskData, targets)
 
-        self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile"
-        self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None
-        self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION2", True) or None
-        self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None
+        self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY") or "perfile"
+        self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION") or None
+        self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION2") or None
+        self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID") or None
 
         self.state = runQueuePrepare
 
         # For disk space monitor
+        # Invoked at regular time intervals via the bitbake heartbeat event
+        # while the build is running. We generate a unique name for the handler
+        # here, just in case that there ever is more than one RunQueue instance,
+        # start the handler when reaching runQueueSceneRun, and stop it when
+        # done with the build.
         self.dm = monitordisk.diskMonitor(cfgData)
-
+        self.dm_event_handler_name = '_bb_diskmonitor_' + str(id(self))
+        self.dm_event_handler_registered = False
         self.rqexe = None
         self.worker = {}
         self.fakeworker = {}
@@ -997,8 +1003,8 @@ class RunQueue:
             magic = "decafbadbad"
         if fakeroot:
             magic = magic + "beef"
-            fakerootcmd = self.cfgData.getVar("FAKEROOTCMD", True)
-            fakerootenv = (self.cfgData.getVar("FAKEROOTBASEENV", True) or "").split()
+            fakerootcmd = self.cfgData.getVar("FAKEROOTCMD")
+            fakerootenv = (self.cfgData.getVar("FAKEROOTBASEENV") or "").split()
             env = os.environ.copy()
             for key, value in (var.split('=') for var in fakerootenv):
                 env[key] = value
@@ -1024,12 +1030,13 @@ class RunQueue:
             "logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
             "logdefaultdomain" : bb.msg.loggerDefaultDomains,
             "prhost" : self.cooker.prhost,
-            "buildname" : self.cfgData.getVar("BUILDNAME", True),
-            "date" : self.cfgData.getVar("DATE", True),
-            "time" : self.cfgData.getVar("TIME", True),
+            "buildname" : self.cfgData.getVar("BUILDNAME"),
+            "date" : self.cfgData.getVar("DATE"),
+            "time" : self.cfgData.getVar("TIME"),
         }
 
         worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
+        worker.stdin.write(b"<extraconfigdata>" + pickle.dumps(self.cooker.extraconfigdata) + b"</extraconfigdata>")
         worker.stdin.write(b"<workerdata>" + pickle.dumps(workerdata) + b"</workerdata>")
         worker.stdin.flush()
 
@@ -1208,10 +1215,12 @@ class RunQueue:
                 self.rqdata.init_progress_reporter.next_stage()
                 self.rqexe = RunQueueExecuteScenequeue(self)
 
-        if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]:
-            self.dm.check(self)
-
         if self.state is runQueueSceneRun:
+            if not self.dm_event_handler_registered:
+                 res = bb.event.register(self.dm_event_handler_name,
+                                         lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False,
+                                         ('bb.event.HeartbeatEvent',))
+                 self.dm_event_handler_registered = True
             retval = self.rqexe.execute()
 
         if self.state is runQueueRunInit:
@@ -1230,7 +1239,13 @@ class RunQueue:
         if self.state is runQueueCleanUp:
             retval = self.rqexe.finish()
 
-        if (self.state is runQueueComplete or self.state is runQueueFailed) and self.rqexe:
+        build_done = self.state is runQueueComplete or self.state is runQueueFailed
+
+        if build_done and self.dm_event_handler_registered:
+            bb.event.remove(self.dm_event_handler_name, None)
+            self.dm_event_handler_registered = False
+
+        if build_done and self.rqexe:
             self.teardown_workers()
             if self.rqexe.stats.failed:
                 logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed)
@@ -1326,7 +1341,7 @@ class RunQueue:
             sq_hash.append(self.rqdata.runtaskentries[tid].hash)
             sq_taskname.append(taskname)
             sq_task.append(tid)
-        locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data }
+        locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data }
         try:
             call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)"
             valid = bb.utils.better_eval(call, locs)
@@ -1427,8 +1442,8 @@ class RunQueueExecute:
         self.cfgData = rq.cfgData
         self.rqdata = rq.rqdata
 
-        self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1)
-        self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed"
+        self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS") or 1)
+        self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed"
 
         self.runq_buildable = set()
         self.runq_running = set()
@@ -1510,7 +1525,7 @@ class RunQueueExecute:
             pn = self.rqdata.dataCaches[mc].pkg_fn[fn]
             taskdata[dep] = [pn, taskname, fn]
         call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
-        locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data }
+        locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.data }
         valid = bb.utils.better_eval(call, locs)
         return valid
 
@@ -1578,7 +1593,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 invalidtasks.append(tid)
 
             call = self.rq.setsceneverify + "(covered, tasknames, fns, d, invalidtasks=invalidtasks)"
-            locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : tasknames, "fns" : fns, "d" : self.cooker.expanded_data, "invalidtasks" : invalidtasks }
+            locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : tasknames, "fns" : fns, "d" : self.cooker.data, "invalidtasks" : invalidtasks }
             covered_remove = bb.utils.better_eval(call, locs)
 
         def removecoveredtask(tid):
@@ -1630,7 +1645,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
                              if type(obj) is type and
                                 issubclass(obj, RunQueueScheduler))
 
-        user_schedulers = self.cfgData.getVar("BB_SCHEDULERS", True)
+        user_schedulers = self.cfgData.getVar("BB_SCHEDULERS")
         if user_schedulers:
             for sched in user_schedulers.split():
                 if not "." in sched:
@@ -1839,7 +1854,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
                 deps = self.rqdata.runtaskentries[revdep].depends
                 provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides]
+                taskhash = self.rqdata.runtaskentries[revdep].hash
+                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash]
                 for revdep2 in deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
@@ -2070,7 +2086,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 sq_taskname.append(taskname)
                 sq_task.append(tid)
             call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
-            locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data }
+            locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data }
             valid = bb.utils.better_eval(call, locs)
 
             valid_new = stamppresent
@@ -2197,14 +2213,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
             bb.event.fire(startevent, self.cfgData)
 
+            taskdepdata = self.build_taskdepdata(task)
+
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not self.rq.fakeworker:
                     self.rq.start_fakeworker(self)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), None)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), None)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.runq_running.add(task)
@@ -2237,6 +2255,44 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
     def runqueue_process_waitpid(self, task, status):
         RunQueueExecute.runqueue_process_waitpid(self, task, status)
 
+
+    def build_taskdepdata(self, task):
+        def getsetscenedeps(tid):
+            deps = set()
+            (mc, fn, taskname, _) = split_tid_mcfn(tid)
+            realtid = fn + ":" + taskname + "_setscene"
+            idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends
+            for (depname, idependtask) in idepends:
+                if depname not in self.rqdata.taskData[mc].build_targets:
+                    continue
+
+                depfn = self.rqdata.taskData[mc].build_targets[depname][0]
+                if depfn is None:
+                     continue
+                deptid = depfn + ":" + idependtask.replace("_setscene", "")
+                deps.add(deptid)
+            return deps
+
+        taskdepdata = {}
+        next = getsetscenedeps(task)
+        next.add(task)
+        while next:
+            additional = []
+            for revdep in next:
+                (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
+                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
+                deps = getsetscenedeps(revdep)
+                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
+                taskhash = self.rqdata.runtaskentries[revdep].hash
+                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash]
+                for revdep2 in deps:
+                    if revdep2 not in taskdepdata:
+                        additional.append(revdep2)
+            next = additional
+
+        #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n"))
+        return taskdepdata
+
 class TaskFailure(Exception):
     """
     Exception raised when a task in a runqueue fails
@@ -2402,9 +2458,9 @@ class runQueuePipe():
         self.input.close()
 
 def get_setscene_enforce_whitelist(d):
-    if d.getVar('BB_SETSCENE_ENFORCE', True) != '1':
+    if d.getVar('BB_SETSCENE_ENFORCE') != '1':
         return None
-    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST", True) or "").split()
+    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST") or "").split()
     outlist = []
     for item in whitelist[:]:
         if item.startswith('%:'):

bitbake/lib/bb/server/process.py

@@ -92,6 +92,8 @@ class ProcessServer(Process, BaseImplServer):
         self.event = EventAdapter(event_queue)
         self.featurelist = featurelist
         self.quit = False
+        self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
+        self.next_heartbeat = time.time()
 
         self.quitin, self.quitout = Pipe()
         self.event_handle = multiprocessing.Value("i")
@@ -101,6 +103,14 @@ class ProcessServer(Process, BaseImplServer):
             self.event_queue.put(event)
         self.event_handle.value = bb.event.register_UIHhandler(self, True)
 
+        heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
+        if heartbeat_event:
+            try:
+                self.heartbeat_seconds = float(heartbeat_event)
+            except:
+                # Throwing an exception here causes bitbake to hang.
+                # Just warn about the invalid setting and continue
+                bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
         bb.cooker.server_main(self.cooker, self.main)
 
     def main(self):
@@ -160,6 +170,21 @@ class ProcessServer(Process, BaseImplServer):
                 del self._idlefuns[function]
                 self.quit = True
 
+        # Create new heartbeat event?
+        now = time.time()
+        if now >= self.next_heartbeat:
+            # We might have missed heartbeats. Just trigger once in
+            # that case and continue after the usual delay.
+            self.next_heartbeat += self.heartbeat_seconds
+            if self.next_heartbeat <= now:
+                self.next_heartbeat = now + self.heartbeat_seconds
+            heartbeat = bb.event.HeartbeatEvent(now)
+            bb.event.fire(heartbeat, self.cooker.data)
+        if nextsleep and now + nextsleep > self.next_heartbeat:
+            # Shorten timeout so that we we wake up in time for
+            # the heartbeat.
+            nextsleep = self.next_heartbeat - now
+
         if nextsleep is not None:
             select.select(fds,[],[],nextsleep)
 
@@ -199,7 +224,6 @@ class BitBakeProcessServerConnection(BitBakeBaseServerConnection):
                 if isinstance(event, logging.LogRecord):
                     logger.handle(event)
 
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
         self.procserver.stop()
 
         while self.procserver.is_alive():

bitbake/lib/bb/server/xmlrpc.py

@@ -190,7 +190,7 @@ class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
         self.send_header("Content-type", "text/plain")
         self.send_header("Content-length", str(len(response)))
         self.end_headers()
-        self.wfile.write(response)
+        self.wfile.write(bytes(response, 'utf-8'))
 
 
 class XMLRPCProxyServer(BaseImplServer):

bitbake/lib/bb/siggen.py

@@ -13,7 +13,7 @@ def init(d):
     siggens = [obj for obj in globals().values()
                       if type(obj) is type and issubclass(obj, SignatureGenerator)]
 
-    desired = d.getVar("BB_SIGNATURE_HANDLER", True) or "noop"
+    desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop"
     for sg in siggens:
         if desired == sg.name:
             return sg(d)
@@ -30,6 +30,7 @@ class SignatureGenerator(object):
     name = "noop"
 
     def __init__(self, data):
+        self.basehash = {}
         self.taskhash = {}
         self.runtaskdeps = {}
         self.file_checksum_values = {}
@@ -61,11 +62,10 @@ class SignatureGenerator(object):
         return
 
     def get_taskdata(self):
-       return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints)
+        return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash)
 
     def set_taskdata(self, data):
-        self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints = data
-
+        self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data
 
 class SignatureGeneratorBasic(SignatureGenerator):
     """
@@ -82,10 +82,10 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.gendeps = {}
         self.lookupcache = {}
         self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
-        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split())
+        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
         self.taskwhitelist = None
         self.init_rundepcheck(data)
-        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True)
+        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
         if checksum_cache_file:
             self.checksum_cache = FileChecksumCache()
             self.checksum_cache.init_cache(data, checksum_cache_file)
@@ -93,7 +93,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
             self.checksum_cache = None
 
     def init_rundepcheck(self, data):
-        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None
+        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
         if self.taskwhitelist:
             self.twl = re.compile(self.taskwhitelist)
         else:
@@ -101,6 +101,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
     def _build_data(self, fn, d):
 
+        ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
         tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
 
         taskdeps = {}
@@ -133,7 +134,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 var = lookupcache[dep]
                 if var is not None:
                     data = data + str(var)
-            self.basehash[fn + "." + task] = hashlib.md5(data.encode("utf-8")).hexdigest()
+            datahash = hashlib.md5(data.encode("utf-8")).hexdigest()
+            k = fn + "." + task
+            if not ignore_mismatch and k in self.basehash and self.basehash[k] != datahash:
+                bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash))
+            self.basehash[k] = datahash
             taskdeps[task] = alldeps
 
         self.taskdeps[fn] = taskdeps
@@ -156,7 +161,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         #Slow but can be useful for debugging mismatched basehashes
         #for task in self.taskdeps[fn]:
-        #    self.dump_sigtask(fn, task, d.getVar("STAMP", True), False)
+        #    self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
 
         for task in taskdeps:
             d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
@@ -182,6 +187,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def get_taskhash(self, fn, task, deps, dataCache):
         k = fn + "." + task
         data = dataCache.basetaskhash[k]
+        self.basehash[k] = data
         self.runtaskdeps[k] = []
         self.file_checksum_values[k] = []
         recipename = dataCache.pkg_fn[fn]
@@ -278,6 +284,15 @@ class SignatureGeneratorBasic(SignatureGenerator):
             if 'nostamp:' in self.taints[k]:
                 data['taint'] = self.taints[k]
 
+        computed_basehash = calc_basehash(data)
+        if computed_basehash != self.basehash[k]:
+            bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k))
+        if runtime and k in self.taskhash:
+            computed_taskhash = calc_taskhash(data)
+            if computed_taskhash != self.taskhash[k]:
+                bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k))
+                sigfile = sigfile.replace(self.taskhash[k], computed_taskhash)
+
         fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
         try:
             with os.fdopen(fd, "wb") as stream:
@@ -292,15 +307,6 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 pass
             raise err
 
-        computed_basehash = calc_basehash(data)
-        if computed_basehash != self.basehash[k]:
-            bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k))
-        if runtime and k in self.taskhash:
-            computed_taskhash = calc_taskhash(data)
-            if computed_taskhash != self.taskhash[k]:
-                bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k))
-
-
     def dump_sigs(self, dataCaches, options):
         for fn in self.taskdeps:
             for task in self.taskdeps[fn]:
@@ -340,8 +346,8 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
 
 def dump_this_task(outfile, d):
     import bb.parse
-    fn = d.getVar("BB_FILENAME", True)
-    task = "do_" + d.getVar("BB_CURRENTTASK", True)
+    fn = d.getVar("BB_FILENAME")
+    task = "do_" + d.getVar("BB_CURRENTTASK")
     referencestamp = bb.build.stamp_internal(task, d, None, True)
     bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
 

bitbake/lib/bb/taskdata.py

@@ -89,6 +89,19 @@ class TaskData:
 
         self.add_extra_deps(fn, dataCache)
 
+        # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
+        def handle_deps(task, dep_name, depends, seen):
+            if dep_name in task_deps and task in task_deps[dep_name]:
+                ids = []
+                for dep in task_deps[dep_name][task].split():
+                    if dep:
+                        parts = dep.split(":")
+                        if len(parts) != 2:
+                            bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, dep, dep_name))
+                        ids.append((parts[0], parts[1]))
+                        seen(parts[0])
+                depends.extend(ids)
+
         for task in task_deps['tasks']:
 
             tid = "%s:%s" % (fn, task)
@@ -105,24 +118,8 @@ class TaskData:
             self.taskentries[tid].tdepends.extend(parentids)
 
             # Touch all intertask dependencies
-            if 'depends' in task_deps and task in task_deps['depends']:
-                ids = []
-                for dep in task_deps['depends'][task].split():
-                    if dep:
-                        if ":" not in dep:
-                            bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep))
-                        ids.append(((dep.split(":")[0]), dep.split(":")[1]))
-                        self.seen_build_target(dep.split(":")[0])
-                self.taskentries[tid].idepends.extend(ids)
-            if 'rdepends' in task_deps and task in task_deps['rdepends']:
-                ids = []
-                for dep in task_deps['rdepends'][task].split():
-                    if dep:
-                        if ":" not in dep:
-                            bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep))
-                        ids.append(((dep.split(":")[0]), dep.split(":")[1]))
-                        self.seen_run_target(dep.split(":")[0])
-                self.taskentries[tid].irdepends.extend(ids)
+            handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
+            handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
 
         # Work out build dependencies
         if not fn in self.depids:

bitbake/lib/bb/tests/codeparser.py

@@ -68,7 +68,7 @@ class VariableReferenceTest(ReferenceTest):
 
     def test_python_reference(self):
         self.setEmptyVars(["BAR"])
-        self.parseExpression("${@bb.data.getVar('BAR', d, True) + 'foo'}")
+        self.parseExpression("${@d.getVar('BAR') + 'foo'}")
         self.assertReferences(set(["BAR"]))
 
 class ShellReferenceTest(ReferenceTest):
@@ -209,17 +209,17 @@ be. These unit tests are testing snippets."""
         return " " + value
 
     def test_getvar_reference(self):
-        self.parseExpression("bb.data.getVar('foo', d, True)")
+        self.parseExpression("d.getVar('foo')")
         self.assertReferences(set(["foo"]))
         self.assertExecs(set())
 
     def test_getvar_computed_reference(self):
-        self.parseExpression("bb.data.getVar('f' + 'o' + 'o', d, True)")
+        self.parseExpression("d.getVar('f' + 'o' + 'o')")
         self.assertReferences(set())
         self.assertExecs(set())
 
     def test_getvar_exec_reference(self):
-        self.parseExpression("eval('bb.data.getVar(\"foo\", d, True)')")
+        self.parseExpression("eval('d.getVar(\"foo\")')")
         self.assertReferences(set())
         self.assertExecs(set(["eval"]))
 
@@ -269,11 +269,11 @@ be. These unit tests are testing snippets."""
 class DependencyReferenceTest(ReferenceTest):
 
     pydata = """
-bb.data.getVar('somevar', d, True)
+d.getVar('somevar')
 def test(d):
     foo = 'bar %s' % 'foo'
 def test2(d):
-    d.getVar(foo, True)
+    d.getVar(foo)
     d.getVar('bar', False)
     test2(d)
 
@@ -285,9 +285,9 @@ def a():
 
 test(d)
 
-bb.data.expand(bb.data.getVar("something", False, d), d)
+bb.data.expand(d.getVar("something", False), d)
 bb.data.expand("${inexpand} somethingelse", d)
-bb.data.getVar(a(), d, False)
+d.getVar(a(), False)
 """
 
     def test_python(self):

bitbake/lib/bb/tests/data.py

@@ -77,13 +77,13 @@ class DataExpansions(unittest.TestCase):
         self.assertEqual(str(val), "boo value_of_foo")
 
     def test_python_snippet_getvar(self):
-        val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}")
+        val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
         self.assertEqual(str(val), "value_of_foo value_of_bar")
 
     def test_python_unexpanded(self):
         self.d.setVar("bar", "${unsetvar}")
-        val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}")
-        self.assertEqual(str(val), "${@d.getVar('foo', True) + ' ${unsetvar}'}")
+        val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
+        self.assertEqual(str(val), "${@d.getVar('foo') + ' ${unsetvar}'}")
 
     def test_python_snippet_syntax_error(self):
         self.d.setVar("FOO", "${@foo = 5}")
@@ -99,7 +99,7 @@ class DataExpansions(unittest.TestCase):
         self.assertRaises(bb.data_smart.ExpansionError, self.d.getVar, "FOO", True)
 
     def test_value_containing_value(self):
-        val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}")
+        val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
         self.assertEqual(str(val), "value_of_foo value_of_bar")
 
     def test_reference_undefined_var(self):
@@ -109,7 +109,7 @@ class DataExpansions(unittest.TestCase):
     def test_double_reference(self):
         self.d.setVar("BAR", "bar value")
         self.d.setVar("FOO", "${BAR} foo ${BAR}")
-        val = self.d.getVar("FOO", True)
+        val = self.d.getVar("FOO")
         self.assertEqual(str(val), "bar value foo bar value")
 
     def test_direct_recursion(self):
@@ -129,12 +129,12 @@ class DataExpansions(unittest.TestCase):
 
     def test_incomplete_varexp_single_quotes(self):
         self.d.setVar("FOO", "sed -i -e 's:IP{:I${:g' $pc")
-        val = self.d.getVar("FOO", True)
+        val = self.d.getVar("FOO")
         self.assertEqual(str(val), "sed -i -e 's:IP{:I${:g' $pc")
 
     def test_nonstring(self):
         self.d.setVar("TEST", 5)
-        val = self.d.getVar("TEST", True)
+        val = self.d.getVar("TEST")
         self.assertEqual(str(val), "5")
 
     def test_rename(self):
@@ -234,19 +234,19 @@ class TestConcat(unittest.TestCase):
     def test_prepend(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.prependVar("TEST", "${FOO}:")
-        self.assertEqual(self.d.getVar("TEST", True), "foo:val")
+        self.assertEqual(self.d.getVar("TEST"), "foo:val")
 
     def test_append(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.appendVar("TEST", ":${BAR}")
-        self.assertEqual(self.d.getVar("TEST", True), "val:bar")
+        self.assertEqual(self.d.getVar("TEST"), "val:bar")
 
     def test_multiple_append(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.prependVar("TEST", "${FOO}:")
         self.d.appendVar("TEST", ":val2")
         self.d.appendVar("TEST", ":${BAR}")
-        self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar")
+        self.assertEqual(self.d.getVar("TEST"), "foo:val:val2:bar")
 
 class TestConcatOverride(unittest.TestCase):
     def setUp(self):
@@ -259,13 +259,13 @@ class TestConcatOverride(unittest.TestCase):
         self.d.setVar("TEST", "${VAL}")
         self.d.setVar("TEST_prepend", "${FOO}:")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "foo:val")
+        self.assertEqual(self.d.getVar("TEST"), "foo:val")
 
     def test_append(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.setVar("TEST_append", ":${BAR}")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "val:bar")
+        self.assertEqual(self.d.getVar("TEST"), "val:bar")
 
     def test_multiple_append(self):
         self.d.setVar("TEST", "${VAL}")
@@ -273,47 +273,47 @@ class TestConcatOverride(unittest.TestCase):
         self.d.setVar("TEST_append", ":val2")
         self.d.setVar("TEST_append", ":${BAR}")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar")
+        self.assertEqual(self.d.getVar("TEST"), "foo:val:val2:bar")
 
     def test_append_unset(self):
         self.d.setVar("TEST_prepend", "${FOO}:")
         self.d.setVar("TEST_append", ":val2")
         self.d.setVar("TEST_append", ":${BAR}")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "foo::val2:bar")
+        self.assertEqual(self.d.getVar("TEST"), "foo::val2:bar")
 
     def test_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "bar")
+        self.assertEqual(self.d.getVar("TEST"), "bar")
 
     def test_doubleref_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
         self.d.setVar("TEST_TEST", "${TEST} ${TEST}")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST_TEST", True), "bar bar")
+        self.assertEqual(self.d.getVar("TEST_TEST"), "bar bar")
 
     def test_empty_remove(self):
         self.d.setVar("TEST", "")
         self.d.setVar("TEST_remove", "val")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "")
+        self.assertEqual(self.d.getVar("TEST"), "")
 
     def test_remove_expansion(self):
         self.d.setVar("BAR", "Z")
         self.d.setVar("TEST", "${BAR}/X Y")
         self.d.setVar("TEST_remove", "${BAR}/X")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "Y")
+        self.assertEqual(self.d.getVar("TEST"), "Y")
 
     def test_remove_expansion_items(self):
         self.d.setVar("TEST", "A B C D")
         self.d.setVar("BAR", "B D")
         self.d.setVar("TEST_remove", "${BAR}")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "A C")
+        self.assertEqual(self.d.getVar("TEST"), "A C")
 
 class TestOverrides(unittest.TestCase):
     def setUp(self):
@@ -323,17 +323,17 @@ class TestOverrides(unittest.TestCase):
 
     def test_no_override(self):
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue")
 
     def test_one_override(self):
         self.d.setVar("TEST_bar", "testvalue2")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue2")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue2")
 
     def test_one_override_unset(self):
         self.d.setVar("TEST2_bar", "testvalue2")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST2", True), "testvalue2")
+        self.assertEqual(self.d.getVar("TEST2"), "testvalue2")
         self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar'])
 
     def test_multiple_override(self):
@@ -341,18 +341,18 @@ class TestOverrides(unittest.TestCase):
         self.d.setVar("TEST_local", "testvalue3")
         self.d.setVar("TEST_foo", "testvalue4")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue3")
         self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local'])
 
     def test_multiple_combined_overrides(self):
         self.d.setVar("TEST_local_foo_bar", "testvalue3")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_multiple_overrides_unset(self):
         self.d.setVar("TEST2_local_foo_bar", "testvalue3")
         bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST2", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST2"), "testvalue3")
 
     def test_keyexpansion_override(self):
         self.d.setVar("LOCAL", "local")
@@ -361,21 +361,21 @@ class TestOverrides(unittest.TestCase):
         self.d.setVar("TEST_foo", "testvalue4")
         bb.data.update_data(self.d)
         bb.data.expandKeys(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_rename_override(self):
         self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a")
         self.d.setVar("OVERRIDES", "class-target")
         bb.data.update_data(self.d)
         self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools")
-        self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools", True), "a")
+        self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools"), "a")
 
     def test_underscore_override(self):
         self.d.setVar("TEST_bar", "testvalue2")
         self.d.setVar("TEST_some_val", "testvalue3")
         self.d.setVar("TEST_foo", "testvalue4")
         self.d.setVar("OVERRIDES", "foo:bar:some_val")
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
 class TestKeyExpansion(unittest.TestCase):
     def setUp(self):
@@ -389,7 +389,7 @@ class TestKeyExpansion(unittest.TestCase):
         with LogRecord() as logs:
             bb.data.expandKeys(self.d)
             self.assertTrue(logContains("Variable key VAL_${FOO} (A) replaces original key VAL_foo (B)", logs))
-        self.assertEqual(self.d.getVar("VAL_foo", True), "A")
+        self.assertEqual(self.d.getVar("VAL_foo"), "A")
 
 class TestFlags(unittest.TestCase):
     def setUp(self):
@@ -444,3 +444,71 @@ class Contains(unittest.TestCase):
 
         self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x", True, False, self.d))
         self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x y z", True, False, self.d))
+
+
+class Serialize(unittest.TestCase):
+
+    def test_serialize(self):
+        import tempfile
+        import pickle
+        d = bb.data.init()
+        d.enableTracking()
+        d.setVar('HELLO', 'world')
+        d.setVarFlag('HELLO', 'other', 'planet')
+        with tempfile.NamedTemporaryFile(delete=False) as tmpfile:
+            tmpfilename = tmpfile.name
+            pickle.dump(d, tmpfile)
+
+        with open(tmpfilename, 'rb') as f:
+            newd = pickle.load(f)
+
+        os.remove(tmpfilename)
+
+        self.assertEqual(d, newd)
+        self.assertEqual(newd.getVar('HELLO'), 'world')
+        self.assertEqual(newd.getVarFlag('HELLO', 'other'), 'planet')
+
+
+class Remote(unittest.TestCase):
+    def test_remote(self):
+        class TestConnector:
+            d = None
+            def __init__(self, d):
+                self.d = d
+            def getVar(self, name):
+                return self.d._findVar(name)
+            def getKeys(self):
+                return self.d.localkeys()
+            def getVarHistory(self, name):
+                return self.d.varhistory.variable(name)
+            def expandPythonRef(self, varname, expr, d):
+                localdata = self.d.createCopy()
+                for key in d.localkeys():
+                    localdata.setVar(d.getVar(key))
+                varparse = bb.data_smart.VariableParse(varname, localdata)
+                return varparse.python_sub(expr)
+            def setVar(self, name, value):
+                self.d.setVar(name, value)
+
+        d1 = bb.data.init()
+        d1.enableTracking()
+        d2 = bb.data.init()
+        d2.enableTracking()
+        connector = TestConnector(d1)
+
+        d2.setVar('_remote_data', connector)
+
+        d1.setVar('HELLO', 'world')
+        d1.setVarFlag('OTHER', 'flagname', 'flagvalue')
+        self.assertEqual(d2.getVar('HELLO'), 'world')
+        self.assertEqual(d2.expand('${HELLO}'), 'world')
+        self.assertEqual(d2.expand('${@d.getVar("HELLO")}'), 'world')
+        self.assertIn('flagname', d2.getVarFlags('OTHER'))
+        self.assertEqual(d2.getVarFlag('OTHER', 'flagname'), 'flagvalue')
+        self.assertEqual(d1.varhistory.variable('HELLO'), d2.varhistory.variable('HELLO'))
+        # Test setVar on client side affects server
+        d2.setVar('HELLO', 'other-world')
+        self.assertEqual(d1.getVar('HELLO'), 'other-world')
+        # Test client side data is incorporated in python expansion (which is done on server)
+        d2.setVar('FOO', 'bar')
+        self.assertEqual(d2.expand('${@d.getVar("FOO")}'), 'bar')

bitbake/lib/bb/tests/parse.py

@@ -58,9 +58,9 @@ C = "3"
     def test_parse_simple(self):
         f = self.parsehelper(self.testfile)
         d = bb.parse.handle(f.name, self.d)['']
-        self.assertEqual(d.getVar("A", True), "1")
-        self.assertEqual(d.getVar("B", True), "2")
-        self.assertEqual(d.getVar("C", True), "3")
+        self.assertEqual(d.getVar("A"), "1")
+        self.assertEqual(d.getVar("B"), "2")
+        self.assertEqual(d.getVar("C"), "3")
 
     def test_parse_incomplete_function(self):
         testfileB = self.testfile.replace("}", "")
@@ -80,9 +80,9 @@ unset B[flag]
     def test_parse_unset(self):
         f = self.parsehelper(self.unsettest)
         d = bb.parse.handle(f.name, self.d)['']
-        self.assertEqual(d.getVar("A", True), None)
-        self.assertEqual(d.getVarFlag("A","flag", True), None)
-        self.assertEqual(d.getVar("B", True), "2")
+        self.assertEqual(d.getVar("A"), None)
+        self.assertEqual(d.getVarFlag("A","flag"), None)
+        self.assertEqual(d.getVar("B"), "2")
         
 
     overridetest = """
@@ -95,11 +95,11 @@ PN = "gtk+"
     def test_parse_overrides(self):
         f = self.parsehelper(self.overridetest)
         d = bb.parse.handle(f.name, self.d)['']
-        self.assertEqual(d.getVar("RRECOMMENDS", True), "b")
+        self.assertEqual(d.getVar("RRECOMMENDS"), "b")
         bb.data.expandKeys(d)
-        self.assertEqual(d.getVar("RRECOMMENDS", True), "b")
+        self.assertEqual(d.getVar("RRECOMMENDS"), "b")
         d.setVar("RRECOMMENDS_gtk+", "c")
-        self.assertEqual(d.getVar("RRECOMMENDS", True), "c")
+        self.assertEqual(d.getVar("RRECOMMENDS"), "c")
 
     overridetest2 = """
 EXTRA_OECONF = ""
@@ -112,7 +112,7 @@ EXTRA_OECONF_append = " c"
         d = bb.parse.handle(f.name, self.d)['']
         d.appendVar("EXTRA_OECONF", " d")
         d.setVar("OVERRIDES", "class-target")
-        self.assertEqual(d.getVar("EXTRA_OECONF", True), "b c d")
+        self.assertEqual(d.getVar("EXTRA_OECONF"), "b c d")
 
     overridetest3 = """
 DESCRIPTION = "A"
@@ -124,11 +124,11 @@ PN = "bc"
         f = self.parsehelper(self.overridetest3)
         d = bb.parse.handle(f.name, self.d)['']
         bb.data.expandKeys(d)
-        self.assertEqual(d.getVar("DESCRIPTION_bc-dev", True), "A B")
+        self.assertEqual(d.getVar("DESCRIPTION_bc-dev"), "A B")
         d.setVar("DESCRIPTION", "E")
         d.setVar("DESCRIPTION_bc-dev", "C D")
         d.setVar("OVERRIDES", "bc-dev")
-        self.assertEqual(d.getVar("DESCRIPTION", True), "C D")
+        self.assertEqual(d.getVar("DESCRIPTION"), "C D")
 
 
     classextend = """
@@ -159,6 +159,6 @@ python () {
         alldata = bb.parse.handle(f.name, self.d)
         d1 = alldata['']
         d2 = alldata[cls.name]
-        self.assertEqual(d1.getVar("VAR_var", True), "B")
-        self.assertEqual(d2.getVar("VAR_var", True), None)
+        self.assertEqual(d1.getVar("VAR_var"), "B")
+        self.assertEqual(d2.getVar("VAR_var"), None)
 

bitbake/lib/bb/tinfoil.py

@@ -1,6 +1,6 @@
 # tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
 #
-# Copyright (C) 2012 Intel Corporation
+# Copyright (C) 2012-2016 Intel Corporation
 # Copyright (C) 2011 Mentor Graphics Corporation
 #
 # This program is free software; you can redistribute it and/or modify
@@ -17,47 +17,175 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import logging
-import warnings
 import os
 import sys
+import atexit
+import re
+from collections import OrderedDict, defaultdict
 
 import bb.cache
 import bb.cooker
 import bb.providers
+import bb.taskdata
 import bb.utils
-from bb.cooker import state, BBCooker, CookerFeatures
+import bb.command
+import bb.remotedata
 from bb.cookerdata import CookerConfiguration, ConfigParameters
+from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException
 import bb.fetch2
 
+
+# We need this in order to shut down the connection to the bitbake server,
+# otherwise the process will never properly exit
+_server_connections = []
+def _terminate_connections():
+    for connection in _server_connections:
+        connection.terminate()
+atexit.register(_terminate_connections)
+
+class TinfoilUIException(Exception):
+    """Exception raised when the UI returns non-zero from its main function"""
+    def __init__(self, returncode):
+        self.returncode = returncode
+    def __repr__(self):
+        return 'UI module main returned %d' % self.returncode
+
+class TinfoilCommandFailed(Exception):
+    """Exception raised when run_command fails"""
+
+class TinfoilDataStoreConnector:
+
+    def __init__(self, tinfoil, dsindex):
+        self.tinfoil = tinfoil
+        self.dsindex = dsindex
+    def getVar(self, name):
+        value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
+        if isinstance(value, dict):
+            if '_connector_origtype' in value:
+                value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
+                del value['_connector_origtype']
+
+        return value
+    def getKeys(self):
+        return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
+    def getVarHistory(self, name):
+        return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
+    def expandPythonRef(self, varname, expr, d):
+        ds = bb.remotedata.RemoteDatastores.transmit_datastore(d)
+        ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', ds, varname, expr)
+        return ret
+    def setVar(self, varname, value):
+        if self.dsindex is None:
+            self.tinfoil.run_command('setVariable', varname, value)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+
+class TinfoilCookerAdapter:
+    """
+    Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
+    since now Tinfoil is on the client side it no longer has direct access.
+    """
+
+    class TinfoilCookerCollectionAdapter:
+        """ cooker.collection adapter """
+        def __init__(self, tinfoil):
+            self.tinfoil = tinfoil
+        def get_file_appends(self, fn):
+            return self.tinfoil.get_file_appends(fn)
+        def __getattr__(self, name):
+            if name == 'overlayed':
+                return self.tinfoil.get_overlayed_recipes()
+            elif name == 'bbappends':
+                return self.tinfoil.run_command('getAllAppends')
+            else:
+                raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
+
+    class TinfoilRecipeCacheAdapter:
+        """ cooker.recipecache adapter """
+        def __init__(self, tinfoil):
+            self.tinfoil = tinfoil
+            self._cache = {}
+
+        def get_pkg_pn_fn(self):
+            pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
+            pkg_fn = {}
+            for pn, fnlist in pkg_pn.items():
+                for fn in fnlist:
+                    pkg_fn[fn] = pn
+            self._cache['pkg_pn'] = pkg_pn
+            self._cache['pkg_fn'] = pkg_fn
+
+        def __getattr__(self, name):
+            # Grab these only when they are requested since they aren't always used
+            if name in self._cache:
+                return self._cache[name]
+            elif name == 'pkg_pn':
+                self.get_pkg_pn_fn()
+                return self._cache[name]
+            elif name == 'pkg_fn':
+                self.get_pkg_pn_fn()
+                return self._cache[name]
+            elif name == 'deps':
+                attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
+            elif name == 'rundeps':
+                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
+            elif name == 'runrecs':
+                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
+            elif name == 'pkg_pepvpr':
+                attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
+            elif name == 'inherits':
+                attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
+            elif name == 'bbfile_priority':
+                attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
+            elif name == 'pkg_dp':
+                attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
+            else:
+                raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
+
+            self._cache[name] = attrvalue
+            return attrvalue
+
+    def __init__(self, tinfoil):
+        self.tinfoil = tinfoil
+        self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
+        self.recipecaches = {}
+        # FIXME all machines
+        self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
+        self._cache = {}
+    def __getattr__(self, name):
+        # Grab these only when they are requested since they aren't always used
+        if name in self._cache:
+            return self._cache[name]
+        elif name == 'skiplist':
+            attrvalue = self.tinfoil.get_skipped_recipes()
+        elif name == 'bbfile_config_priorities':
+            ret = self.tinfoil.run_command('getLayerPriorities')
+            bbfile_config_priorities = []
+            for collection, pattern, regex, pri in ret:
+                bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
+
+            attrvalue = bbfile_config_priorities
+        else:
+            raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
+
+        self._cache[name] = attrvalue
+        return attrvalue
+
+    def findBestProvider(self, pn):
+        return self.tinfoil.find_best_provider(pn)
+
+
 class Tinfoil:
+
     def __init__(self, output=sys.stdout, tracking=False):
-        # Needed to avoid deprecation warnings with python 2.6
-        warnings.filterwarnings("ignore", category=DeprecationWarning)
-
-        # Set up logging
         self.logger = logging.getLogger('BitBake')
-        self._log_hdlr = logging.StreamHandler(output)
-        bb.msg.addDefaultlogFilter(self._log_hdlr)
-        format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-        if output.isatty():
-            format.enable_color()
-        self._log_hdlr.setFormatter(format)
-        self.logger.addHandler(self._log_hdlr)
-
-        self.config = CookerConfiguration()
-        configparams = TinfoilConfigParameters(parse_only=True)
-        self.config.setConfigParameters(configparams)
-        self.config.setServerRegIdleCallback(self.register_idle_function)
-        features = []
-        if tracking:
-            features.append(CookerFeatures.BASEDATASTORE_TRACKING)
-        self.cooker = BBCooker(self.config, features)
-        self.config_data = self.cooker.data
-        bb.providers.logger.setLevel(logging.ERROR)
-        self.cooker_data = None
-
-    def register_idle_function(self, function, data):
-        pass
+        self.config_data = None
+        self.cooker = None
+        self.tracking = tracking
+        self.ui_module = None
+        self.server_connection = None
 
     def __enter__(self):
         return self
@@ -65,30 +193,148 @@ class Tinfoil:
     def __exit__(self, type, value, traceback):
         self.shutdown()
 
-    def parseRecipes(self):
-        sys.stderr.write("Parsing recipes..")
-        self.logger.setLevel(logging.WARNING)
+    def prepare(self, config_only=False, config_params=None, quiet=0):
+        if self.tracking:
+            extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
+        else:
+            extrafeatures = []
 
-        try:
-            while self.cooker.state in (state.initial, state.parsing):
-                self.cooker.updateCache()
-        except KeyboardInterrupt:
-            self.cooker.shutdown()
-            self.cooker.updateCache()
-            sys.exit(2)
+        if not config_params:
+            config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
 
-        self.logger.setLevel(logging.INFO)
-        sys.stderr.write("done.\n")
+        cookerconfig = CookerConfiguration()
+        cookerconfig.setConfigParameters(config_params)
 
-        self.cooker_data = self.cooker.recipecaches['']
+        server, self.server_connection, ui_module = setup_bitbake(config_params,
+                            cookerconfig,
+                            extrafeatures)
 
-    def prepare(self, config_only = False):
-        if not self.cooker_data:
+        self.ui_module = ui_module
+
+        if self.server_connection:
+            _server_connections.append(self.server_connection)
             if config_only:
-                self.cooker.parseConfiguration()
-                self.cooker_data = self.cooker.recipecaches['']
+                config_params.updateToServer(self.server_connection.connection, os.environ.copy())
+                self.run_command('parseConfiguration')
             else:
-                self.parseRecipes()
+                self.run_actions(config_params)
+
+            self.config_data = bb.data.init()
+            connector = TinfoilDataStoreConnector(self, None)
+            self.config_data.setVar('_remote_data', connector)
+            self.cooker = TinfoilCookerAdapter(self)
+            self.cooker_data = self.cooker.recipecaches['']
+        else:
+            raise Exception('Failed to start bitbake server')
+
+    def run_actions(self, config_params):
+        """
+        Run the actions specified in config_params through the UI.
+        """
+        ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
+        if ret:
+            raise TinfoilUIException(ret)
+
+    def parseRecipes(self):
+        """
+        Force a parse of all recipes. Normally you should specify
+        config_only=False when calling prepare() instead of using this
+        function; this function is designed for situations where you need
+        to initialise Tinfoil and use it with config_only=True first and
+        then conditionally call this function to parse recipes later.
+        """
+        config_params = TinfoilConfigParameters(config_only=False)
+        self.run_actions(config_params)
+
+    def run_command(self, command, *params):
+        """
+        Run a command on the server (as implemented in bb.command).
+        Note that there are two types of command - synchronous and
+        asynchronous; in order to receive the results of asynchronous
+        commands you will need to set an appropriate event mask
+        using set_event_mask() and listen for the result using
+        wait_event() - with the correct event mask you'll at least get
+        bb.command.CommandCompleted and possibly other events before
+        that depending on the command.
+        """
+        if not self.server_connection:
+            raise Exception('Not connected to server (did you call .prepare()?)')
+
+        commandline = [command]
+        if params:
+            commandline.extend(params)
+        result = self.server_connection.connection.runCommand(commandline)
+        if result[1]:
+            raise TinfoilCommandFailed(result[1])
+        return result[0]
+
+    def set_event_mask(self, eventlist):
+        """Set the event mask which will be applied within wait_event()"""
+        if not self.server_connection:
+            raise Exception('Not connected to server (did you call .prepare()?)')
+        llevel, debug_domains = bb.msg.constructLogOptions()
+        ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
+        if not ret:
+            raise Exception('setEventMask failed')
+
+    def wait_event(self, timeout=0):
+        """
+        Wait for an event from the server for the specified time.
+        A timeout of 0 means don't wait if there are no events in the queue.
+        Returns the next event in the queue or None if the timeout was
+        reached. Note that in order to recieve any events you will
+        first need to set the internal event mask using set_event_mask()
+        (otherwise whatever event mask the UI set up will be in effect).
+        """
+        if not self.server_connection:
+            raise Exception('Not connected to server (did you call .prepare()?)')
+        return self.server_connection.events.waitEvent(timeout)
+
+    def get_overlayed_recipes(self):
+        return defaultdict(list, self.run_command('getOverlayedRecipes'))
+
+    def get_skipped_recipes(self):
+        return OrderedDict(self.run_command('getSkippedRecipes'))
+
+    def get_all_providers(self):
+        return defaultdict(list, self.run_command('allProviders'))
+
+    def find_providers(self):
+        return self.run_command('findProviders')
+
+    def find_best_provider(self, pn):
+        return self.run_command('findBestProvider', pn)
+
+    def get_runtime_providers(self, rdep):
+        return self.run_command('getRuntimeProviders', rdep)
+
+    def get_recipe_file(self, pn):
+        """
+        Get the file name for the specified recipe/target. Raises
+        bb.providers.NoProvider if there is no match or the recipe was
+        skipped.
+        """
+        best = self.find_best_provider(pn)
+        if not best:
+            skiplist = self.get_skipped_recipes()
+            taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
+            skipreasons = taskdata.get_reasons(pn)
+            if skipreasons:
+                raise bb.providers.NoProvider(skipreasons)
+            else:
+                raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn)
+        return best[3]
+
+    def get_file_appends(self, fn):
+        return self.run_command('getFileAppends', fn)
+
+    def parse_recipe(self, pn):
+        """
+        Parse the specified recipe and return a datastore object
+        representing the environment for the recipe.
+        """
+        fn = self.get_recipe_file(pn)
+        return self.parse_recipe_file(fn)
 
     def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
         """
@@ -107,41 +353,82 @@ class Tinfoil:
         """
         if appends and appendlist == []:
             appends = False
-        if appends:
-            if appendlist:
-                appendfiles = appendlist
-            else:
-                if not hasattr(self.cooker, 'collection'):
-                    raise Exception('You must call tinfoil.prepare() with config_only=False in order to get bbappends')
-                appendfiles = self.cooker.collection.get_file_appends(fn)
-        else:
-            appendfiles = None
         if config_data:
-            # We have to use a different function here if we're passing in a datastore
-            localdata = bb.data.createCopy(config_data)
-            envdata = bb.cache.parse_recipe(localdata, fn, appendfiles)['']
+            dctr = bb.remotedata.RemoteDatastores.transmit_datastore(config_data)
+            dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, dctr)
         else:
-            # Use the standard path
-            parser = bb.cache.NoCache(self.cooker.databuilder)
-            envdata = parser.loadDataFull(fn, appendfiles)
-        return envdata
+            dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
+        if dscon:
+            return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
+        else:
+            return None
+
+    def build_file(self, buildfile, task):
+        """
+        Runs the specified task for just a single recipe (i.e. no dependencies).
+        This is equivalent to bitbake -b, except no warning will be printed.
+        """
+        return self.run_command('buildFile', buildfile, task, True)
 
     def shutdown(self):
-        self.cooker.shutdown(force=True)
-        self.cooker.post_serve()
-        self.cooker.unlockBitbake()
-        self.logger.removeHandler(self._log_hdlr)
+        if self.server_connection:
+            self.run_command('clientComplete')
+            _server_connections.remove(self.server_connection)
+            bb.event.ui_queue = []
+            self.server_connection.terminate()
+            self.server_connection = None
 
-class TinfoilConfigParameters(ConfigParameters):
+    def _reconvert_type(self, obj, origtypename):
+        """
+        Convert an object back to the right type, in the case
+        that marshalling has changed it (especially with xmlrpc)
+        """
+        supported_types = {
+            'set': set,
+            'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
+        }
 
-    def __init__(self, **options):
+        origtype = supported_types.get(origtypename, None)
+        if origtype is None:
+            raise Exception('Unsupported type "%s"' % origtypename)
+        if type(obj) == origtype:
+            newobj = obj
+        elif isinstance(obj, dict):
+            # New style class
+            newobj = origtype()
+            for k,v in obj.items():
+                setattr(newobj, k, v)
+        else:
+            # Assume we can coerce the type
+            newobj = origtype(obj)
+
+        if isinstance(newobj, bb.command.DataStoreConnectionHandle):
+            connector = TinfoilDataStoreConnector(self, newobj.dsindex)
+            newobj = bb.data.init()
+            newobj.setVar('_remote_data', connector)
+
+        return newobj
+
+
+class TinfoilConfigParameters(BitBakeConfigParameters):
+
+    def __init__(self, config_only, **options):
         self.initial_options = options
+        # Apply some sane defaults
+        if not 'parse_only' in options:
+            self.initial_options['parse_only'] = not config_only
+        #if not 'status_only' in options:
+        #    self.initial_options['status_only'] = config_only
+        if not 'ui' in options:
+            self.initial_options['ui'] = 'knotty'
+        if not 'argv' in options:
+            self.initial_options['argv'] = []
+
         super(TinfoilConfigParameters, self).__init__()
 
-    def parseCommandLine(self, argv=sys.argv):
-        class DummyOptions:
-            def __init__(self, initial_options):
-                for key, val in initial_options.items():
-                    setattr(self, key, val)
-
-        return DummyOptions(self.initial_options), None
+    def parseCommandLine(self, argv=None):
+        # We don't want any parameters parsed from the command line
+        opts = super(TinfoilConfigParameters, self).parseCommandLine([])
+        for key, val in self.initial_options.items():
+            setattr(opts[0], key, val)
+        return opts

bitbake/lib/bb/ui/buildinfohelper.py

@@ -42,10 +42,12 @@ from orm.models import Variable, VariableHistory
 from orm.models import Package, Package_File, Target_Installed_Package, Target_File
 from orm.models import Task_Dependency, Package_Dependency
 from orm.models import Recipe_Dependency, Provides
-from orm.models import Project, CustomImagePackage, CustomImageRecipe
+from orm.models import Project, CustomImagePackage
 from orm.models import signal_runbuilds
 
 from bldcontrol.models import BuildEnvironment, BuildRequest
+from bldcontrol.models import BRLayer
+from bldcontrol import bbcontroller
 
 from bb.msg import BBLogFormatter as formatter
 from django.db import models
@@ -361,11 +363,6 @@ class ORMWrapper(object):
 
     def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information):
         if isinstance(layer_obj, Layer_Version):
-            # Special case the toaster-custom-images layer which is created
-            # on the fly so don't update the values which may cause the layer
-            # to be duplicated on a future get_or_create
-            if layer_obj.layer.name == CustomImageRecipe.LAYER_NAME:
-                return layer_obj
             # We already found our layer version for this build so just
             # update it with the new build information
             logger.debug("We found our layer from toaster")
@@ -384,8 +381,8 @@ class ORMWrapper(object):
                 local_path=layer_version_information['local_path'],
             )
 
-            logger.info("created new historical layer version %d",
-                        layer_copy.pk)
+            logger.debug("Created new layer version %s for build history",
+                         layer_copy.layer.name)
 
             self.layer_version_built.append(layer_copy)
 
@@ -441,48 +438,33 @@ class ORMWrapper(object):
         else:
             br_id, be_id = brbe.split(":")
 
-            # find layer by checkout path;
-            from bldcontrol import bbcontroller
-            bc = bbcontroller.getBuildEnvironmentController(pk = be_id)
+            # Find the layer version by matching the layer event information
+            # against the metadata we have in Toaster
 
-            # we might have a race condition here, as the project layers may change between the build trigger and the actual build execution
-            # but we can only match on the layer name, so the worst thing can happen is a mis-identification of the layer, not a total failure
+            try:
+                br_layer = BRLayer.objects.get(req=br_id,
+                                               name=layer_information['name'])
+                return br_layer.layer_version
+            except (BRLayer.MultipleObjectsReturned, BRLayer.DoesNotExist):
+                # There are multiple of the same layer name or the name
+                # hasn't been determined by the toaster.bbclass layer
+                # so let's filter by the local_path
+                bc = bbcontroller.getBuildEnvironmentController(pk=be_id)
+                for br_layer in BRLayer.objects.filter(req=br_id):
+                    if br_layer.giturl and \
+                       layer_information['local_path'].endswith(
+                           bc.getGitCloneDirectory(br_layer.giturl,
+                                                   br_layer.commit)):
+                            return br_layer.layer_version
 
-            # note that this is different
-            buildrequest = BuildRequest.objects.get(pk = br_id)
-            for brl in buildrequest.brlayer_set.all():
-                if brl.local_source_dir:
-                    localdirname = os.path.join(brl.local_source_dir,
-                                                brl.dirpath)
-                else:
-                    localdirname = os.path.join(bc.getGitCloneDirectory(brl.giturl, brl.commit), brl.dirpath)
-                # we get a relative path, unless running in HEAD mode where the path is absolute
-                if not localdirname.startswith("/"):
-                    localdirname = os.path.join(bc.be.sourcedir, localdirname)
-                #logger.debug(1, "Localdirname %s lcal_path %s" % (localdirname, layer_information['local_path']))
-                if localdirname.startswith(layer_information['local_path']):
-                  # If the build request came from toaster this field
-                  # should contain the information from the layer_version
-                  # That created this build request.
-                    if brl.layer_version:
-                        return brl.layer_version
-
-                # This might be a local layer (i.e. no git info) so try
-                # matching local_source_dir
-                if brl.local_source_dir and brl.local_source_dir == layer_information["local_path"]:
-                    return brl.layer_version
-
-                    # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build()
-                    #logger.debug(1, "Matched %s to BRlayer %s" % (pformat(layer_information["local_path"]), localdirname))
-
-                    for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name):
-                        if pl.layercommit.layer.vcs_url == brl.giturl :
-                            layer = pl.layercommit.layer
-                            layer.save()
-                            return layer
-
-            raise NotExisting("Unidentified layer %s" % pformat(layer_information))
+                    if br_layer.local_source_dir == \
+                            layer_information['local_path']:
+                        return br_layer.layer_version
 
+        # We've reached the end of our search and couldn't find the layer
+        # we can continue but some data may be missing
+        raise NotExisting("Unidentified layer %s" %
+                          pformat(layer_information))
 
     def save_target_file_information(self, build_obj, target_obj, filedata):
         assert isinstance(build_obj, Build)
@@ -982,6 +964,39 @@ class BuildInfoHelper(object):
             pass
         return task_information
 
+    def _get_layer_version_for_dependency(self, pathRE):
+        """ Returns the layer in the toaster db that has a full regex
+        match to the pathRE. pathRE - the layer path passed as a regex in the
+        event. It is created in cooker.py as a collection for the layer
+        priorities.
+        """
+        self._ensure_build()
+
+        def _sort_longest_path(layer_version):
+            assert isinstance(layer_version, Layer_Version)
+            return len(layer_version.local_path)
+
+        # Our paths don't append a trailing slash
+        if pathRE.endswith("/"):
+            pathRE = pathRE[:-1]
+
+        p = re.compile(pathRE)
+        # Heuristics: we always match recipe to the deepest layer path in
+        # the discovered layers
+        for lvo in sorted(self.orm_wrapper.layer_version_objects,
+                          reverse=True, key=_sort_longest_path):
+            if p.fullmatch(os.path.abspath(lvo.local_path)):
+                return lvo
+            if lvo.layer.local_source_dir:
+                if p.fullmatch(os.path.abspath(lvo.layer.local_source_dir)):
+                    return lvo
+
+        # if we get here, we didn't read layers correctly;
+        # dump whatever information we have on the error log
+        logger.warning("Could not match layer dependency for path %s : %s",
+                       pathRE,
+                       self.orm_wrapper.layer_version_objects)
+
     def _get_layer_version_for_path(self, path):
         self._ensure_build()
 
@@ -1372,7 +1387,7 @@ class BuildInfoHelper(object):
         if 'layer-priorities' in event._depgraph.keys():
             for lv in event._depgraph['layer-priorities']:
                 (_, path, _, priority) = lv
-                layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^
+                layer_version_obj = self._get_layer_version_for_dependency(path)
                 assert layer_version_obj is not None
                 layer_version_obj.priority = priority
                 layer_version_obj.save()

bitbake/lib/bb/ui/depexp.py

@@ -313,7 +313,7 @@ def main(server, eventHandler, params):
                         extra = ". Close matches:\n  %s" % '\n  '.join(event._close_matches)
 
                 if event._dependees:
-                    print("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s" % r, event._item, ", ".join(event._dependees), r, extra)
+                    print("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s" % (r, event._item, ", ".join(event._dependees), r, extra))
                 else:
                     print("Nothing %sPROVIDES '%s'%s" % (r, event._item, extra))
                 if event._reasons:

bitbake/lib/bb/ui/knotty.py

@@ -284,7 +284,7 @@ class TerminalFilter(object):
             content = self.main_progress.update(progress)
             print('')
         lines = 1 + int(len(content) / (self.columns + 1))
-        if not self.quiet:
+        if self.quiet == 0:
             for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
                 if isinstance(task, tuple):
                     pbar, progress, rate, start_time = task
@@ -312,7 +312,7 @@ class TerminalFilter(object):
             fd = sys.stdin.fileno()
             self.termios.tcsetattr(fd, self.termios.TCSADRAIN, self.stdinbackup)
 
-def _log_settings_from_server(server):
+def _log_settings_from_server(server, observe_only):
     # Get values of variables which control our output
     includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
     if error:
@@ -322,7 +322,11 @@ def _log_settings_from_server(server):
     if error:
         logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error)
         raise BaseException(error)
-    consolelogfile, error = server.runCommand(["getSetVariable", "BB_CONSOLELOG"])
+    if observe_only:
+        cmd = 'getVariable'
+    else:
+        cmd = 'getSetVariable'
+    consolelogfile, error = server.runCommand([cmd, "BB_CONSOLELOG"])
     if error:
         logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error)
         raise BaseException(error)
@@ -340,7 +344,7 @@ _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.Lo
 
 def main(server, eventHandler, params, tf = TerminalFilter):
 
-    includelogs, loglines, consolelogfile = _log_settings_from_server(server)
+    includelogs, loglines, consolelogfile = _log_settings_from_server(server, params.observe_only)
 
     if sys.stdin.isatty() and sys.stdout.isatty():
         log_exec_tty = True
@@ -353,10 +357,13 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     errconsole = logging.StreamHandler(sys.stderr)
     format_str = "%(levelname)s: %(message)s"
     format = bb.msg.BBLogFormatter(format_str)
-    if params.options.quiet:
-        bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, bb.msg.BBLogFormatter.WARNING)
+    if params.options.quiet == 0:
+        forcelevel = None
+    elif params.options.quiet > 2:
+        forcelevel = bb.msg.BBLogFormatter.ERROR
     else:
-        bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut)
+        forcelevel = bb.msg.BBLogFormatter.WARNING
+    bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, forcelevel)
     bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr)
     console.setFormatter(format)
     errconsole.setFormatter(format)
@@ -506,35 +513,47 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 logger.info(event._message)
                 continue
             if isinstance(event, bb.event.ParseStarted):
+                if params.options.quiet > 1:
+                    continue
                 if event.total == 0:
                     continue
                 parseprogress = new_progress("Parsing recipes", event.total).start()
                 continue
             if isinstance(event, bb.event.ParseProgress):
+                if params.options.quiet > 1:
+                    continue
                 if parseprogress:
                     parseprogress.update(event.current)
                 else:
                     bb.warn("Got ParseProgress event for parsing that never started?")
                 continue
             if isinstance(event, bb.event.ParseCompleted):
+                if params.options.quiet > 1:
+                    continue
                 if not parseprogress:
                     continue
                 parseprogress.finish()
                 pasreprogress = None
-                if not params.options.quiet:
+                if params.options.quiet == 0:
                     print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
                         % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
                 continue
 
             if isinstance(event, bb.event.CacheLoadStarted):
+                if params.options.quiet > 1:
+                    continue
                 cacheprogress = new_progress("Loading cache", event.total).start()
                 continue
             if isinstance(event, bb.event.CacheLoadProgress):
+                if params.options.quiet > 1:
+                    continue
                 cacheprogress.update(event.current)
                 continue
             if isinstance(event, bb.event.CacheLoadCompleted):
+                if params.options.quiet > 1:
+                    continue
                 cacheprogress.finish()
-                if not params.options.quiet:
+                if params.options.quiet == 0:
                     print("Loaded %d entries from dependency cache." % event.num_entries)
                 continue
 
@@ -620,16 +639,22 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 continue
 
             if isinstance(event, bb.event.ProcessStarted):
+                if params.options.quiet > 1:
+                    continue
                 parseprogress = new_progress(event.processname, event.total)
                 parseprogress.start(False)
                 continue
             if isinstance(event, bb.event.ProcessProgress):
+                if params.options.quiet > 1:
+                    continue
                 if parseprogress:
                     parseprogress.update(event.progress)
                 else:
                     bb.warn("Got ProcessProgress event for someting that never started?")
                 continue
             if isinstance(event, bb.event.ProcessFinished):
+                if params.options.quiet > 1:
+                    continue
                 if parseprogress:
                     parseprogress.finish()
                 parseprogress = None
@@ -647,6 +672,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                                   bb.event.OperationCompleted,
                                   bb.event.OperationProgress,
                                   bb.event.DiskFull,
+                                  bb.event.HeartbeatEvent,
                                   bb.build.TaskProgress)):
                 continue
 
@@ -700,7 +726,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if return_value and errors:
             summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
                                  "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
-        if summary and not params.options.quiet:
+        if summary and params.options.quiet == 0:
             print(summary)
 
         if interrupted:

bitbake/lib/bb/ui/toasterui.py

@@ -168,6 +168,9 @@ def main(server, eventHandler, params):
         logger.warning("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.")
         build_history_enabled = False
 
+    if not "buildstats" in inheritlist.split(" "):
+        logger.warning("buildstats is not enabled. Please enable INHERIT += \"buildstats\" to generate build statistics.")
+
     if not params.observe_only:
         params.updateFromServer(server)
         params.updateToServer(server, os.environ.copy())
@@ -233,6 +236,9 @@ def main(server, eventHandler, params):
             # pylint: disable=protected-access
             # the code will look into the protected variables of the event; no easy way around this
 
+            if isinstance(event, bb.event.HeartbeatEvent):
+                continue
+
             if isinstance(event, bb.event.ParseStarted):
                 if not (build_log and build_log_file_path):
                     build_log, build_log_file_path = _open_build_log(log_dir)

bitbake/lib/bb/utils.py

@@ -378,7 +378,7 @@ def _print_exception(t, value, tb, realfile, text, context):
 
         # If the exception is from spwaning a task, let's be helpful and display
         # the output (which hopefully includes stderr).
-        if isinstance(value, subprocess.CalledProcessError):
+        if isinstance(value, subprocess.CalledProcessError) and value.output:
             error.append("Subprocess output:")
             error.append(value.output.decode("utf-8", errors="ignore"))
     finally:
@@ -665,7 +665,7 @@ def build_environment(d):
     for var in bb.data.keys(d):
         export = d.getVarFlag(var, "export", False)
         if export:
-            os.environ[var] = d.getVar(var, True) or ""
+            os.environ[var] = d.getVar(var) or ""
 
 def _check_unsafe_delete_path(path):
     """
@@ -953,7 +953,7 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
     Arguments:
 
     variable -- the variable name. This will be fetched and expanded (using
-    d.getVar(variable, True)) and then split into a set().
+    d.getVar(variable)) and then split into a set().
 
     checkvalues -- if this is a string it is split on whitespace into a set(),
     otherwise coerced directly into a set().
@@ -966,7 +966,7 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
     d -- the data store.
     """
 
-    val = d.getVar(variable, True)
+    val = d.getVar(variable)
     if not val:
         return falsevalue
     val = set(val.split())
@@ -979,7 +979,7 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
     return falsevalue
 
 def contains_any(variable, checkvalues, truevalue, falsevalue, d):
-    val = d.getVar(variable, True)
+    val = d.getVar(variable)
     if not val:
         return falsevalue
     val = set(val.split())
@@ -1378,10 +1378,10 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
 
 def get_file_layer(filename, d):
     """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
-    collections = (d.getVar('BBFILE_COLLECTIONS', True) or '').split()
+    collections = (d.getVar('BBFILE_COLLECTIONS') or '').split()
     collection_res = {}
     for collection in collections:
-        collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection, True) or ''
+        collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection) or ''
 
     def path_to_layer(path):
         # Use longest path so we handle nested layers
@@ -1394,7 +1394,7 @@ def get_file_layer(filename, d):
         return match
 
     result = None
-    bbfiles = (d.getVar('BBFILES', True) or '').split()
+    bbfiles = (d.getVar('BBFILES') or '').split()
     bbfilesmatch = False
     for bbfilesentry in bbfiles:
         if fnmatch.fnmatch(filename, bbfilesentry):
@@ -1471,7 +1471,7 @@ def export_proxies(d):
         if v in os.environ.keys():
             exported = True
         else:
-            v_proxy = d.getVar(v, True)
+            v_proxy = d.getVar(v)
             if v_proxy is not None:
                 os.environ[v] = v_proxy
                 exported = True
@@ -1503,3 +1503,14 @@ def load_plugins(logger, plugins, pluginpath):
                 plugins.append(obj or plugin)
             else:
                 plugins.append(plugin)
+
+
+class LogCatcher(logging.Handler):
+    """Logging handler for collecting logged messages so you can check them later"""
+    def __init__(self):
+        self.messages = []
+        logging.Handler.__init__(self, logging.WARNING)
+    def emit(self, record):
+        self.messages.append(bb.build.logformatter.format(record))
+    def contains(self, message):
+        return (message in self.messages)

bitbake/lib/bblayers/action.py

@@ -180,7 +180,7 @@ build results (as the layer priority order has effectively changed).
 
         if first_regex:
             # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
-            bbfiles = str(self.tinfoil.config_data.getVar('BBFILES', True)).split()
+            bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split()
             bbfiles_layer = []
             for item in bbfiles:
                 if first_regex.match(item):

bitbake/lib/bblayers/common.py

@@ -12,7 +12,7 @@ class LayerPlugin():
 
     def tinfoil_init(self, tinfoil):
         self.tinfoil = tinfoil
-        self.bblayers = (self.tinfoil.config_data.getVar('BBLAYERS', True) or "").split()
+        self.bblayers = (self.tinfoil.config_data.getVar('BBLAYERS') or "").split()
         layerconfs = self.tinfoil.config_data.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', self.tinfoil.config_data)
         self.bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.items()}
 

bitbake/lib/bblayers/layerindex.py

@@ -151,7 +151,7 @@ class LayerIndexPlugin(ActionPlugin):
     def do_layerindex_fetch(self, args):
         """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
 """
-        apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL', True)
+        apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL')
         if not apiurl:
             logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
             return 1
@@ -173,8 +173,8 @@ class LayerIndexPlugin(ActionPlugin):
             return 1
 
         ignore_layers = []
-        for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS', True).split():
-            lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection, True)
+        for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS').split():
+            lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection)
             if lname:
                 ignore_layers.append(lname)
 
@@ -225,7 +225,7 @@ class LayerIndexPlugin(ActionPlugin):
             printedlayers.append(dependency)
 
         if repourls:
-            fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR', True)
+            fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
             if not fetchdir:
                 logger.error("Cannot get BBLAYERS_FETCH_DIR")
                 return 1

bitbake/lib/bblayers/query.py

@@ -5,8 +5,6 @@ import sys
 import os
 import re
 
-import bb.cache
-import bb.providers
 import bb.utils
 
 from bblayers.common import LayerPlugin
@@ -62,7 +60,7 @@ are overlayed will also be listed, with a " (skipped)" suffix.
         # factor - however, each layer.conf is free to either prepend or append to
         # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
         # not be exactly the order present in bblayers.conf either.
-        bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
+        bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
         overlayed_class_found = False
         for (classfile, classdirs) in classes.items():
             if len(classdirs) > 1:
@@ -114,7 +112,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
 
     def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only, inherits):
         if inherits:
-            bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
+            bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
             for classname in inherits:
                 classfile = 'classes/%s.bbclass' % classname
                 if not bb.utils.which(bbpath, classfile, history=False):
@@ -122,15 +120,13 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                     sys.exit(1)
 
         pkg_pn = self.tinfoil.cooker.recipecaches[''].pkg_pn
-        (latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecaches[''], pkg_pn)
-        allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecaches[''])
+        (latest_versions, preferred_versions) = self.tinfoil.find_providers()
+        allproviders = self.tinfoil.get_all_providers()
 
         # Ensure we list skipped recipes
         # We are largely guessing about PN, PV and the preferred version here,
         # but we have no choice since skipped recipes are not fully parsed
         skiplist = list(self.tinfoil.cooker.skiplist.keys())
-        skiplist.sort( key=lambda fileitem: self.tinfoil.cooker.collection.calc_bbfile_priority(fileitem) )
-        skiplist.reverse()
         for fn in skiplist:
             recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
             p = recipe_parts[0]
@@ -158,7 +154,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                     logger.plain("%s:", pn)
                 logger.plain("  %s %s%s", layer.ljust(20), ver, skipped)
 
-        global_inherit = (self.tinfoil.config_data.getVar('INHERIT', True) or "").split()
+        global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split()
         cls_re = re.compile('classes/')
 
         preffiles = []
@@ -265,10 +261,7 @@ Lists recipes with the bbappends that apply to them as subitems.
     def show_appends_for_pn(self, pn):
         filenames = self.tinfoil.cooker_data.pkg_pn[pn]
 
-        best = bb.providers.findBestProvider(pn,
-                                             self.tinfoil.config_data,
-                                             self.tinfoil.cooker_data,
-                                             self.tinfoil.cooker_data.pkg_pn)
+        best = self.tinfoil.find_best_provider(pn)
         best_filename = os.path.basename(best[3])
 
         return self.show_appends_output(filenames, best_filename)
@@ -319,12 +312,12 @@ NOTE: .bbappend files can impact the dependencies.
         ignore_layers = (args.ignore or '').split(',')
 
         pkg_fn = self.tinfoil.cooker_data.pkg_fn
-        bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
+        bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
         self.require_re = re.compile(r"require\s+(.+)")
         self.include_re = re.compile(r"include\s+(.+)")
         self.inherit_re = re.compile(r"inherit\s+(.+)")
 
-        global_inherit = (self.tinfoil.config_data.getVar('INHERIT', True) or "").split()
+        global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split()
 
         # The bb's DEPENDS and RDEPENDS
         for f in pkg_fn:
@@ -336,10 +329,7 @@ NOTE: .bbappend files can impact the dependencies.
             deps = self.tinfoil.cooker_data.deps[f]
             for pn in deps:
                 if pn in self.tinfoil.cooker_data.pkg_pn:
-                    best = bb.providers.findBestProvider(pn,
-                            self.tinfoil.config_data,
-                            self.tinfoil.cooker_data,
-                            self.tinfoil.cooker_data.pkg_pn)
+                    best = self.tinfoil.find_best_provider(pn)
                     self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers)
 
             # The RDPENDS
@@ -352,14 +342,11 @@ NOTE: .bbappend files can impact the dependencies.
                     sorted_rdeps[k2] = 1
             all_rdeps = sorted_rdeps.keys()
             for rdep in all_rdeps:
-                all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rdep)
+                all_p, best = self.tinfoil.get_runtime_providers(rdep)
                 if all_p:
                     if f in all_p:
                         # The recipe provides this one itself, ignore
                         continue
-                    best = bb.providers.filterProvidersRunTime(all_p, rdep,
-                                    self.tinfoil.config_data,
-                                    self.tinfoil.cooker_data)[0][0]
                     self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers)
 
             # The RRECOMMENDS
@@ -372,14 +359,11 @@ NOTE: .bbappend files can impact the dependencies.
                     sorted_rrecs[k2] = 1
             all_rrecs = sorted_rrecs.keys()
             for rrec in all_rrecs:
-                all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rrec)
+                all_p, best = self.tinfoil.get_runtime_providers(rrec)
                 if all_p:
                     if f in all_p:
                         # The recipe provides this one itself, ignore
                         continue
-                    best = bb.providers.filterProvidersRunTime(all_p, rrec,
-                                    self.tinfoil.config_data,
-                                    self.tinfoil.cooker_data)[0][0]
                     self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
 
             # The inherit class

bitbake/lib/bs4/builder/_html5lib.py

@@ -15,7 +15,14 @@ from bs4.element import (
     whitespace_re,
 )
 import html5lib
+try:
+    # html5lib >= 0.99999999/1.0b9
+    from html5lib.treebuilders import base as treebuildersbase
+except ImportError:
+    # html5lib <= 0.9999999/1.0b8
+    from html5lib.treebuilders import _base as treebuildersbase
 from html5lib.constants import namespaces
+
 from bs4.element import (
     Comment,
     Doctype,
@@ -67,7 +74,7 @@ class HTML5TreeBuilder(HTMLTreeBuilder):
         return '<html><head></head><body>%s</body></html>' % fragment
 
 
-class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
+class TreeBuilderForHtml5lib(treebuildersbase.TreeBuilder):
 
     def __init__(self, soup, namespaceHTMLElements):
         self.soup = soup
@@ -105,7 +112,7 @@ class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
         return self.soup
 
     def getFragment(self):
-        return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
+        return treebuildersbase.TreeBuilder.getFragment(self).element
 
 class AttrList(object):
     def __init__(self, element):
@@ -137,9 +144,9 @@ class AttrList(object):
         return name in list(self.attrs.keys())
 
 
-class Element(html5lib.treebuilders._base.Node):
+class Element(treebuildersbase.Node):
     def __init__(self, element, soup, namespace):
-        html5lib.treebuilders._base.Node.__init__(self, element.name)
+        treebuildersbase.Node.__init__(self, element.name)
         self.element = element
         self.soup = soup
         self.namespace = namespace
@@ -324,7 +331,7 @@ class Element(html5lib.treebuilders._base.Node):
 
 class TextNode(Element):
     def __init__(self, element, soup):
-        html5lib.treebuilders._base.Node.__init__(self, None)
+        treebuildersbase.Node.__init__(self, None)
         self.element = element
         self.soup = soup
 

bitbake/lib/prserv/serv.py

@@ -420,7 +420,7 @@ class PRServiceConfigError(Exception):
 def auto_start(d):
     global singleton
 
-    host_params = list(filter(None, (d.getVar('PRSERV_HOST', True) or '').split(':')))
+    host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':')))
     if not host_params:
         return None
 
@@ -431,7 +431,7 @@ def auto_start(d):
 
     if is_local_special(host_params[0], int(host_params[1])) and not singleton:
         import bb.utils
-        cachedir = (d.getVar("PERSISTENT_DIR", True) or d.getVar("CACHE", True))
+        cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE"))
         if not cachedir:
             logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
             raise PRServiceConfigError

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -27,7 +27,7 @@ import shutil
 from django.db import transaction
 from django.db.models import Q
 from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
-from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer
+from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting
 import subprocess
 
 from toastermain import settings
@@ -201,71 +201,88 @@ class LocalhostBEController(BuildEnvironmentController):
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
         # 5. create custom layer and add custom recipes to it
-        layerpath = os.path.join(self.be.builddir,
-                                 CustomImageRecipe.LAYER_NAME)
         for target in targets:
             try:
-                customrecipe = CustomImageRecipe.objects.get(name=target.target,
-                                                             project=bitbake.req.project)
+                customrecipe = CustomImageRecipe.objects.get(
+                    name=target.target,
+                    project=bitbake.req.project)
+
+                custom_layer_path = self.setup_custom_image_recipe(
+                    customrecipe, layers)
+
+                if os.path.isdir(custom_layer_path):
+                    layerlist.append(custom_layer_path)
+
             except CustomImageRecipe.DoesNotExist:
-                continue # not a custom recipe, skip
+                continue  # not a custom recipe, skip
 
-            # create directory structure
-            for name in ("conf", "recipes"):
-                path = os.path.join(layerpath, name)
-                if not os.path.isdir(path):
-                    os.makedirs(path)
-
-            # create layer.oonf
-            config = os.path.join(layerpath, "conf", "layer.conf")
-            if not os.path.isfile(config):
-                with open(config, "w") as conf:
-                    conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
-
-            # Update the Layer_Version dirpath that has our base_recipe in
-            # to be able to read the base recipe to then  generate the
-            # custom recipe.
-            br_layer_base_recipe = layers.get(
-                layer_version=customrecipe.base_recipe.layer_version)
-
-            br_layer_base_dirpath = \
-                    os.path.join(self.be.sourcedir,
-                                 self.getGitCloneDirectory(
-                                     br_layer_base_recipe.giturl,
-                                     br_layer_base_recipe.commit),
-                                 customrecipe.base_recipe.layer_version.dirpath
-                                )
-
-            customrecipe.base_recipe.layer_version.dirpath = \
-                         br_layer_base_dirpath
-
-            customrecipe.base_recipe.layer_version.save()
-
-            # create recipe
-            recipe_path = \
-                    os.path.join(layerpath, "recipes", "%s.bb" % target.target)
-            with open(recipe_path, "w") as recipef:
-                recipef.write(customrecipe.generate_recipe_file_contents())
-
-            # Update the layer and recipe objects
-            customrecipe.layer_version.dirpath = layerpath
-            customrecipe.layer_version.save()
-
-            customrecipe.file_path = recipe_path
-            customrecipe.save()
-
-            # create *Layer* objects needed for build machinery to work
-            BRLayer.objects.get_or_create(req=target.req,
-                                          name=layer.name,
-                                          dirpath=layerpath,
-                                          giturl="file://%s" % layerpath)
-        if os.path.isdir(layerpath):
-            layerlist.append(layerpath)
-
-        self.islayerset = True
         layerlist.extend(nongitlayerlist)
+        logger.debug("\n\nset layers gives this list \n %s" % ''.join(layerlist))
+        self.islayerset = True
         return layerlist
 
+    def setup_custom_image_recipe(self, customrecipe, layers):
+        """ Set up toaster-custom-images layer and recipe files """
+        layerpath = os.path.join(self.be.builddir,
+                                 CustomImageRecipe.LAYER_NAME)
+
+        # create directory structure
+        for name in ("conf", "recipes"):
+            path = os.path.join(layerpath, name)
+            if not os.path.isdir(path):
+                os.makedirs(path)
+
+        # create layer.conf
+        config = os.path.join(layerpath, "conf", "layer.conf")
+        if not os.path.isfile(config):
+            with open(config, "w") as conf:
+                conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
+
+        # Update the Layer_Version dirpath that has our base_recipe in
+        # to be able to read the base recipe to then  generate the
+        # custom recipe.
+        br_layer_base_recipe = layers.get(
+            layer_version=customrecipe.base_recipe.layer_version)
+
+        # If the layer is one that we've cloned we know where it lives
+        if br_layer_base_recipe.giturl and br_layer_base_recipe.commit:
+            layer_path = self.getGitCloneDirectory(
+                br_layer_base_recipe.giturl,
+                br_layer_base_recipe.commit)
+        # Otherwise it's a local layer
+        elif br_layer_base_recipe.local_source_dir:
+            layer_path = br_layer_base_recipe.local_source_dir
+        else:
+            logger.error("Unable to workout the dir path for the custom"
+                         " image recipe")
+
+        br_layer_base_dirpath = os.path.join(
+            self.be.sourcedir,
+            layer_path,
+            customrecipe.base_recipe.layer_version.dirpath)
+
+        customrecipe.base_recipe.layer_version.dirpath = br_layer_base_dirpath
+
+        customrecipe.base_recipe.layer_version.save()
+
+        # create recipe
+        recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
+                                   customrecipe.name)
+        with open(recipe_path, "w") as recipef:
+            recipef.write(customrecipe.generate_recipe_file_contents())
+
+        # Update the layer and recipe objects
+        customrecipe.layer_version.dirpath = layerpath
+        customrecipe.layer_version.layer.local_source_dir = layerpath
+        customrecipe.layer_version.layer.save()
+        customrecipe.layer_version.save()
+
+        customrecipe.file_path = recipe_path
+        customrecipe.save()
+
+        return layerpath
+
+
     def readServerLogFile(self):
         return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
 
@@ -277,23 +294,17 @@ class LocalhostBEController(BuildEnvironmentController):
         builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
         oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
         # init build environment
-        self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
+        try:
+            custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
+            custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
+            self._shellcmd("bash -c 'source %s'" % (custom_script))
+        except ToasterSetting.DoesNotExist:
+            self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
                        self.be.sourcedir)
 
         # update bblayers.conf
-        bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
-        conflines = open(bblconfpath, "r").readlines()
-        skip = False
+        bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
         with open(bblconfpath, 'w') as bblayers:
-            for line in conflines:
-                if line.startswith("# line added by toaster"):
-                    skip = True
-                    continue
-                if skip:
-                    skip = False
-                else:
-                    bblayers.write(line)
-
             bblayers.write('# line added by toaster build control\n'
                            'BBLAYERS = "%s"' % ' '.join(layers))
 
@@ -306,9 +317,10 @@ class LocalhostBEController(BuildEnvironmentController):
 
         # run bitbake server from the clone
         bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
-        self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s '
+        toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
+        self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
                        '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init,
-                       builddir, bitbake, confpath), self.be.sourcedir)
+                       builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
 
         # read port number from bitbake.lock
         self.be.bbport = ""

bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py

@@ -11,9 +11,11 @@ from orm.models import Build, LogMessage, Target
 import logging
 import traceback
 import signal
+import os
 
 logger = logging.getLogger("toaster")
 
+
 class Command(NoArgsCommand):
     args = ""
     help = "Schedules and executes build requests as possible. "\
@@ -50,7 +52,7 @@ class Command(NoArgsCommand):
                 logger.debug("runbuilds: No build env")
                 return
 
-            logger.info("runbuilds: starting build %s, environment %s" % \
+            logger.info("runbuilds: starting build %s, environment %s" %
                         (br, bec.be))
 
             # let the build request know where it is being executed
@@ -80,7 +82,7 @@ class Command(NoArgsCommand):
 
     def archive(self):
         for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
-            if br.build == None:
+            if br.build is None:
                 br.state = BuildRequest.REQ_FAILED
             else:
                 br.state = BuildRequest.REQ_COMPLETED
@@ -99,10 +101,10 @@ class Command(NoArgsCommand):
             Q(updated__lt=timezone.now() - timedelta(seconds=30))
         ).update(lock=BuildEnvironment.LOCK_FREE)
 
-
         # update all Builds that were in progress and failed to start
-        for br in BuildRequest.objects.filter(state=BuildRequest.REQ_FAILED,
-                                              build__outcome=Build.IN_PROGRESS):
+        for br in BuildRequest.objects.filter(
+                state=BuildRequest.REQ_FAILED,
+                build__outcome=Build.IN_PROGRESS):
             # transpose the launch errors in ToasterExceptions
             br.build.outcome = Build.FAILED
             for brerror in br.brerror_set.all():
@@ -117,7 +119,6 @@ class Command(NoArgsCommand):
             br.environment.lock = BuildEnvironment.LOCK_FREE
             br.environment.save()
 
-
         # update all BuildRequests without a build created
         for br in BuildRequest.objects.filter(build=None):
             br.build = Build.objects.create(project=br.project,
@@ -144,7 +145,7 @@ class Command(NoArgsCommand):
 
         # Make sure the LOCK is removed for builds which have been fully
         # cancelled
-        for br in BuildRequest.objects.filter(\
+        for br in BuildRequest.objects.filter(
                       Q(build__outcome=Build.CANCELLED) &
                       Q(state=BuildRequest.REQ_CANCELLING) &
                       ~Q(environment=None)):
@@ -168,6 +169,12 @@ class Command(NoArgsCommand):
             logger.warn("runbuilds: schedule exception %s" % str(e))
 
     def handle_noargs(self, **options):
+        pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
+                                    ".runbuilds.pid")
+
+        with open(pidfile_path, 'w') as pidfile:
+            pidfile.write("%s" % os.getpid())
+
         self.runbuild()
 
         signal.signal(signal.SIGUSR1, lambda sig, frame: None)

bitbake/lib/toaster/bldcontrol/tests.py

@@ -1,141 +0,0 @@
-"""
-This file demonstrates writing tests using the unittest module. These will pass
-when you run "manage.py test".
-
-Replace this with more appropriate tests for your application.
-"""
-
-from django.test import TestCase
-
-from bldcontrol.bbcontroller import BitbakeController, BuildSetupException
-from bldcontrol.localhostbecontroller import LocalhostBEController
-from bldcontrol.models import BuildEnvironment, BuildRequest
-from bldcontrol.management.commands.runbuilds import Command
-
-import socket
-import subprocess
-import os
-
-# standard poky data hardcoded for testing
-BITBAKE_LAYER = type('bitbake_info', (object,), { "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "", "commit": "HEAD"})
-POKY_LAYERS = [
-    type('poky_info', (object,), { "name": "meta", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta", "commit": "HEAD"}),
-    type('poky_info', (object,), { "name": "meta-yocto", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto", "commit": "HEAD"}),
-    type('poky_info', (object,), { "name": "meta-yocto-bsp", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto-bsp", "commit": "HEAD"}),
-    ]
-
-
-
-# we have an abstract test class designed to ensure that the controllers use a single interface
-# specific controller tests only need to override the _getBuildEnvironment() method
-
-test_sourcedir = os.getenv("TTS_SOURCE_DIR")
-test_builddir = os.getenv("TTS_BUILD_DIR")
-test_address = os.getenv("TTS_TEST_ADDRESS", "localhost")
-
-if test_sourcedir == None or test_builddir == None or test_address == None:
-    raise Exception("Please set TTTS_SOURCE_DIR, TTS_BUILD_DIR and TTS_TEST_ADDRESS")
-
-# The bb server will expect a toaster-pre.conf file to exist. If it doesn't exit then we make
-# an empty one here.
-open(test_builddir + 'conf/toaster-pre.conf', 'a').close()
-
-class BEControllerTests(object):
-
-    def _serverForceStop(self, bc):
-        err = bc._shellcmd("netstat  -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill")
-        self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
-
-    def test_serverStartAndStop(self):
-        obe =  self._getBuildEnvironment()
-        bc = self._getBEController(obe)
-        try:
-            # setting layers, skip any layer info
-            bc.setLayers(BITBAKE_LAYER, POKY_LAYERS)
-        except NotImplementedError:
-            print("Test skipped due to command not implemented yet")
-            return True
-        # We are ok with the exception as we're handling the git already exists
-        except BuildSetupException:
-            pass
-
-        bc.pokydirname = test_sourcedir
-        bc.islayerset = True
-
-        hostname = test_address.split("@")[-1]
-
-        # test start server and stop
-        bc.startBBServer()
-
-        self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, int(bc.be.bbport))), "Server not answering")
-
-        self._serverForceStop(bc)
-
-    def test_getBBController(self):
-        obe = self._getBuildEnvironment()
-        bc = self._getBEController(obe)
-        layerSet = False
-        try:
-            # setting layers, skip any layer info
-            layerSet = bc.setLayers(BITBAKE_LAYER, POKY_LAYERS)
-        except NotImplementedError:
-            print("Test skipped due to command not implemented yet")
-            return True
-        # We are ok with the exception as we're handling the git already exists
-        except BuildSetupException:
-            pass
-
-        bc.pokydirname = test_sourcedir
-        bc.islayerset = True
-
-        bbc = bc.getBBController()
-        self.assertTrue(isinstance(bbc, BitbakeController))
-
-        self._serverForceStop(bc)
-
-class LocalhostBEControllerTests(TestCase, BEControllerTests):
-    def __init__(self, *args):
-        super(LocalhostBEControllerTests, self).__init__(*args)
-
-
-    def _getBuildEnvironment(self):
-        return BuildEnvironment.objects.create(
-                lock = BuildEnvironment.LOCK_FREE,
-                betype = BuildEnvironment.TYPE_LOCAL,
-                address = test_address,
-                sourcedir = test_sourcedir,
-                builddir = test_builddir )
-
-    def _getBEController(self, obe):
-        return LocalhostBEController(obe)
-
-class RunBuildsCommandTests(TestCase):
-    def test_bec_select(self):
-        """
-        Tests that we can find and lock a build environment
-        """
-
-        obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL)
-        command = Command()
-        bec = command._selectBuildEnvironment()
-
-        # make sure we select the object we've just built
-        self.assertTrue(bec.be.id == obe.id, "Environment is not properly selected")
-        # we have a locked environment
-        self.assertTrue(bec.be.lock == BuildEnvironment.LOCK_LOCK, "Environment is not locked")
-        # no more selections possible here
-        self.assertRaises(IndexError, command._selectBuildEnvironment)
-
-    def test_br_select(self):
-        from orm.models import Project, Release, BitbakeVersion, Branch
-        p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch=Branch.objects.get_or_create(name="HEAD"))[0])[0])
-        obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p)
-        command = Command()
-        br = command._selectBuildRequest()
-
-        # make sure we select the object we've just built
-        self.assertTrue(obr.id == br.id, "Request is not properly selected")
-        # we have a locked environment
-        self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated")
-        # no more selections possible here
-        self.assertRaises(IndexError, command._selectBuildRequest)

bitbake/lib/toaster/contrib/README

@@ -1,6 +0,0 @@
-contrib directory for toaster
-
-This directory holds code that works with Toaster, without being an integral part of the Toaster project.
-It is intended for testing code, testing fixtures, tools for Toaster, etc.
-
-NOTE: This directory is NOT a Python module.

bitbake/lib/toaster/contrib/tts/README

@@ -1,41 +0,0 @@
-
-Toaster Testing Framework
-Yocto Project
-
-
-Rationale
-------------
-As Toaster contributions grow with the number of people that contribute code, verifying each patch prior to submitting upstream becomes a hard-to-scale problem for humans. We devised this system in order to run patch-level validation, trying to eliminate common problems from submitted patches, in an automated fashion.
-
-The Toaster Testing Framework is a set of Python scripts that provides an extensible way to write smoke and regression tests that will be run on each patch set sent for review on the toaster mailing list.
-
-
-Usage
-------------
-There are three main executable scripts in this directory.
-    *    runner.py  is designed to be run from the command line. It requires, as mandatory parameter, a branch name on poky-contrib, branch which contains the patches to be tested. The program will auto-discover the available tests residing in this directory by looking for unittest classes, and will run the tests on the branch dumping the output to the standard output. Optionally, it can take parameters inhibiting the branch checkout, or specifying a single test to be run, for debugging purposes.
-    *    launcher.py  is a designed to be run from a crontab or similar scheduling mechanism. It looks up a backlog file containing branches-to-test (named tasks in the source code), select the first one in FIFO manner, and launch runner.py on it. It will await for completion, and email the standard output and standard error dumps from the runner.py execution
-    *    recv.py  is an email receiver, designed to be called as a pipe from a .forward file. It is used to monitor a mailing list, for example, and add tasks to the backlog based on review requests coming on the mailing list.
-
-
-Installation
-------------
-As prerequisite, we expect a functioning email system on a machine with Python2.
-
-The broad steps to installation
-* set up the .forward on the receiving email account to pipe to the recv.py file
-* edit config.py and settings.json to alter for local installation settings
-* on email receive, verify backlog.txt to see that the tasks are received and marked for processing
-* execute launcher.py in command line to verify that a test occurs with no problems, and that the outgoing email is delivered
-* add launcher.py
-
-
-
-Contribute
-------------
-What we need are tests. Add your own tests to either tests.py file, or to a new file.
-Use "config.logger" to write logs that will make it to email.
-
-Commonly used code should be going to shellutils, and configuration to config.py.
-
-Contribute code by emailing patches to the list: toaster@yoctoproject.org (membership required)

bitbake/lib/toaster/contrib/tts/TODO

@@ -1,9 +0,0 @@
-We need to implement tests:
-
-automated link checker; currently
-$ linkchecker -t 1000 -F csv http://localhost:8000/
-
-integrate the w3c-validation service; currently
-$ python urlcheck.py
-
-

bitbake/lib/toaster/contrib/tts/config.py

@@ -1,98 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# This is the configuration/single module for tts
-# everything that would be a global variable goes here
-
-import os, sys, logging
-import socket
-
-LOGDIR = "log"
-SETTINGS_FILE = os.path.join(os.path.dirname(__file__), "settings.json")
-TEST_DIR_NAME = "tts_testdir"
-
-DEBUG = True
-
-OWN_PID = os.getpid()
-
-W3C_VALIDATOR = "http://icarus.local/w3c-validator/check?doctype=HTML5&uri="
-
-TOASTER_PORT = 56789
-
-TESTDIR = None
-
-#we parse the w3c URL to know where to connect
-
-import urlparse
-
-def get_public_ip():
-    temp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    parsed_url = urlparse.urlparse("http://icarus.local/w3c-validator/check?doctype=HTML5&uri=")
-    temp_socket.connect((parsed_url.netloc, 80 if parsed_url.port is None else parsed_url.port))
-    public_ip = temp_socket.getsockname()[0]
-    temp_socket.close()
-    return public_ip
-
-TOASTER_BASEURL = "http://%s:%d/" % (get_public_ip(), TOASTER_PORT)
-
-
-OWN_EMAIL_ADDRESS = "Toaster Testing Framework <alexandru.damian@intel.com>"
-REPORT_EMAIL_ADDRESS = "alexandru.damian@intel.com"
-
-# make sure we have the basic logging infrastructure
-
-#pylint: disable=invalid-name
-# we disable the invalid name because the module-level "logger" is used througout bitbake
-logger = logging.getLogger("toastertest")
-__console__ = logging.StreamHandler(sys.stdout)
-__console__.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
-logger.addHandler(__console__)
-logger.setLevel(logging.DEBUG)
-
-
-# singleton file names
-LOCKFILE = "/tmp/ttf.lock"
-BACKLOGFILE = os.path.join(os.path.dirname(__file__), "backlog.txt")
-
-# task states
-def enum(*sequential, **named):
-    enums = dict(zip(sequential, range(len(sequential))), **named)
-    reverse = dict((value, key) for key, value in enums.items())
-    enums['reverse_mapping'] = reverse
-    return type('Enum', (), enums)
-
-
-class TASKS(object):
-    #pylint: disable=too-few-public-methods
-    PENDING = "PENDING"
-    INPROGRESS = "INPROGRESS"
-    DONE = "DONE"
-
-    @staticmethod
-    def next_task(task):
-        if task == TASKS.PENDING:
-            return TASKS.INPROGRESS
-        if task == TASKS.INPROGRESS:
-            return TASKS.DONE
-        raise Exception("Invalid next task state for %s" % task)
-
-# TTS specific
-CONTRIB_REPO = "git@git.yoctoproject.org:poky-contrib"
-

bitbake/lib/toaster/contrib/tts/launcher.py

@@ -1,101 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Program to run the next task listed from the backlog.txt; designed to be
-# run from crontab.
-
-from __future__ import print_function
-import sys, os, config, shellutils
-from shellutils import ShellCmdException
-
-# Import smtplib for the actual sending function
-import smtplib
-
-# Import the email modules we'll need
-from email.mime.text import MIMEText
-
-def _take_lockfile():
-    return shellutils.lockfile(shellutils.mk_lock_filename())
-
-
-def read_next_task_by_state(task_state, task_name=None):
-    if not os.path.exists(os.path.join(os.path.dirname(__file__), config.BACKLOGFILE)):
-        return None
-    os.rename(config.BACKLOGFILE, config.BACKLOGFILE + ".tmp")
-    task = None
-    with open(config.BACKLOGFILE + ".tmp", "r") as f_in:
-        with open(config.BACKLOGFILE, "w") as f_out:
-            for line in f_in.readlines():
-                if task is None:
-                    fields = line.strip().split("|", 2)
-                    if fields[1] == task_state:
-                        if task_name is None or task_name == fields[0]:
-                            task = fields[0]
-                            print("Updating %s %s to %s" % (task, task_state, config.TASKS.next_task(task_state)))
-                            line = "%s|%s\n" % (task, config.TASKS.next_task(task_state))
-                f_out.write(line)
-    os.remove(config.BACKLOGFILE + ".tmp")
-    return task
-
-def send_report(task_name, plaintext, errtext=None):
-    if errtext is None:
-        msg = MIMEText(plaintext)
-    else:
-        if plaintext is None:
-            plaintext = ""
-        msg = MIMEText("--STDOUT dump--\n\n%s\n\n--STDERR dump--\n\n%s" % (plaintext, errtext))
-
-    msg['Subject'] = "[review-request] %s - smoke test results" % task_name
-    msg['From'] = config.OWN_EMAIL_ADDRESS
-    msg['To'] = config.REPORT_EMAIL_ADDRESS
-
-    smtp_connection = smtplib.SMTP("localhost")
-    smtp_connection.sendmail(config.OWN_EMAIL_ADDRESS, [config.REPORT_EMAIL_ADDRESS], msg.as_string())
-    smtp_connection.quit()
-
-def main():
-    # we don't do anything if we have another instance of us running
-    lock_file = _take_lockfile()
-
-    if lock_file is None:
-        if config.DEBUG:
-            print("Concurrent script in progress, exiting")
-        sys.exit(1)
-
-    next_task = read_next_task_by_state(config.TASKS.PENDING)
-    if next_task is not None:
-        print("Next task is", next_task)
-        errtext = None
-        out = None
-        try:
-            out = shellutils.run_shell_cmd("%s %s" % (os.path.join(os.path.dirname(__file__), "runner.py"), next_task))
-        except ShellCmdException as exc:
-            print("Failed while running the test runner: %s", exc)
-            errtext = exc.__str__()
-        send_report(next_task, out, errtext)
-        read_next_task_by_state(config.TASKS.INPROGRESS, next_task)
-    else:
-        print("No task")
-
-    shellutils.unlockfile(lock_file)
-
-
-if __name__ == "__main__":
-    main()

bitbake/lib/toaster/contrib/tts/recv.py

@@ -1,56 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Program to receive review requests by email and log tasks to backlog.txt
-# Designed to be run by the email system from a .forward file:
-#
-# cat .forward
-# |[full/path]/recv.py
-
-from __future__ import print_function
-import sys, config, shellutils
-
-from email.parser import Parser
-
-def recv_mail(datastring):
-    headers = Parser().parsestr(datastring)
-    return headers['subject']
-
-def main():
-    lock_file = shellutils.lockfile(shellutils.mk_lock_filename(), retry=True)
-
-    if lock_file is None:
-        if config.DEBUG:
-            print("Concurrent script in progress, exiting")
-        sys.exit(1)
-
-    subject = recv_mail(sys.stdin.read())
-
-    subject_parts = subject.split()
-    if "[review-request]" in subject_parts:
-        task_name = subject_parts[subject_parts.index("[review-request]") + 1]
-        with open(config.BACKLOGFILE, "a") as fout:
-            line = "%s|%s\n" % (task_name, config.TASKS.PENDING)
-            fout.write(line)
-
-    shellutils.unlockfile(lock_file)
-
-if __name__ == "__main__":
-    main()

bitbake/lib/toaster/contrib/tts/runner.py

@@ -1,222 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-# This is the main test execution controller. It is designed to be run
-# manually from the command line, or to be called from a different program
-# that schedules test execution.
-#
-# Execute   runner.py -h   for help.
-
-
-
-from __future__ import print_function
-import sys, os
-import unittest, importlib
-import logging, pprint, json
-import re
-from shellutils import ShellCmdException, mkdirhier, run_shell_cmd
-
-import config
-
-# we also log to a file, in addition to console, because our output is important
-__log_file_name__ = os.path.join(os.path.dirname(__file__), "log/tts_%d.log" % config.OWN_PID)
-mkdirhier(os.path.dirname(__log_file_name__))
-__log_file__ = open(__log_file_name__, "w")
-__file_handler__ = logging.StreamHandler(__log_file__)
-__file_handler__.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
-
-config.logger.addHandler(__file_handler__)
-
-# set up log directory
-try:
-    if not os.path.exists(config.LOGDIR):
-        os.mkdir(config.LOGDIR)
-    else:
-        if not os.path.isdir(config.LOGDIR):
-            raise Exception("Expected log dir '%s' is not actually a directory." % config.LOGDIR)
-except OSError as exc:
-    raise exc
-
-# creates the under-test-branch as a separate directory
-def set_up_test_branch(settings, branch_name):
-    testdir = "%s/%s.%d" % (settings['workdir'], config.TEST_DIR_NAME, config.OWN_PID)
-
-    # creates the host dir
-    if os.path.exists(testdir):
-        raise Exception("Test dir '%s'is already there, aborting" % testdir)
-
-    # may raise OSError, is to be handled by the caller
-    os.makedirs(testdir)
-
-
-    # copies over the .git from the localclone
-    run_shell_cmd("cp -a '%s'/.git '%s'" % (settings['localclone'], testdir))
-
-    # add the remote if it doesn't exist
-    crt_remotes = run_shell_cmd("git remote -v", cwd=testdir)
-    remotes = [word for line in crt_remotes.split("\n") for word in line.split()]
-    if not config.CONTRIB_REPO in remotes:
-        remote_name = "tts_contrib"
-        run_shell_cmd("git remote add %s %s" % (remote_name, config.CONTRIB_REPO), cwd=testdir)
-    else:
-        remote_name = remotes[remotes.index(config.CONTRIB_REPO) - 1]
-
-    # do the fetch
-    run_shell_cmd("git fetch %s -p" % remote_name, cwd=testdir)
-
-    # do the checkout
-    run_shell_cmd("git checkout origin/master && git branch -D %s; git checkout %s/%s -b %s && git reset --hard" % (branch_name, remote_name, branch_name, branch_name), cwd=testdir)
-
-    return testdir
-
-
-def __search_for_tests():
-    # we find all classes that can run, and run them
-    tests = []
-    for _, _, files_list in os.walk(os.path.dirname(os.path.abspath(__file__))):
-        for module_file in [f[:-3] for f in files_list if f.endswith(".py") and not f.startswith("__init__")]:
-            config.logger.debug("Inspecting module %s", module_file)
-            current_module = importlib.import_module(module_file)
-            crtclass_names = vars(current_module)
-            for name in crtclass_names:
-                tested_value = crtclass_names[name]
-                if isinstance(tested_value, type(unittest.TestCase)) and issubclass(tested_value, unittest.TestCase):
-                    tests.append((module_file, name))
-        break
-    return tests
-
-
-# boilerplate to self discover tests and run them
-def execute_tests(dir_under_test, testname):
-
-    if testname is not None and "." in testname:
-        tests = []
-        tests.append(tuple(testname.split(".", 2)))
-    else:
-        tests = __search_for_tests()
-
-    # let's move to the directory under test
-    crt_dir = os.getcwd()
-    os.chdir(dir_under_test)
-
-    # execute each module
-    # pylint: disable=broad-except
-    # we disable the broad-except because we want to actually catch all possible exceptions
-    try:
-        # sorting the tests by the numeric order in the class name
-        tests = sorted(tests, key=lambda x: int(re.search(r"[0-9]+", x[1]).group(0)))
-        config.logger.debug("Discovered test clases: %s", pprint.pformat(tests))
-        unittest.installHandler()
-        suite = unittest.TestSuite()
-        loader = unittest.TestLoader()
-        result = unittest.TestResult()
-        result.failfast = True
-        for module_file, test_name in tests:
-            suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, test_name)))
-        config.logger.info("Running %d test(s)", suite.countTestCases())
-        suite.run(result)
-
-        for error in result.errors:
-            config.logger.error("Exception on test: %s\n%s", error[0],
-                                "\n".join(["-- %s" % x for x in error[1].split("\n")]))
-
-        for failure in result.failures:
-            config.logger.error("Failed test: %s:\n%s\n", failure[0],
-                                "\n".join(["--  %s" % x for x in failure[1].split("\n")]))
-
-        config.logger.info("Test results: %d ran, %d errors, %d failures", result.testsRun, len(result.errors), len(result.failures))
-
-    except Exception as exc:
-        import traceback
-        config.logger.error("Exception while running test. Tracedump: \n%s", traceback.format_exc())
-    finally:
-        os.chdir(crt_dir)
-    return len(result.failures)
-
-# verify that we had a branch-under-test name as parameter
-def validate_args():
-    from optparse import OptionParser
-    parser = OptionParser(usage="usage: %prog [options] branch_under_test")
-
-    parser.add_option("-t", "--test-dir", dest="testdir", default=None, help="Use specified directory to run tests, inhibits the checkout.")
-    parser.add_option("-s", "--single", dest="singletest", default=None, help="Run only the specified test")
-
-    (options, args) = parser.parse_args()
-    if len(args) < 1:
-        raise Exception("Please specify the branch to run on. Use option '-h' when in doubt.")
-    return (options, args)
-
-
-
-
-# load the configuration options
-def read_settings():
-    if not os.path.exists(config.SETTINGS_FILE) or not os.path.isfile(config.SETTINGS_FILE):
-        raise Exception("Config file '%s' cannot be openend" % config.SETTINGS_FILE)
-    return json.loads(open(config.SETTINGS_FILE, "r").read())
-
-
-# cleanup !
-def clean_up(testdir):
-    run_shell_cmd("rm -rf -- '%s'" % testdir)
-
-def dump_info(settings, options, args):
-    """ detailed information about current run configuration, for debugging purposes.
-    """
-    config.logger.debug("Settings:\n%s\nOptions:\n%s\nArguments:\n%s\n", settings, options, args)
-
-def main():
-    (options, args) = validate_args()
-
-    settings = read_settings()
-    need_cleanup = False
-
-    # dump debug info
-    dump_info(settings, options, args)
-
-    testdir = None
-    no_failures = 1
-    try:
-        if options.testdir is not None and os.path.exists(options.testdir):
-            testdir = os.path.abspath(options.testdir)
-            config.logger.info("No checkout, using %s", testdir)
-        else:
-            need_cleanup = True
-            testdir = set_up_test_branch(settings, args[0]) # we expect a branch name as first argument
-
-        config.TESTDIR = testdir    # we let tests know where to run
-
-        # ensure that the test dir only contains no *.pyc leftovers
-        run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \\;" % testdir)
-
-        no_failures = execute_tests(testdir, options.singletest)
-
-    except ShellCmdException as exc:
-        import traceback
-        config.logger.error("Error while setting up testing. Traceback: \n%s", traceback.format_exc())
-    finally:
-        if need_cleanup and testdir is not None:
-            clean_up(testdir)
-
-    sys.exit(no_failures)
-
-if __name__ == "__main__":
-    main()

bitbake/lib/toaster/contrib/tts/settings.json

@@ -1,5 +0,0 @@
-{
-	"repo": "git@git.yoctoproject.org:poky-contrib",
-	"localclone": "/home/ddalex/ssd/yocto/poky",
-	"workdir": "/home/ddalex/ssd/yocto"
-}

bitbake/lib/toaster/contrib/tts/shellutils.py

@@ -1,141 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Utilities shared by tests and other common bits of code.
-
-import sys, os, subprocess, fcntl, errno
-import config
-from config import logger
-
-
-# License warning; this code is copied from the BitBake project, file bitbake/lib/bb/utils.py
-# The code is originally licensed GPL-2.0, and we redistribute it under still GPL-2.0
-
-# End of copy is marked with #ENDOFCOPY marker
-
-def mkdirhier(directory):
-    """Create a directory like 'mkdir -p', but does not complain if
-    directory already exists like os.makedirs
-    """
-
-    try:
-        os.makedirs(directory)
-    except OSError as exc:
-        if exc.errno != errno.EEXIST:
-            raise exc
-
-def lockfile(name, shared=False, retry=True):
-    """
-    Use the file fn as a lock file, return when the lock has been acquired.
-    Returns a variable to pass to unlockfile().
-    """
-    config.logger.debug("take lockfile %s", name)
-    dirname = os.path.dirname(name)
-    mkdirhier(dirname)
-
-    if not os.access(dirname, os.W_OK):
-        logger.error("Unable to acquire lock '%s', directory is not writable",
-                     name)
-        sys.exit(1)
-
-    operation = fcntl.LOCK_EX
-    if shared:
-        operation = fcntl.LOCK_SH
-    if not retry:
-        operation = operation | fcntl.LOCK_NB
-
-    while True:
-        # If we leave the lockfiles lying around there is no problem
-        # but we should clean up after ourselves. This gives potential
-        # for races though. To work around this, when we acquire the lock
-        # we check the file we locked was still the lock file on disk.
-        # by comparing inode numbers. If they don't match or the lockfile
-        # no longer exists, we start again.
-
-        # This implementation is unfair since the last person to request the
-        # lock is the most likely to win it.
-
-        # pylint: disable=broad-except
-        # we disable the broad-except because we want to actually catch all possible exceptions
-        try:
-            lock_file = open(name, 'a+')
-            fileno = lock_file.fileno()
-            fcntl.flock(fileno, operation)
-            statinfo = os.fstat(fileno)
-            if os.path.exists(lock_file.name):
-                statinfo2 = os.stat(lock_file.name)
-                if statinfo.st_ino == statinfo2.st_ino:
-                    return lock_file
-            lock_file.close()
-        except Exception as exc:
-            try:
-                lock_file.close()
-            except Exception as exc2:
-                config.logger.error("Failed to close the lockfile: %s", exc2)
-            config.logger.error("Failed to acquire the lockfile: %s", exc)
-        if not retry:
-            return None
-
-def unlockfile(lock_file):
-    """
-    Unlock a file locked using lockfile()
-    """
-    try:
-        # If we had a shared lock, we need to promote to exclusive before
-        # removing the lockfile. Attempt this, ignore failures.
-        fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
-        os.unlink(lock_file.name)
-    except (IOError, OSError):
-        pass
-    fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
-    lock_file.close()
-
-#ENDOFCOPY
-
-
-def mk_lock_filename():
-    our_name = os.path.basename(__file__)
-    our_name = ".%s" % ".".join(reversed(our_name.split(".")))
-    return config.LOCKFILE + our_name
-
-
-
-class ShellCmdException(Exception):
-    pass
-
-def run_shell_cmd(command, cwd=None):
-    if cwd is None:
-        cwd = os.getcwd()
-
-    config.logger.debug("_shellcmd: (%s) %s", cwd, command)
-    process = subprocess.Popen(command, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (out, err) = process.communicate()
-    process.wait()
-    if process.returncode:
-        if len(err) == 0:
-            err = "command: %s \n%s" % (command, out)
-        else:
-            err = "command: %s \n%s" % (command, err)
-        config.logger.warning("_shellcmd: error \n%s\n%s", out, err)
-        raise ShellCmdException(err)
-    else:
-        #config.logger.debug("localhostbecontroller: shellcmd success\n%s" % out)
-        return out
-

bitbake/lib/toaster/contrib/tts/tests.py

@@ -1,115 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-# Test definitions. The runner will look for and auto-discover the tests
-# no matter what they file are they in, as long as they are in the same directory
-# as this file.
-
-import unittest
-from shellutils import run_shell_cmd, ShellCmdException
-import config
-
-import pexpect
-import sys, os, signal, time
-
-class Test00PyCompilable(unittest.TestCase):
-    ''' Verifies that all Python files are syntactically correct '''
-    def test_compile_file(self):
-        try:
-            run_shell_cmd("find . -name *py -type f -print0 | xargs -0 -n1 -P20 python -m py_compile", config.TESTDIR)
-        except ShellCmdException as exc:
-            self.fail("Error compiling python files: %s" % (exc))
-
-    def test_pylint_file(self):
-        try:
-            run_shell_cmd(r"find . -iname \"*\.py\" -type f -print0 | PYTHONPATH=${PYTHONPATH}:. xargs -r -0 -n1 pylint --load-plugins pylint_django -E --reports=n 2>&1", cwd=config.TESTDIR + "/bitbake/lib/toaster")
-        except ShellCmdException as exc:
-            self.fail("Pylint fails: %s\n" % exc)
-
-class Test01PySystemStart(unittest.TestCase):
-    ''' Attempts to start Toaster, verify that it is succesfull, and stop it '''
-    def setUp(self):
-        run_shell_cmd("bash -c 'rm -f build/*log'")
-
-    def test_start_interactive_mode(self):
-        try:
-            run_shell_cmd("bash -c 'source %s/oe-init-build-env && source toaster start webport=%d && source toaster stop'" % (config.TESTDIR, config.TOASTER_PORT), config.TESTDIR)
-        except ShellCmdException as exc:
-            self.fail("Failed starting interactive mode: %s" % (exc))
-
-    def test_start_managed_mode(self):
-        try:
-            run_shell_cmd("%s/bitbake/bin/toaster webport=%d nobrowser & sleep 10 && curl http://localhost:%d/ && kill -2 %%1" % (config.TESTDIR, config.TOASTER_PORT, config.TOASTER_PORT), config.TESTDIR)
-        except ShellCmdException as exc:
-            self.fail("Failed starting managed mode: %s" % (exc))
-
-class Test02HTML5Compliance(unittest.TestCase):
-    def setUp(self):
-        self.origdir = os.getcwd()
-        self.crtdir = os.path.dirname(config.TESTDIR)
-        self.cleanup_database = False
-        os.chdir(self.crtdir)
-        if not os.path.exists(os.path.join(self.crtdir, "toaster.sqlite")):
-            self.cleanup_database = True
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py syncdb --noinput" % config.TESTDIR)
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate orm" % config.TESTDIR)
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate bldcontrol" % config.TESTDIR)
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py loadconf %s/meta-yocto/conf/toasterconf.json" % (config.TESTDIR, config.TESTDIR))
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py lsupdates" % config.TESTDIR)
-
-            setup = pexpect.spawn("%s/bitbake/lib/toaster/manage.py checksettings" % config.TESTDIR)
-            setup.logfile = sys.stdout
-            setup.expect(r".*or type the full path to a different directory: ")
-            setup.sendline('')
-            setup.sendline('')
-            setup.expect(r".*or type the full path to a different directory: ")
-            setup.sendline('')
-            setup.expect(r"Enter your option: ")
-            setup.sendline('0')
-
-        self.child = pexpect.spawn("bash", ["%s/bitbake/bin/toaster" % config.TESTDIR, "webport=%d" % config.TOASTER_PORT, "nobrowser"], cwd=self.crtdir)
-        self.child.logfile = sys.stdout
-        self.child.expect("Toaster is now running. You can stop it with Ctrl-C")
-
-    def test_html5_compliance(self):
-        import urllist, urlcheck
-        results = {}
-        for url in urllist.URLS:
-            results[url] = urlcheck.validate_html5(config.TOASTER_BASEURL + url)
-
-        failed = []
-        for url in results:
-            if results[url][1] != 0:
-                failed.append((url, results[url]))
-
-
-        self.assertTrue(len(failed) == 0, "Not all URLs validate: \n%s " % "\n".join(["".join(str(x)) for x in failed]))
-
-        #(config.TOASTER_BASEURL + url, status, errors, warnings))
-
-    def tearDown(self):
-        while self.child.isalive():
-            self.child.kill(signal.SIGINT)
-            time.sleep(1)
-        os.chdir(self.origdir)
-        toaster_sqlite_path = os.path.join(self.crtdir, "toaster.sqlite")
-        if self.cleanup_database and os.path.exists(toaster_sqlite_path):
-            os.remove(toaster_sqlite_path)

bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py

@@ -1,155 +0,0 @@
-#!/usr/bin/python
-
-# Copyright
-
-# DESCRIPTION
-# This is script for running all selected toaster cases on
-# selected web browsers manifested in toaster_test.cfg.
-
-# 1. How to start toaster in yocto:
-# $ source poky/oe-init-build-env
-# $ source toaster start
-# $ bitbake core-image-minimal
-
-# 2. How to install selenium on Ubuntu:
-# $ sudo apt-get install scrot python-pip
-# $ sudo pip install selenium
-
-# 3. How to install selenium addon in firefox:
-# Download the lastest firefox addon from http://release.seleniumhq.org/selenium-ide/
-# Then install it. You can also install firebug and firepath addon
-
-# 4. How to start writing a new case:
-# All you need to do is to implement the function test_xxx() and  pile it on.
-
-# 5. How to test with Chrome browser
-# Download/install chrome on host
-# Download chromedriver from https://code.google.com/p/chromedriver/downloads/list  according to your host type
-# put chromedriver in PATH, (e.g. /usr/bin/, bear in mind to chmod)
-# For windows host, you may put chromedriver.exe in the same directory as chrome.exe
-
-import unittest, sys, os, platform
-import ConfigParser
-import argparse
-from toaster_automation_test import toaster_cases
-
-
-def get_args_parser():
-    description = "Script that runs toaster auto tests."
-    parser = argparse.ArgumentParser(description=description)
-    parser.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False,
-                       help='Run all tests.')
-    parser.add_argument('--run-suite', required=False, dest='run_suite', default=False,
-                       help='run suite (defined in cfg file)')
-
-    return parser
-
-
-def get_tests():
-    testslist = []
-
-    prefix = 'toaster_automation_test.toaster_cases'
-
-    for t in dir(toaster_cases):
-        if t.startswith('test_'):
-            testslist.append('.'.join((prefix, t)))
-
-    return testslist
-
-
-def get_tests_from_cfg(suite=None):
-
-    testslist = []
-    config = ConfigParser.SafeConfigParser()
-    config.read('toaster_test.cfg')
-
-    if suite is not None:
-        target_suite = suite.lower()
-
-        # TODO: if suite is valid suite
-
-    else:
-        target_suite = platform.system().lower()
-
-    try:
-        tests_from_cfg = eval(config.get('toaster_test_' + target_suite, 'test_cases'))
-    except:
-        print('Failed to get test cases from cfg file. Make sure the format is correct.')
-        return None
-
-    prefix = 'toaster_automation_test.toaster_cases.test_'
-    for t in tests_from_cfg:
-        testslist.append(prefix + str(t))
-
-    return testslist
-
-def main():
-
-    # In case this script is called from other directory
-    os.chdir(os.path.abspath(sys.path[0]))
-
-    parser = get_args_parser()
-    args = parser.parse_args()
-
-    if args.run_all_tests:
-        testslist = get_tests()
-    elif args.run_suite:
-        testslist = get_tests_from_cfg(args.run_suite)
-        os.environ['TOASTER_SUITE'] = args.run_suite
-    else:
-        testslist = get_tests_from_cfg()
-
-    if not testslist:
-        print('Failed to get test cases.')
-        exit(1)
-
-    suite = unittest.TestSuite()
-    loader = unittest.TestLoader()
-    loader.sortTestMethodsUsing = None
-    runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args))
-
-    for test in testslist:
-        try:
-            suite.addTests(loader.loadTestsFromName(test))
-        except:
-            return 1
-
-    result = runner.run(suite)
-
-    if result.wasSuccessful():
-        return 0
-    else:
-        return 1
-
-
-def buildResultClass(args):
-    """Build a Result Class to use in the testcase execution"""
-
-    class StampedResult(unittest.TextTestResult):
-        """
-        Custom TestResult that prints the time when a test starts.  As toaster-auto
-        can take a long time (ie a few hours) to run, timestamps help us understand
-        what tests are taking a long time to execute.
-        """
-        def startTest(self, test):
-            import time
-            self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
-            super(StampedResult, self).startTest(test)
-
-    return StampedResult
-
-
-if __name__ == "__main__":
-
-    try:
-        ret = main()
-    except:
-        ret = 1
-        import traceback
-        traceback.print_exc()
-    finally:
-        if os.getenv('TOASTER_SUITE'):
-            del os.environ['TOASTER_SUITE']
-    sys.exit(ret)
-
-

bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_test.cfg

@@ -1,25 +0,0 @@
-# Configuration file for toaster_test
-# Sorted by different host type
-
-# test browser could be: firefox; chrome; ie(still under development)
-# logging_level could be: CRITICAL; ERROR; WARNING; INFO; DEBUG; NOTSET
-
-
-[toaster_test_linux]
-toaster_url = 'http://127.0.0.1:8000'
-test_browser = 'firefox'
-test_cases = [946]
-logging_level = 'INFO'
-
-
-[toaster_test_windows]
-toaster_url = 'http://127.0.0.1:8000'
-test_browser = ['ie', 'firefox', 'chrome']
-test_cases = [901, 902, 903]
-logging_level = 'DEBUG'
-
-[toaster_test_darwin]
-toaster_url = 'http://127.0.0.1:8000'
-test_browser = 'firefox'
-test_cases = [901, 902, 903, 904, 906, 910, 911, 912, 913, 914, 915, 916, 923, 924, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 955, 956]
-logging_level = 'INFO'

bitbake/lib/toaster/contrib/tts/urlcheck.py

@@ -1,53 +0,0 @@
-from __future__ import print_function
-import sys
-
-import httplib2
-import config
-import urllist
-
-
-config.logger.info("Testing %s with %s", config.TOASTER_BASEURL, config.W3C_VALIDATOR)
-
-def validate_html5(url):
-    http_client = httplib2.Http(None)
-    status = "Failed"
-    errors = -1
-    warnings = -1
-
-    urlrequest = config.W3C_VALIDATOR+url
-
-    # pylint: disable=broad-except
-    # we disable the broad-except because we want to actually catch all possible exceptions
-    try:
-        resp, _ = http_client.request(urlrequest, "HEAD")
-        if resp['x-w3c-validator-status'] != "Abort":
-            status = resp['x-w3c-validator-status']
-            errors = int(resp['x-w3c-validator-errors'])
-            warnings = int(resp['x-w3c-validator-warnings'])
-
-            if status == 'Invalid':
-                config.logger.warning("Failed %s is %s\terrors %s warnings %s (check at %s)", url, status, errors, warnings, urlrequest)
-            else:
-                config.logger.debug("OK! %s", url)
-
-    except Exception as exc:
-        config.logger.warning("Failed validation call: %s", exc)
-    return (status, errors, warnings)
-
-
-def print_validation(url):
-    status, errors, warnings = validate_html5(url)
-    config.logger.error("url %s is %s\terrors %s warnings %s (check at %s)", url, status, errors, warnings, config.W3C_VALIDATOR+url)
-
-
-def main():
-    print("Testing %s with %s" % (config.TOASTER_BASEURL, config.W3C_VALIDATOR))
-
-    if len(sys.argv) > 1:
-        print_validation(sys.argv[1])
-    else:
-        for url in urllist.URLS:
-            print_validation(config.TOASTER_BASEURL+url)
-
-if __name__ == "__main__":
-    main()

bitbake/lib/toaster/contrib/tts/urllist.py

@@ -1,39 +0,0 @@
-URLS = [
-    'toastergui/landing/',
-    'toastergui/builds/',
-    'toastergui/build/1',
-    'toastergui/build/1/tasks/',
-    'toastergui/build/1/tasks/1/',
-    'toastergui/build/1/task/1',
-    'toastergui/build/1/recipes/',
-    'toastergui/build/1/recipe/1/active_tab/1',
-    'toastergui/build/1/recipe/1',
-    'toastergui/build/1/recipe_packages/1',
-    'toastergui/build/1/packages/',
-    'toastergui/build/1/package/1',
-    'toastergui/build/1/package_built_dependencies/1',
-    'toastergui/build/1/package_included_detail/1/1',
-    'toastergui/build/1/package_included_dependencies/1/1',
-    'toastergui/build/1/package_included_reverse_dependencies/1/1',
-    'toastergui/build/1/target/1',
-    'toastergui/build/1/target/1/targetpkg',
-    'toastergui/build/1/target/1/dirinfo',
-    'toastergui/build/1/target/1/dirinfo_filepath/_/bin/bash',
-    'toastergui/build/1/configuration',
-    'toastergui/build/1/configvars',
-    'toastergui/build/1/buildtime',
-    'toastergui/build/1/cpuusage',
-    'toastergui/build/1/diskio',
-    'toastergui/build/1/target/1/packagefile/1',
-    'toastergui/newproject/',
-    'toastergui/projects/',
-    'toastergui/project/1',
-    'toastergui/project/1/configuration',
-    'toastergui/project/1/builds/',
-    'toastergui/project/1/layers/',
-    'toastergui/project/1/layer/1',
-    'toastergui/project/1/importlayer',
-    'toastergui/project/1/targets/',
-    'toastergui/project/1/machines/',
-    'toastergui/',
-]

bitbake/lib/toaster/orm/fixtures/oe-core.xml

@@ -14,6 +14,13 @@
   </object>
   <object model="orm.bitbakeversion" pk="2">
     <field type="CharField" name="name">HEAD</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">HEAD</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="3">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">master</field>
   </object>
 
   <!-- Releases available -->
@@ -31,18 +38,30 @@
     <field type="CharField" name="branch_name">HEAD</field>
     <field type="TextField" name="helptext">Toaster will run your builds with the version of OpenEmbedded that you have cloned or downloaded to your computer.</field>
   </object>
+  <object model="orm.release" pk="3">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="description">OpenEmbedded core master</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">3</field>
+    <field type="CharField" name="branch_name">master</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch.</field>
+  </object>
 
   <!-- Default layers for each release -->
   <object model="orm.releasedefaultlayer" pk="1">
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
   </object>
-  <object model="orm.releasedefaultlayer" pk="4">
+  <object model="orm.releasedefaultlayer" pk="2">
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="3">
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
 
-  <!-- TYPE_LOCAL = 0 Layers for the Local release -->
+
+  <!-- Layer for the Local release -->
   <object model="orm.layer" pk="1">
     <field type="CharField" name="name">openembedded-core</field>
     <field type="CharField" name="vcs_url">git://git.openembedded.org/openembedded-core</field>

bitbake/lib/toaster/orm/fixtures/poky.xml

@@ -19,6 +19,13 @@
     <field type="CharField" name="branch">HEAD</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
+  <object model="orm.bitbakeversion" pk="3">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+
 
   <!-- Releases available -->
   <object model="orm.release" pk="1">
@@ -35,8 +42,15 @@
     <field type="CharField" name="branch_name">HEAD</field>
     <field type="TextField" name="helptext">Toaster will run your builds with the version of the Yocto Project you have cloned or downloaded to your computer.</field>
   </object>
+  <object model="orm.release" pk="3">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="description">Yocto Project master</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">3</field>
+    <field type="CharField" name="branch_name">master</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project Master branch&lt;/a&gt;.</field>
+  </object>
 
-  <!-- Default layers for each release -->
+  <!-- Default project layers for each release -->
   <object model="orm.releasedefaultlayer" pk="1">
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
@@ -61,9 +75,23 @@
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
     <field type="CharField" name="layer_name">meta-yocto-bsp</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="7">
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="8">
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="9">
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
 
-  <!-- Layers for the Local release
-       layersource TYPE_LOCAL = 0
+  <!-- Default layers provided by poky
+       openembedded-core
+       meta-poky
+       meta-yocto-bsp
   -->
   <object model="orm.layer" pk="1">
     <field type="CharField" name="name">openembedded-core</field>
@@ -71,6 +99,13 @@
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
   </object>
   <object model="orm.layer_version" pk="1">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="branch">morty</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
+  <object model="orm.layer_version" pk="2">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -78,14 +113,27 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
-
+  <object model="orm.layer_version" pk="3">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
 
   <object model="orm.layer" pk="2">
     <field type="CharField" name="name">meta-poky</field>
     <field type="CharField" name="layer_index_url"></field>
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
   </object>
-  <object model="orm.layer_version" pk="2">
+  <object model="orm.layer_version" pk="4">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="branch">morty</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+  <object model="orm.layer_version" pk="5">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -93,14 +141,27 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-
+  <object model="orm.layer_version" pk="6">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
 
   <object model="orm.layer" pk="3">
     <field type="CharField" name="name">meta-yocto-bsp</field>
     <field type="CharField" name="layer_index_url"></field>
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
   </object>
-  <object model="orm.layer_version" pk="3">
+  <object model="orm.layer_version" pk="7">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="branch">morty</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.layer_version" pk="8">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -108,4 +169,11 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
+  <object model="orm.layer_version" pk="9">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
 </django-objects>

bitbake/lib/toaster/orm/fixtures/settings.xml

@@ -4,7 +4,7 @@
   <!-- pk=1 is DISTRO -->
   <object model="orm.toastersetting" pk="2">
     <field type="CharField" name="name">DEFAULT_RELEASE</field>
-    <field type="CharField" name="value">morty</field>
+    <field type="CharField" name="value">master</field>
   </object>
   <object model="orm.toastersetting" pk="3">
     <field type="CharField" name="name">DEFCONF_PACKAGE_CLASSES</field>

bitbake/lib/toaster/orm/management/commands/lsupdates.py

@@ -90,7 +90,6 @@ class Command(NoArgsCommand):
             from urlparse import urlparse
 
         proxy_settings = os.environ.get("http_proxy", None)
-        oe_core_layer = 'openembedded-core'
 
         def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER):
             http_progress = Spinner()
@@ -154,41 +153,17 @@ class Command(NoArgsCommand):
 
         total = len(layers_info)
         for i, li in enumerate(layers_info):
-            # Special case for the openembedded-core layer
-            if li['name'] == oe_core_layer:
-                try:
-                    # If we have an existing openembedded-core for example
-                    # from the toasterconf.json augment the info using the
-                    # layerindex rather than duplicate it
-                    oe_core_l = Layer.objects.get(name=oe_core_layer)
-                    # Take ownership of the layer as now coming from the
-                    # layerindex
-                    oe_core_l.summary = li['summary']
-                    oe_core_l.description = li['description']
-                    oe_core_l.vcs_web_url = li['vcs_web_url']
-                    oe_core_l.vcs_web_tree_base_url = \
-                        li['vcs_web_tree_base_url']
-                    oe_core_l.vcs_web_file_base_url = \
-                        li['vcs_web_file_base_url']
-
-                    oe_core_l.save()
-                    li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
-                    self.mini_progress("layers", i, total)
-                    continue
-
-                except Layer.DoesNotExist:
-                    pass
-
             try:
-                l, created = Layer.objects.get_or_create(name=li['name'],
-                                                         vcs_url=li['vcs_url'])
+                l, created = Layer.objects.get_or_create(name=li['name'])
                 l.up_date = li['updated']
-                l.vcs_url = li['vcs_url']
-                l.vcs_web_url = li['vcs_web_url']
-                l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
-                l.vcs_web_file_base_url = li['vcs_web_file_base_url']
                 l.summary = li['summary']
                 l.description = li['description']
+
+                if created:
+                    l.vcs_url = li['vcs_url']
+                    l.vcs_web_url = li['vcs_web_url']
+                    l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
+                    l.vcs_web_file_base_url = li['vcs_web_file_base_url']
                 l.save()
             except Layer.MultipleObjectsReturned:
                 logger.info("Skipped %s as we found multiple layers and "
@@ -211,12 +186,14 @@ class Command(NoArgsCommand):
 
         total = len(layerbranches_info)
         for i, lbi in enumerate(layerbranches_info):
+            # release as defined by toaster map to layerindex branch
+            release = li_branch_id_to_toaster_release[lbi['branch']]
 
             try:
                 lv, created = Layer_Version.objects.get_or_create(
-                    layer_source=LayerSource.TYPE_LAYERINDEX,
                     layer=Layer.objects.get(
-                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
+                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']]),
+                    release=release
                 )
             except KeyError:
                 logger.warning(
@@ -224,11 +201,12 @@ class Command(NoArgsCommand):
                     lbi['layer'])
                 continue
 
-            lv.release = li_branch_id_to_toaster_release[lbi['branch']]
-            lv.up_date = lbi['updated']
-            lv.commit = lbi['actual_branch']
-            lv.dirpath = lbi['vcs_subdir']
-            lv.save()
+            if created:
+                lv.release = li_branch_id_to_toaster_release[lbi['branch']]
+                lv.up_date = lbi['updated']
+                lv.commit = lbi['actual_branch']
+                lv.dirpath = lbi['vcs_subdir']
+                lv.save()
 
             li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
                 lv.pk
@@ -255,9 +233,8 @@ class Command(NoArgsCommand):
                 layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
 
                 dependlist[lv].append(
-                    Layer_Version.objects.get(
-                        layer_source=LayerSource.TYPE_LAYERINDEX,
-                        layer__pk=layer_id))
+                    Layer_Version.objects.get(layer__pk=layer_id,
+                                              release=lv.release))
 
             except Layer_Version.DoesNotExist:
                 logger.warning("Cannot find layer version (ls:%s),"

bitbake/lib/toaster/orm/models.py

@@ -38,6 +38,7 @@ import re
 import itertools
 from signal import SIGUSR1
 
+
 import logging
 logger = logging.getLogger("toaster")
 
@@ -178,24 +179,27 @@ class ProjectManager(models.Manager):
         else:
             return projects[0]
 
+
 class Project(models.Model):
-    search_allowed_fields = ['name', 'short_description', 'release__name', 'release__branch_name']
+    search_allowed_fields = ['name', 'short_description', 'release__name',
+                             'release__branch_name']
     name = models.CharField(max_length=100)
     short_description = models.CharField(max_length=50, blank=True)
     bitbake_version = models.ForeignKey('BitbakeVersion', null=True)
-    release     = models.ForeignKey("Release", null=True)
-    created     = models.DateTimeField(auto_now_add = True)
-    updated     = models.DateTimeField(auto_now = True)
+    release = models.ForeignKey("Release", null=True)
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
     # This is a horrible hack; since Toaster has no "User" model available when
     # running in interactive mode, we can't reference the field here directly
-    # Instead, we keep a possible null reference to the User id, as not to force
+    # Instead, we keep a possible null reference to the User id,
+    # as not to force
     # hard links to possibly missing models
-    user_id     = models.IntegerField(null = True)
-    objects     = ProjectManager()
+    user_id = models.IntegerField(null=True)
+    objects = ProjectManager()
 
     # set to True for the project which is the default container
     # for builds initiated by the command line etc.
-    is_default  = models.BooleanField(default = False)
+    is_default= models.BooleanField(default=False)
 
     def __unicode__(self):
         return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version)
@@ -333,20 +337,45 @@ class Project(models.Model):
 
         return queryset
 
-
     def schedule_build(self):
-        from bldcontrol.models import BuildRequest, BRTarget, BRLayer, BRVariable, BRBitbake
-        br = BuildRequest.objects.create(project = self)
-        try:
 
-            BRBitbake.objects.create(req = br,
-                giturl = self.bitbake_version.giturl,
-                commit = self.bitbake_version.branch,
-                dirpath = self.bitbake_version.dirpath)
+        from bldcontrol.models import BuildRequest, BRTarget, BRLayer
+        from bldcontrol.models import BRBitbake, BRVariable
+
+        try:
+            now = timezone.now()
+            build = Build.objects.create(project=self,
+                                         completed_on=now,
+                                         started_on=now)
+
+            br = BuildRequest.objects.create(project=self,
+                                             state=BuildRequest.REQ_QUEUED,
+                                             build=build)
+            BRBitbake.objects.create(req=br,
+                                     giturl=self.bitbake_version.giturl,
+                                     commit=self.bitbake_version.branch,
+                                     dirpath=self.bitbake_version.dirpath)
+
+            for t in self.projecttarget_set.all():
+                BRTarget.objects.create(req=br, target=t.target, task=t.task)
+                Target.objects.create(build=br.build, target=t.target,
+                                      task=t.task)
+                # If we're about to build a custom image recipe make sure
+                # that layer is currently in the project before we create the
+                # BRLayer objects
+                customrecipe = CustomImageRecipe.objects.filter(
+                    name=t.target,
+                    project=self).first()
+                if customrecipe:
+                    ProjectLayer.objects.get_or_create(
+                        project=self,
+                        layercommit=customrecipe.layer_version,
+                        optional=False)
 
             for l in self.projectlayer_set.all().order_by("pk"):
                 commit = l.layercommit.get_vcs_reference()
-                print("ii Building layer ", l.layercommit.layer.name, " at vcs point ", commit)
+                logger.debug("Adding layer to build %s" %
+                             l.layercommit.layer.name)
                 BRLayer.objects.create(
                     req=br,
                     name=l.layercommit.layer.name,
@@ -357,25 +386,16 @@ class Project(models.Model):
                     local_source_dir=l.layercommit.layer.local_source_dir
                 )
 
-            br.state = BuildRequest.REQ_QUEUED
-            now = timezone.now()
-            br.build = Build.objects.create(project = self,
-                                completed_on=now,
-                                started_on=now,
-                                )
-            for t in self.projecttarget_set.all():
-                BRTarget.objects.create(req = br, target = t.target, task = t.task)
-                Target.objects.create(build = br.build, target = t.target, task = t.task)
-
             for v in self.projectvariable_set.all():
-                BRVariable.objects.create(req = br, name = v.name, value = v.value)
-
+                BRVariable.objects.create(req=br, name=v.name, value=v.value)
 
             try:
-                br.build.machine = self.projectvariable_set.get(name = 'MACHINE').value
+                br.build.machine = self.projectvariable_set.get(
+                    name='MACHINE').value
                 br.build.save()
             except ProjectVariable.DoesNotExist:
                 pass
+
             br.save()
             signal_runbuilds()
 
@@ -1365,7 +1385,7 @@ class Layer(models.Model):
     name = models.CharField(max_length=100)
     layer_index_url = models.URLField()
     vcs_url = GitURLField(default=None, null=True)
-    local_source_dir = models.TextField(null = True, default = None)
+    local_source_dir = models.TextField(null=True, default=None)
     vcs_web_url = models.URLField(null=True, default=None)
     vcs_web_tree_base_url = models.URLField(null=True, default=None)
     vcs_web_file_base_url = models.URLField(null=True, default=None)
@@ -1473,22 +1493,33 @@ class Layer_Version(models.Model):
             return self.commit
         return 'N/A'
 
-    def get_detailspage_url(self, project_id):
+    def get_detailspage_url(self, project_id=None):
+        """ returns the url to the layer details page uses own project
+        field if project_id is not specified """
+
+        if project_id is None:
+            project_id = self.project.pk
+
         return reverse('layerdetails', args=(project_id, self.pk))
 
     def get_alldeps(self, project_id):
         """Get full list of unique layer dependencies."""
-        def gen_layerdeps(lver, project):
+        def gen_layerdeps(lver, project, depth):
+            if depth == 0:
+                return
             for ldep in lver.dependencies.all():
                 yield ldep.depends_on
                 # get next level of deps recursively calling gen_layerdeps
-                for subdep in gen_layerdeps(ldep.depends_on, project):
+                for subdep in gen_layerdeps(ldep.depends_on, project, depth-1):
                     yield subdep
 
         project = Project.objects.get(pk=project_id)
         result = []
-        projectlvers = [player.layercommit for player in project.projectlayer_set.all()]
-        for dep in gen_layerdeps(self, project):
+        projectlvers = [player.layercommit for player in
+                        project.projectlayer_set.all()]
+        # protect against infinite layer dependency loops
+        maxdepth = 20
+        for dep in gen_layerdeps(self, project, maxdepth):
             # filter out duplicates and layers already belonging to the project
             if dep not in result + projectlvers:
                 result.append(dep)
@@ -1631,7 +1662,8 @@ class CustomImageRecipe(Recipe):
         if base_recipe_path:
             base_recipe = open(base_recipe_path, 'r').read()
         else:
-            raise IOError("Based on recipe file not found")
+            raise IOError("Based on recipe file not found: %s" %
+                          base_recipe_path)
 
         # Add a special case for when the recipe we have based a custom image
         # recipe on requires another recipe.
@@ -1741,8 +1773,12 @@ def invalidate_cache(**kwargs):
 
 def signal_runbuilds():
     """Send SIGUSR1 to runbuilds process"""
-    with open(os.path.join(os.getenv('BUILDDIR'), '.runbuilds.pid')) as pidf:
-        os.kill(int(pidf.read()), SIGUSR1)
+    try:
+        with open(os.path.join(os.getenv('BUILDDIR', '.'),
+                               '.runbuilds.pid')) as pidf:
+            os.kill(int(pidf.read()), SIGUSR1)
+    except FileNotFoundError:
+        logger.info("Stopping existing runbuilds: no current process found")
 
 django.db.models.signals.post_save.connect(invalidate_cache)
 django.db.models.signals.post_delete.connect(invalidate_cache)

bitbake/lib/toaster/tests/browser/README

@@ -2,9 +2,9 @@
 
 These tests require Selenium to be installed in your Python environment.
 
-The simplest way to install this is via pip:
+The simplest way to install this is via pip3:
 
-  pip install selenium==2.53.2
+  pip3 install selenium==2.53.2
 
 Note that if you use other versions of Selenium, some of the tests (such as
 tests.browser.test_js_unit_tests.TestJsUnitTests) may fail, as these rely on
@@ -18,7 +18,7 @@ To run tests against Chrome:
 * On Windows, put chromedriver.exe in the same directory as chrome.exe
 
 To run tests against PhantomJS (headless):
-
+--NOTE - Selenium seems to be deprecating support for this mode ---
 * Download and install PhantomJS:
   http://phantomjs.org/download.html
 * On *nix systems, put phantomjs on PATH
@@ -43,13 +43,30 @@ Marionette driver.)
 The test cases will instantiate a Selenium driver set by the
 TOASTER_TESTS_BROWSER environment variable, or Chrome if this is not specified.
 
+To run tests against the Selenium Firefox Docker container:
+More explanation is located at https://wiki.yoctoproject.org/wiki/TipsAndTricks/TestingToasterWithContainers
+* Run the Selenium container:
+  ** docker run -it --rm=true -p 5900:5900 -p 4444:4444 --name=selenium selenium/standalone-firefox-debug:2.53.0
+  *** 5900 is the default vnc port. If you are runing a vnc server on your machine map a different port e.g. -p 6900:5900 and connect vnc client to 127.0.0.1:6900
+  *** 4444 is the default selenium sever port.
+* Run the tests
+  ** TOASTER_TESTS_BROWSER=http://127.0.0.1:4444/wd/hub TOASTER_TESTS_URL=http://172.17.0.1:8000 ./bitbake/lib/toaster/manage.py test --liveserver=172.17.0.1:8000 tests.browser
+  ** TOASTER_TESTS_BROWSER=remote TOASTER_REMOTE_HUB=http://127.0.0.1:4444/wd/hub ./bitbake/lib/toaster/manage.py test --liveserver=172.17.0.1:8000 tests.browser
+  *** TOASTER_REMOTE_HUB - This is the address for the Selenium Remote Web Driver hub. Assuming you ran the contianer with -p 4444:4444 it will be http://127.0.0.1:4444/wd/hub.
+  ***  --liveserver=xxx tells Django to run the test server on an interface and port reachable by both host and container.
+  **** 172.17.0.1 is the default docker bridge on linux, viewable from inside and outside the contianers. Find it with "ip -4 addr show dev docker0"
+* connect to the vnc server to see the tests if you would like
+   ** xtightvncviewer 127.0.0.1:5900
+   ** note, you need to wait for the test container to come up before this can connect.
+
 Available drivers:
 
 * chrome (default)
 * firefox
 * marionette (for newer Firefoxes)
 * ie
-* phantomjs
+* phantomjs (deprecated)
+* remote
 
 e.g. to run the test suite with phantomjs where you have phantomjs installed
 in /home/me/apps/phantomjs:

bitbake/lib/toaster/tests/browser/selenium_helpers_base.py

@@ -39,7 +39,7 @@ from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.common.exceptions import NoSuchElementException, \
         StaleElementReferenceException, TimeoutException
 
-def create_selenium_driver(browser='chrome'):
+def create_selenium_driver(cls,browser='chrome'):
     # set default browser string based on env (if available)
     env_browser = os.environ.get('TOASTER_TESTS_BROWSER')
     if env_browser:
@@ -59,6 +59,15 @@ def create_selenium_driver(browser='chrome'):
         return webdriver.Ie()
     elif browser == 'phantomjs':
         return webdriver.PhantomJS()
+    elif browser == 'remote':
+        # if we were to add yet another env variable like TOASTER_REMOTE_BROWSER
+        # we could let people pick firefox or chrome, left for later
+        remote_hub= os.environ.get('TOASTER_REMOTE_HUB')
+        driver = webdriver.Remote(remote_hub,
+                                  webdriver.DesiredCapabilities.FIREFOX.copy())
+
+        driver.get("http://%s:%s"%(cls.server_thread.host,cls.server_thread.port))
+        return driver
     else:
         msg = 'Selenium driver for browser %s is not available' % browser
         raise RuntimeError(msg)
@@ -135,7 +144,7 @@ class SeleniumTestCaseBase(unittest.TestCase):
 
         # instantiate the Selenium webdriver once for all the test methods
         # in this test case
-        cls.driver = create_selenium_driver()
+        cls.driver = create_selenium_driver(cls)
         cls.driver.maximize_window()
 
     @classmethod

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -91,9 +91,10 @@ class TestLayerDetailsPage(SeleniumTestCase):
         for btn in self.find_all("dd .glyphicon-edit"):
             btn.click()
 
-        # Wait for the inputs to become visible
+        # Wait for the inputs to become visible after animation
         self.wait_until_visible("#layer-git input[type=text]")
         self.wait_until_visible("dd textarea")
+        self.wait_until_visible("dd .change-btn")
 
         # Edit each value
         for inputs in self.find_all("#layer-git input[type=text]") + \

bitbake/lib/toaster/tests/builds/buildtest.py

@@ -24,16 +24,18 @@ import sys
 import time
 import unittest
 
-from orm.models import Project, Release, ProjectTarget, Build
+from orm.models import Project, Release, ProjectTarget, Build, ProjectVariable
 from bldcontrol.models import BuildEnvironment
 
-from bldcontrol.management.commands.loadconf import Command\
-    as LoadConfigCommand
-
 from bldcontrol.management.commands.runbuilds import Command\
     as RunBuildsCommand
 
+from django.core.management import call_command
+
 import subprocess
+import logging
+
+logger = logging.getLogger("toaster")
 
 # We use unittest.TestCase instead of django.test.TestCase because we don't
 # want to wrap everything in a database transaction as an external process
@@ -43,63 +45,92 @@ import subprocess
 class BuildTest(unittest.TestCase):
 
     PROJECT_NAME = "Testbuild"
+    BUILDDIR = "/tmp/build/"
 
     def build(self, target):
         # So that the buildinfo helper uses the test database'
         self.assertEqual(
             os.environ.get('DJANGO_SETTINGS_MODULE', ''),
-            'toastermain.settings-test',
+            'toastermain.settings_test',
             "Please initialise django with the tests settings:  "
-            "DJANGO_SETTINGS_MODULE='toastermain.settings-test'")
+            "DJANGO_SETTINGS_MODULE='toastermain.settings_test'")
 
-        if self.target_already_built(target):
-            return
+        built = self.target_already_built(target)
+        if built:
+            return built
 
-        # Take a guess at the location of the toasterconf
-        poky_toaster_conf = '../../../meta-poky/conf/toasterconf.json'
-        oe_toaster_conf = '../../../meta/conf/toasterconf.json'
-        env_toaster_conf = os.environ.get('TOASTER_CONF')
+        call_command('loaddata', 'settings.xml', app_label="orm")
+        call_command('loaddata', 'poky.xml', app_label="orm")
 
-        config_file = None
-        if env_toaster_conf:
-            config_file = env_toaster_conf
+        current_builddir = os.environ.get("BUILDDIR")
+        if current_builddir:
+            BuildTest.BUILDDIR = current_builddir
         else:
-            if os.path.exists(poky_toaster_conf):
-                config_file = poky_toaster_conf
-            elif os.path.exists(oe_toaster_conf):
-                config_file = oe_toaster_conf
+            # Setup a builddir based on default layout
+            # bitbake inside openebedded-core
+            oe_init_build_env_path = os.path.join(
+                os.path.dirname(os.path.abspath(__file__)),
+                os.pardir,
+                os.pardir,
+                os.pardir,
+                os.pardir,
+                os.pardir,
+                'oe-init-build-env'
+            )
+            if not os.path.exists(oe_init_build_env_path):
+                raise Exception("We had no BUILDDIR set and couldn't "
+                                "find oe-init-build-env to set this up "
+                                "ourselves please run oe-init-build-env "
+                                "before running these tests")
 
-        self.assertIsNotNone(config_file,
-                             "Default locations for toasterconf not found"
-                             "please set $TOASTER_CONF manually")
+            oe_init_build_env_path = os.path.realpath(oe_init_build_env_path)
+            cmd = "bash -c 'source oe-init-build-env %s'" % BuildTest.BUILDDIR
+            p = subprocess.Popen(
+                cmd,
+                cwd=os.path.dirname(oe_init_build_env_path),
+                shell=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
 
-        # Setup the release information and default layers
-        print("\nImporting file: %s" % config_file)
-        os.environ['TOASTER_CONF'] = config_file
-        LoadConfigCommand()._import_layer_config(config_file)
+            output, err = p.communicate()
+            p.wait()
 
-        os.environ['TOASTER_DIR'] = \
-            os.path.abspath(os.environ['BUILDDIR'] + "/../")
+            logger.info("oe-init-build-env %s %s" % (output, err))
 
-        os.environ['BBBASEDIR'] = \
-            subprocess.check_output('which bitbake', shell=True)
+            os.environ['BUILDDIR'] = BuildTest.BUILDDIR
+
+        # Setup the path to bitbake we know where to find this
+        bitbake_path = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            os.pardir,
+            os.pardir,
+            os.pardir,
+            os.pardir,
+            'bin',
+            'bitbake')
+        if not os.path.exists(bitbake_path):
+            raise Exception("Could not find bitbake at the expected path %s"
+                            % bitbake_path)
+
+        os.environ['BBBASEDIR'] = bitbake_path
 
         BuildEnvironment.objects.get_or_create(
             betype=BuildEnvironment.TYPE_LOCAL,
-            sourcedir=os.environ['TOASTER_DIR'],
-            builddir=os.environ['BUILDDIR']
+            sourcedir=BuildTest.BUILDDIR,
+            builddir=BuildTest.BUILDDIR
         )
 
         release = Release.objects.get(name='local')
 
         # Create a project for this build to run in
-        try:
-            project = Project.objects.get(name=BuildTest.PROJECT_NAME)
-        except Project.DoesNotExist:
-            project = Project.objects.create_project(
-                name=BuildTest.PROJECT_NAME,
-                release=release
-            )
+        project = Project.objects.create_project(name=BuildTest.PROJECT_NAME,
+                                                 release=release)
+
+        if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"):
+            ProjectVariable.objects.get_or_create(
+                name="SSTATE_MIRRORS",
+                value="file://.* http://autobuilder.yoctoproject.org/pub/sstate/PATH;downloadfilename=PATH",
+                project=project)
 
         ProjectTarget.objects.create(project=project,
                                      target=target,
@@ -118,9 +149,11 @@ class BuildTest(unittest.TestCase):
             sys.stdout.flush()
             time.sleep(1)
 
-        self.assertNotEqual(build_request.build.outcome,
-                            Build.SUCCEEDED, "Build did not SUCCEEDED")
-        print("\nBuild finished")
+        self.assertEqual(Build.objects.get(pk=build_pk).outcome,
+                         Build.SUCCEEDED,
+                         "Build did not SUCCEEDED")
+
+        logger.info("\nBuild finished %s" % build_request.build.outcome)
         return build_request.build
 
     def target_already_built(self, target):
@@ -129,6 +162,6 @@ class BuildTest(unittest.TestCase):
                 project__name=BuildTest.PROJECT_NAME):
             targets = build.target_set.values_list('target', flat=True)
             if target in targets:
-                return True
+                return build
 
-        return False
+        return None

bitbake/lib/toaster/tests/builds/test_core_image_min.py

@@ -31,14 +31,14 @@ from orm.models import Package_Dependency, Recipe_Dependency, Build
 from orm.models import Task_Dependency, Package, Target, Recipe
 from orm.models import CustomImagePackage
 
-from buildtest import BuildTest
+from tests.builds.buildtest import BuildTest
 
 
 class BuildCoreImageMinimal(BuildTest):
     """Build core-image-minimal and test the results"""
 
     def setUp(self):
-        self.build("core-image-minimal")
+        self.completed_build = self.build("core-image-minimal")
 
     # Check if build name is unique - tc_id=795
     def test_Build_Unique_Name(self):
@@ -59,38 +59,29 @@ class BuildCoreImageMinimal(BuildTest):
 
     # Check if task order is unique for one build - tc=824
     def test_Task_Unique_Order(self):
-        builds = Build.objects.values('id')
-        cnt_err = []
+        total_task_order = Task.objects.filter(
+            build=self.built).values('order').count()
+        distinct_task_order = Task.objects.filter(
+            build=self.completed_build).values('order').distinct().count()
 
-        for build in builds:
-            total_task_order = Task.objects.filter(
-                build=build['id']).values('order').count()
-            distinct_task_order = Task.objects.filter(
-                build=build['id']).values('order').distinct().count()
-
-            if (total_task_order != distinct_task_order):
-                cnt_err.append(build['id'])
-
-        self.assertEqual(len(cnt_err),
-                         0,
-                         msg='Errors for build id: %s' % cnt_err)
+        self.assertEqual(total_task_order,
+                         distinct_task_order,
+                         msg='Errors task order is not unique')
 
     # Check task order sequence for one build - tc=825
     def test_Task_Order_Sequence(self):
-        builds = builds = Build.objects.values('id')
         cnt_err = []
-        for build in builds:
-            tasks = Task.objects.filter(
-                Q(build=build['id']),
-                ~Q(order=None),
-                ~Q(task_name__contains='_setscene')
-            ).values('id', 'order').order_by("order")
+        tasks = Task.objects.filter(
+            Q(build=self.completed_build),
+            ~Q(order=None),
+            ~Q(task_name__contains='_setscene')
+        ).values('id', 'order').order_by("order")
 
-            cnt_tasks = 0
-            for task in tasks:
-                cnt_tasks += 1
-                if (task['order'] != cnt_tasks):
-                    cnt_err.append(task['id'])
+        cnt_tasks = 0
+        for task in tasks:
+            cnt_tasks += 1
+            if (task['order'] != cnt_tasks):
+                cnt_err.append(task['id'])
         self.assertEqual(
             len(cnt_err), 0, msg='Errors for task id: %s' % cnt_err)
 
@@ -126,8 +117,7 @@ class BuildCoreImageMinimal(BuildTest):
                     task['sstate_result'] != Task.SSTATE_MISS):
                 cnt_err.append({'id': task['id'],
                                 'name': task['task_name'],
-                                'sstate_result': task['sstate_result'],
-                               })
+                                'sstate_result': task['sstate_result']})
 
         self.assertEqual(len(cnt_err),
                          0,

bitbake/lib/toaster/tests/commands/test_loaddata.py

@@ -0,0 +1,61 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.test import TestCase
+from django.core import management
+
+from orm.models import Layer_Version, Layer, Release, ToasterSetting
+
+
+class TestLoadDataFixtures(TestCase):
+    """ Test loading our 3 provided fixtures """
+    def test_run_loaddata_poky_command(self):
+        management.call_command('loaddata', 'poky')
+
+        num_releases = Release.objects.count()
+
+        self.assertTrue(
+            Layer_Version.objects.filter(
+                layer__name="meta-poky").count() == num_releases,
+            "Loaded poky fixture but don't have a meta-poky for all releases"
+            " defined")
+
+    def test_run_loaddata_oecore_command(self):
+        management.call_command('loaddata', 'oe-core')
+
+        # We only have the one layer for oe-core setup
+        self.assertTrue(
+            Layer.objects.filter(name="openembedded-core").count() > 0,
+            "Loaded oe-core fixture but still have no openemebedded-core"
+            " layer")
+
+    def test_run_loaddata_settings_command(self):
+        management.call_command('loaddata', 'settings')
+
+        self.assertTrue(
+            ToasterSetting.objects.filter(name="DEFAULT_RELEASE").count() > 0,
+            "Loaded settings but have no DEFAULT_RELEASE")
+
+        self.assertTrue(
+            ToasterSetting.objects.filter(
+                name__startswith="DEFCONF").count() > 0,
+            "Loaded settings but have no DEFCONF (default project "
+            "configuration values)")

bitbake/lib/toaster/tests/commands/test_lsupdates.py

@@ -0,0 +1,45 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.test import TestCase
+from django.core import management
+
+from orm.models import Layer_Version, Machine, Recipe
+
+
+class TestLayerIndexUpdater(TestCase):
+    def test_run_lsupdates_command(self):
+        # Load some release information for us to fetch from the layer index
+        management.call_command('loaddata', 'poky')
+
+        old_layers_count = Layer_Version.objects.count()
+        old_recipes_count = Recipe.objects.count()
+        old_machines_count = Machine.objects.count()
+
+        # Now fetch the metadata from the layer index
+        management.call_command('lsupdates')
+
+        self.assertTrue(Layer_Version.objects.count() > old_layers_count,
+                        "lsupdates ran but we still have no more layers!")
+        self.assertTrue(Recipe.objects.count() > old_recipes_count,
+                        "lsupdates ran but we still have no more Recipes!")
+        self.assertTrue(Machine.objects.count() > old_machines_count,
+                        "lsupdates ran but we still have no more Machines!")

bitbake/lib/toaster/tests/commands/test_runbuilds.py

@@ -0,0 +1,88 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+
+from django.test import TestCase
+from django.core import management
+
+from orm.models import signal_runbuilds
+
+import threading
+import time
+import subprocess
+import signal
+
+
+class KillRunbuilds(threading.Thread):
+    """ Kill the runbuilds process after an amount of time """
+    def __init__(self, *args, **kwargs):
+        super(KillRunbuilds, self).__init__(*args, **kwargs)
+        self.setDaemon(True)
+
+    def run(self):
+        time.sleep(5)
+        signal_runbuilds()
+        time.sleep(1)
+
+        pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
+                                    ".runbuilds.pid")
+
+        with open(pidfile_path) as pidfile:
+            pid = pidfile.read()
+            os.kill(int(pid), signal.SIGTERM)
+
+
+class TestCommands(TestCase):
+    """ Sanity test that runbuilds executes OK """
+
+    def setUp(self):
+        os.environ.setdefault("DJANGO_SETTINGS_MODULE",
+                              "toastermain.settings_test")
+        os.environ.setdefault("BUILDDIR",
+                              "/tmp/")
+
+        # Setup a real database if needed for runbuilds process
+        # to connect to
+        management.call_command('migrate')
+
+    def test_runbuilds_command(self):
+        kill_runbuilds = KillRunbuilds()
+        kill_runbuilds.start()
+
+        manage_py = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            os.pardir,
+            os.pardir,
+            "manage.py")
+
+        command = "%s runbuilds" % manage_py
+
+        process = subprocess.Popen(command,
+                                   shell=True,
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+
+        (out, err) = process.communicate()
+        process.wait()
+
+        self.assertNotEqual(process.returncode, 1,
+                            "Runbuilds returned an error %s" % err)

bitbake/lib/toaster/tests/eventreplay/README

@@ -0,0 +1,22 @@
+# Running eventreplay tests
+
+These tests use event log files produced by bitbake <target> -w <event log file>
+You need to have event log files produced before running this tests.
+
+At the moment of writing this document tests use 2 event log files: zlib.events
+and core-image-minimal.events. They're not provided with the tests due to their
+significant size.
+
+Here is how to produce them:
+
+$ . oe-init-build-env
+$ rm -r tmp sstate-cache
+$ bitbake core-image-minimal -w core-image-minimal.events
+$ rm -rf tmp sstate-cache
+$ bitbake zlib -w zlib.events
+
+After that it should be possible to run eventreplay tests this way:
+
+$ EVENTREPLAY_DIR=./ DJANGO_SETTINGS_MODULE=toastermain.settings_test ../bitbake/lib/toaster/manage.py test -v2 tests.eventreplay
+
+Note that environment variable EVENTREPLAY_DIR should point to the directory with event log files.

bitbake/lib/toaster/tests/eventreplay/__init__.py

@@ -0,0 +1,97 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Tests were part of openembedded-core oe selftest Authored by: Lucian Musat
+# Ionut Chisanovici, Paul Eggleton and Cristian Iorga
+
+"""
+Test toaster backend by playing build event log files
+using toaster-eventreplay script
+"""
+
+import os
+
+from subprocess import getstatusoutput
+from pathlib import Path
+
+from django.test import TestCase
+
+from orm.models import Target_Installed_Package, Package, Build
+
+class EventReplay(TestCase):
+    """Base class for eventreplay test cases"""
+
+    def setUp(self):
+        """
+        Setup build environment:
+            - set self.script to toaster-eventreplay path
+            - set self.eventplay_dir to the value of EVENTPLAY_DIR env variable
+        """
+        bitbake_dir = Path(__file__.split('lib/toaster')[0])
+        self.script = bitbake_dir /  'bin' / 'toaster-eventreplay'
+        self.assertTrue(self.script.exists(), "%s doesn't exist")
+        self.eventplay_dir = os.getenv("EVENTREPLAY_DIR")
+        self.assertTrue(self.eventplay_dir,
+                        "Environment variable EVENTREPLAY_DIR is not set")
+
+    def _replay(self, eventfile):
+        """Run toaster-eventplay <eventfile>"""
+        eventpath = Path(self.eventplay_dir) / eventfile
+        status, output = getstatusoutput('%s %s' % (self.script, eventpath))
+        if status:
+            print(output)
+
+        self.assertEqual(status, 0)
+
+class CoreImageMinimalEventReplay(EventReplay):
+    """Replay core-image-minimal events"""
+
+    def test_installed_packages(self):
+        """Test if all required packages have been installed"""
+
+        self._replay('core-image-minimal.events')
+
+        # test installed packages
+        packages = sorted(Target_Installed_Package.objects.\
+                          values_list('package__name', flat=True))
+        self.assertEqual(packages, ['base-files', 'base-passwd', 'busybox',
+                                    'busybox-hwclock', 'busybox-syslog',
+                                    'busybox-udhcpc', 'eudev', 'glibc',
+                                    'init-ifupdown', 'initscripts',
+                                    'initscripts-functions', 'kernel-base',
+                                    'kernel-module-uvesafb', 'libkmod',
+                                    'modutils-initscripts', 'netbase',
+                                    'packagegroup-core-boot', 'run-postinsts',
+                                    'sysvinit', 'sysvinit-inittab',
+                                    'sysvinit-pidof', 'udev-cache',
+                                    'update-alternatives-opkg',
+                                    'update-rc.d', 'util-linux-libblkid',
+                                    'util-linux-libuuid', 'v86d', 'zlib'])
+
+class ZlibEventReplay(EventReplay):
+    """Replay zlib events"""
+
+    def test_replay_zlib(self):
+        """Test if zlib build and package are in the database"""
+        self._replay("zlib.events")
+
+        self.assertEqual(Build.objects.last().target_set.last().target, "zlib")
+        self.assertTrue('zlib' in Package.objects.values_list('name', flat=True))

bitbake/lib/toaster/tests/views/README

@@ -0,0 +1,4 @@
+
+Django unit tests to verify classes and functions based on django Views
+
+To run just these tests use ./manage.py test tests.views

bitbake/lib/toaster/tests/views/test_views.py

@@ -24,31 +24,26 @@
 from django.test import TestCase
 from django.test.client import RequestFactory
 from django.core.urlresolvers import reverse
-from django.utils import timezone
 from django.db.models import Q
 
-from orm.models import Project, Release, BitbakeVersion, Package, LogMessage
-from orm.models import LayerSource, Layer, Build
-from orm.models import Layer_Version, Recipe, Machine, ProjectLayer, Target
-from orm.models import CustomImageRecipe, ProjectVariable
+from orm.models import Project, Package
+from orm.models import Layer_Version, Recipe
+from orm.models import CustomImageRecipe
 from orm.models import CustomImagePackage
 
-import toastermain
 import inspect
 import toastergui
 
 from toastergui.tables import SoftwareRecipesTable
 import json
-from datetime import timedelta
 from bs4 import BeautifulSoup
-import re
 import string
-import json
 
 PROJECT_NAME = "test project"
 PROJECT_NAME2 = "test project 2"
 CLI_BUILDS_PROJECT_NAME = 'Command line builds'
 
+
 class ViewTests(TestCase):
     """Tests to verify view APIs."""
 
@@ -75,7 +70,8 @@ class ViewTests(TestCase):
         url = reverse('all-projects')
         response = self.client.get(url, {"format": "json"}, follow=True)
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response['Content-Type'].startswith('application/json'))
+        self.assertTrue(response['Content-Type'].startswith(
+            'application/json'))
 
         data = json.loads(response.content.decode('utf-8'))
 
@@ -98,13 +94,13 @@ class ViewTests(TestCase):
         urls = [layers_url,
                 prj_url,
                 reverse('xhr_recipestypeahead', args=(self.project.id,)),
-                reverse('xhr_machinestypeahead', args=(self.project.id,)),
-               ]
+                reverse('xhr_machinestypeahead', args=(self.project.id,))]
 
         def basic_reponse_check(response, url):
             """Check data structure of http response."""
             self.assertEqual(response.status_code, 200)
-            self.assertTrue(response['Content-Type'].startswith('application/json'))
+            self.assertTrue(response['Content-Type'].startswith(
+                'application/json'))
 
             data = json.loads(response.content.decode('utf-8'))
 
@@ -133,7 +129,6 @@ class ViewTests(TestCase):
 
             return False
 
-
         for url in urls:
             results = False
 
@@ -147,35 +142,36 @@ class ViewTests(TestCase):
             # from each of the urls
             self.assertTrue(results)
 
-    def test_xhr_import_layer(self):
-        """Test xhr_importlayer API"""
-        #Test for importing an already existing layer
-        args = {'vcs_url' : "git://git.example.com/test",
-                'name' : "base-layer",
-                'git_ref': "c12b9596afd236116b25ce26dbe0d793de9dc7ce",
-                'project_id': self.project.id,
-                'local_source_dir': "",
-                'dir_path' : "/path/in/repository"}
-        response = self.client.post(reverse('xhr_importlayer'), args)
+    def test_xhr_add_layer(self):
+        """Test xhr_add API"""
+        # Test for importing an already existing layer
+        api_url = reverse('xhr_layer', args=(self.project.id,))
+
+        layer_data = {'vcs_url': "git://git.example.com/test",
+                      'name': "base-layer",
+                      'git_ref': "c12b9596afd236116b25ce26dbe0d793de9dc7ce",
+                      'project_id': self.project.id,
+                      'local_source_dir': "",
+                      'add_to_project': True,
+                      'dir_path': "/path/in/repository"}
+
+        layer_data_json = json.dumps(layer_data)
+
+        response = self.client.put(api_url, layer_data_json)
         data = json.loads(response.content.decode('utf-8'))
         self.assertEqual(response.status_code, 200)
         self.assertEqual(data["error"], "ok")
 
-        #Test to verify import of a layer successful
-        args['name'] = "meta-oe"
-        response = self.client.post(reverse('xhr_importlayer'), args)
-        data = json.loads(response.content.decode('utf-8'))
-        self.assertTrue(data["error"], "ok")
+        self.assertTrue(
+            layer_data['name'] in
+            self.project.get_all_compatible_layer_versions().values_list(
+                'layer__name',
+                flat=True),
+            "Could not find imported layer in project's all layers list"
+        )
 
-        #Test for html tag in the data
-        args['<'] = "testing html tag"
-        response = self.client.post(reverse('xhr_importlayer'), args)
-        data = json.loads(response.content.decode('utf-8'))
-        self.assertNotEqual(data["error"], "ok")
-
-        #Empty data passed
-        args = {}
-        response = self.client.post(reverse('xhr_importlayer'), args)
+        # Empty data passed
+        response = self.client.put(api_url, "{}")
         data = json.loads(response.content.decode('utf-8'))
         self.assertNotEqual(data["error"], "ok")
 
@@ -232,15 +228,14 @@ class ViewTests(TestCase):
                     "info": {'id': self.customr.id,
                              'name': self.customr.name,
                              'base_recipe_id': self.recipe1.id,
-                             'project_id': self.project.id,
-                            }
-                   }
-        self.assertEqual(json.loads(response.content.decode('utf-8')), expected)
+                             'project_id': self.project.id}}
+        self.assertEqual(json.loads(response.content.decode('utf-8')),
+                         expected)
 
     def test_xhr_custom_del(self):
         """Test deleting custom recipe"""
         name = "to be deleted"
-        recipe = CustomImageRecipe.objects.create(\
+        recipe = CustomImageRecipe.objects.create(
                      name=name, project=self.project,
                      base_recipe=self.recipe1,
                      file_path="/tmp/testing",
@@ -259,7 +254,8 @@ class ViewTests(TestCase):
         url = reverse('xhr_customrecipe_id', args=(recipe.id,))
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 200)
-        self.assertNotEqual(json.loads(response.content.decode('utf-8'))["error"], "ok")
+        self.assertNotEqual(json.loads(
+            response.content.decode('utf-8'))["error"], "ok")
 
     def test_xhr_custom_packages(self):
         """Test adding and deleting package to a custom recipe"""
@@ -280,7 +276,8 @@ class ViewTests(TestCase):
 
         response = self.client.delete(del_url)
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(json.loads(response.content.decode('utf-8')), {"error": "ok"})
+        self.assertEqual(json.loads(response.content.decode('utf-8')),
+                         {"error": "ok"})
         all_packages = self.customr.get_all_packages().values_list('pk',
                                                                    flat=True)
 
@@ -292,7 +289,8 @@ class ViewTests(TestCase):
 
         response = self.client.delete(del_url)
         self.assertEqual(response.status_code, 200)
-        self.assertNotEqual(json.loads(response.content.decode('utf-8'))["error"], "ok")
+        self.assertNotEqual(json.loads(
+            response.content.decode('utf-8'))["error"], "ok")
 
     def test_xhr_custom_packages_err(self):
         """Test error conditions of xhr_customrecipe_packages"""
@@ -303,8 +301,9 @@ class ViewTests(TestCase):
             for method in (self.client.put, self.client.delete):
                 response = method(url)
                 self.assertEqual(response.status_code, 200)
-                self.assertNotEqual(json.loads(response.content.decode('utf-8')),
-                                    {"error": "ok"})
+                self.assertNotEqual(json.loads(
+                    response.content.decode('utf-8')),
+                    {"error": "ok"})
 
     def test_download_custom_recipe(self):
         """Download the recipe file generated for the custom image"""
@@ -490,27 +489,28 @@ class ViewTests(TestCase):
                         # filter string to pass as the option
                         # This is the name of the filter:action
                         # e.g. project_filter:not_in_project
-                        filter_string = "%s:%s" % (column['filter_name'],
-                                                   filter_action['action_name'])
+                        filter_string = "%s:%s" % (
+                            column['filter_name'],
+                            filter_action['action_name'])
                         # Now get the data with the filter applied
                         filtered_data = get_data(table_cls(),
-                                                 {"filter" : filter_string})
+                                                 {"filter": filter_string})
 
                         # date range filter actions can't specify the
                         # number of results they return, so their count is 0
-                        if filter_action['count'] != None:
-                            self.assertEqual(len(filtered_data['rows']),
-                                             int(filter_action['count']),
-                                             "We added a table filter for %s but "
-                                             "the number of rows returned was not "
-                                             "what the filter info said there "
-                                             "would be" % name)
-
+                        if filter_action['count'] is not None:
+                            self.assertEqual(
+                                len(filtered_data['rows']),
+                                int(filter_action['count']),
+                                "We added a table filter for %s but "
+                                "the number of rows returned was not "
+                                "what the filter info said there "
+                                "would be" % name)
 
             # Test search functionality on the table
             something_found = False
             for search in list(string.ascii_letters):
-                search_data = get_data(table_cls(), {'search' : search})
+                search_data = get_data(table_cls(), {'search': search})
 
                 if len(search_data['rows']) > 0:
                     something_found = True
@@ -521,20 +521,20 @@ class ViewTests(TestCase):
                             " was found for the search of table %s" % name)
 
             # Test the limit functionality on the table
-            limited_data = get_data(table_cls(), {'limit' : "1"})
+            limited_data = get_data(table_cls(), {'limit': "1"})
             self.assertEqual(len(limited_data['rows']),
                              1,
                              "Limit 1 set on table %s but not 1 row returned"
                              % name)
 
             # Test the pagination functionality on the table
-            page_one_data = get_data(table_cls(), {'limit' : "1",
+            page_one_data = get_data(table_cls(), {'limit': "1",
                                                    "page": "1"})['rows'][0]
 
-            page_two_data = get_data(table_cls(), {'limit' : "1",
+            page_two_data = get_data(table_cls(), {'limit': "1",
                                                    "page": "2"})['rows'][0]
 
             self.assertNotEqual(page_one_data,
                                 page_two_data,
-                                "Changed page on table %s but first row is the "
-                                "same as the previous page" % name)
+                                "Changed page on table %s but first row is"
+                                " the same as the previous page" % name)

bitbake/lib/toaster/toastergui/api.py

@@ -20,6 +20,7 @@
 
 import re
 import logging
+import json
 from collections import Counter
 
 from orm.models import Project, ProjectTarget, Build, Layer_Version
@@ -136,14 +137,63 @@ class XhrBuildRequest(View):
 
 
 class XhrLayer(View):
-    """ Get and Update Layer information """
+    """ Delete, Get, Add and Update Layer information
+
+        Methods: GET POST DELETE PUT
+    """
+
+    def get(self, request, *args, **kwargs):
+        """
+        Get layer information
+
+        Method: GET
+        Entry point: /xhr_layer/<project id>/<layerversion_id>
+        """
+
+        try:
+            layer_version = Layer_Version.objects.get(
+                pk=kwargs['layerversion_id'])
+
+            project = Project.objects.get(pk=kwargs['pid'])
+
+            project_layers = ProjectLayer.objects.filter(
+                project=project).values_list("layercommit_id",
+                                             flat=True)
+
+            ret = {
+                'error': 'ok',
+                'id': layer_version.pk,
+                'name': layer_version.layer.name,
+                'layerdetailurl':
+                layer_version.get_detailspage_url(project.pk),
+                'vcs_ref': layer_version.get_vcs_reference(),
+                'vcs_url': layer_version.layer.vcs_url,
+                'local_source_dir': layer_version.layer.local_source_dir,
+                'layerdeps': {
+                    "list": [
+                        {
+                            "id": dep.id,
+                            "name": dep.layer.name,
+                            "layerdetailurl":
+                            dep.get_detailspage_url(project.pk),
+                            "vcs_url": dep.layer.vcs_url,
+                            "vcs_reference": dep.get_vcs_reference()
+                        }
+                        for dep in layer_version.get_alldeps(project.id)]
+                },
+                'projectlayers': list(project_layers)
+            }
+
+            return JsonResponse(ret)
+        except Layer_Version.DoesNotExist:
+            error_response("No such layer")
 
     def post(self, request, *args, **kwargs):
         """
           Update a layer
 
-          Entry point: /xhr_layer/<layerversion_id>
           Method: POST
+          Entry point: /xhr_layer/<layerversion_id>
 
           Args:
               vcs_url, dirpath, commit, up_branch, summary, description,
@@ -201,9 +251,100 @@ class XhrLayer(View):
             return error_response("Could not update layer version entry: %s"
                                   % e)
 
-        return JsonResponse({"error": "ok"})
+        return error_response("ok")
+
+    def put(self, request, *args, **kwargs):
+        """ Add a new layer
+
+        Method: PUT
+        Entry point: /xhr_layer/<project id>/
+        Args:
+            project_id, name,
+            [vcs_url, dir_path, git_ref], [local_source_dir], [layer_deps
+            (csv)]
+
+        """
+        try:
+            project = Project.objects.get(pk=kwargs['pid'])
+
+            layer_data = json.loads(request.body.decode('utf-8'))
+
+            # We require a unique layer name as otherwise the lists of layers
+            # becomes very confusing
+            existing_layers = \
+                project.get_all_compatible_layer_versions().values_list(
+                    "layer__name",
+                    flat=True)
+
+            add_to_project = False
+            layer_deps_added = []
+            if 'add_to_project' in layer_data:
+                add_to_project = True
+
+            if layer_data['name'] in existing_layers:
+                return JsonResponse({"error": "layer-name-exists"})
+
+            layer = Layer.objects.create(name=layer_data['name'])
+
+            layer_version = Layer_Version.objects.create(
+                layer=layer,
+                project=project,
+                layer_source=LayerSource.TYPE_IMPORTED)
+
+            # Local layer
+            if 'local_source_dir' in layer_data:
+                layer.local_source_dir = layer_data['local_source_dir']
+            # git layer
+            elif 'vcs_url' in layer_data:
+                layer.vcs_url = layer_data['vcs_url']
+                layer_version.dirpath = layer_data['dir_path']
+                layer_version.commit = layer_data['get_ref']
+                layer_version.branch = layer_data['get_ref']
+
+            layer.save()
+            layer_version.save()
+
+            if add_to_project:
+                ProjectLayer.objects.get_or_create(
+                    layercommit=layer_version, project=project)
+
+            # Add the layer dependencies
+            if 'layer_deps' in layer_data:
+                for layer_dep_id in layer_data['layer_deps'].split(","):
+                    layer_dep = Layer_Version.objects.get(pk=layer_dep_id)
+                    LayerVersionDependency.objects.get_or_create(
+                        layer_version=layer_version, depends_on=layer_dep)
+
+                    # Add layer deps to the project if specified
+                    if add_to_project:
+                        created, pl = ProjectLayer.objects.get_or_create(
+                            layercommit=layer_dep, project=project)
+                        layer_deps_added.append(
+                            {'name': layer_dep.layer.name,
+                             'layerdetailurl':
+                             layer_dep.get_detailspage_url(project.pk)})
+
+        except Layer_Version.DoesNotExist:
+            return error_response("layer-dep-not-found")
+        except Project.DoesNotExist:
+            return error_response("project-not-found")
+        except KeyError:
+            return error_response("incorrect-parameters")
+
+        return JsonResponse({'error': "ok",
+                             'imported_layer': {
+                                 'name': layer.name,
+                                 'layerdetailurl':
+                                 layer_version.get_detailspage_url()},
+                             'deps_added': layer_deps_added})
 
     def delete(self, request, *args, **kwargs):
+        """ Delete an imported layer
+
+        Method: DELETE
+        Entry point: /xhr_layer/<projed id>/<layerversion_id>
+
+        """
         try:
             # We currently only allow Imported layers to be deleted
             layer_version = Layer_Version.objects.get(
@@ -291,10 +432,13 @@ class XhrCustomRecipe(View):
                 return error_response("recipe-already-exists")
 
             # create layer 'Custom layer' and verion if needed
-            layer = Layer.objects.get_or_create(
+            layer, l_created = Layer.objects.get_or_create(
                 name=CustomImageRecipe.LAYER_NAME,
-                summary="Layer for custom recipes",
-                vcs_url="file:///toaster_created_layer")[0]
+                summary="Layer for custom recipes")
+
+            if l_created:
+                layer.local_source_dir = "toaster_created_layer"
+                layer.save()
 
             # Check if we have a layer version already
             # We don't use get_or_create here because the dirpath will change
@@ -303,9 +447,10 @@ class XhrCustomRecipe(View):
                                                 Q(layer=layer) &
                                                 Q(build=None)).last()
             if lver is None:
-                lver, created = Layer_Version.objects.get_or_create(
+                lver, lv_created = Layer_Version.objects.get_or_create(
                     project=params['project'],
                     layer=layer,
+                    layer_source=LayerSource.TYPE_LOCAL,
                     dirpath="toaster_created_layer")
 
             # Add a dependency on our layer to the base recipe's layer
@@ -319,7 +464,7 @@ class XhrCustomRecipe(View):
                                                optional=False)
 
             # Create the actual recipe
-            recipe, created = CustomImageRecipe.objects.get_or_create(
+            recipe, r_created = CustomImageRecipe.objects.get_or_create(
                 name=request.POST["name"],
                 base_recipe=params["base"],
                 project=params["project"],
@@ -329,7 +474,7 @@ class XhrCustomRecipe(View):
             # If we created the object then setup these fields. They may get
             # overwritten later on and cause the get_or_create to create a
             # duplicate if they've changed.
-            if created:
+            if r_created:
                 recipe.file_path = request.POST["name"]
                 recipe.license = "MIT"
                 recipe.version = "0.1"
@@ -789,6 +934,9 @@ class XhrProject(View):
                 "url": layer.layercommit.layer.layer_index_url,
                 "layerdetailurl": layer.layercommit.get_detailspage_url(
                     project.pk),
+                "xhrLayerUrl": reverse("xhr_layer",
+                                       args=(project.pk,
+                                             layer.layercommit.pk)),
                 "layersource": layer.layercommit.layer_source
             })
 

bitbake/lib/toaster/toastergui/static/css/default.css

@@ -210,7 +210,6 @@ fieldset.fields-apart-from-layer-name { margin-top: 20px; }
 #import-layer-name,
 #layer-subdir { width: 20%; }
 #layer-git-repo-url { width: 40%; }
-#layer-git-ref { width: 32%; }
 #local-dir-path { width: 45%; }
 #layer-dependency { width: 16em; }
 #layer-deps-list { margin-top: 0; }

bitbake/lib/toaster/toastergui/static/js/customrecipe.js

@@ -312,5 +312,11 @@ function customRecipePageInit(ctx) {
     });
   });
 
+ /* Stop the download link from working if it is in disabled state
+  * http://getbootstrap.com/css/#forms-disabled-fieldsets
+  */
+ $("a[disabled=disabled]").click(function(e){
+   e.preventDefault();
+ });
 
 }