build-appliance-image: Update to dunfell head revision

(From OE-Core rev: b885888df67eb5cdb3b82f4f0a07369a449e223b)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -1,12 +1,14 @@
 *.pyc
 *.pyo
 /*.patch
+/.repo/
 /build*/
 pyshtables.py
 pstage/
 scripts/oe-git-proxy-socks
 sources/
 meta-*/
+buildtools/
 !meta-skeleton
 !meta-selftest
 hob-image-*.bb

LICENSE

@@ -1,14 +1,20 @@
 Different components of OpenEmbedded are under different licenses (a mix
-of MIT and GPLv2). Please see:
-
-meta/COPYING.GPLv2 (GPLv2)
-meta/COPYING.MIT (MIT)
-meta-selftest/COPYING.MIT (MIT)
-meta-skeleton/COPYING.MIT (MIT)
+of MIT and GPLv2). See LICENSE.GPL-2.0-only and LICENSE.MIT for further 
+details of the individual licenses.
 
 All metadata is MIT licensed unless otherwise stated. Source code
-included in tree for individual recipes is under the LICENSE stated in
-the associated recipe (.bb file) unless otherwise stated.
+included in tree for individual recipes (e.g. patches) are under 
+the LICENSE stated in the associated recipe (.bb file) unless 
+otherwise stated.
 
 License information for any other files is either explicitly stated 
-or defaults to GPL version 2.
+or defaults to GPL version 2 only.
+
+Individual files contain the following style tags instead of the full license 
+text to identify their license:
+
+    SPDX-License-Identifier: GPL-2.0-only
+    SPDX-License-Identifier: MIT
+
+This enables machine processing of license information based on the SPDX
+License Identifiers that are here available: http://spdx.org/licenses/

LICENSE.GPL-2.0-only

@@ -0,0 +1,288 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+Note:
+Individual files contain the following tag instead of the full license text.
+
+    SPDX-License-Identifier: GPL-2.0-only
+
+This enables machine processing of license information based on the SPDX
+License Identifiers that are here available: http://spdx.org/licenses/

LICENSE.MIT

@@ -0,0 +1,25 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Note:
+Individual files contain the following tag instead of the full license text.
+
+    SPDX-License-Identifier: MIT
+
+This enables machine processing of license information based on the SPDX
+License Identifiers that are here available: http://spdx.org/licenses/

MEMORIAM

@@ -0,0 +1,5 @@
+Some project contributors who are sadly no longer with us:
+
+Greg Gilbert (treke) - Ahead of his time with licensing
+Thomas Wood (thos) - Creator of the original sato
+Scott Rifenbark (scottrif) - Our long standing techwriter whose words live on

README.LSB

@@ -1,26 +0,0 @@
-OE-Core aims to be able to provide basic LSB compatible images. There
-are some challenges for OE as LSB isn't always 100% relevant to its
-target embedded and IoT audiences. 
-
-One challenge is that the LSB spec is no longer being actively
-developed [https://github.com/LinuxStandardBase/lsb] and has 
-components which are end of life or significantly dated. OE 
-therefore provides compatibility with the following caveats:
-
-* Qt4 is provided by the separate meta-qt4 layer. Its noted that Qt4 
-  is end of life and this isn't something the core project regularly 
-  tests any longer. Users are recommended to group together to support
-  maintenance of that layer. [http://git.yoctoproject.org/cgit/cgit.cgi/meta-qt4/]
-
-* mailx has been dropped since its no longer being developed upstream 
-  and there are better, more modern replacements such as s-nail 
-  (http://sdaoden.eu/code.html) or mailutils (http://mailutils.org/).
-
-* A few perl modules that were required by LSB 4.x aren't provided:
-  libclass-isa, libenv, libdumpvalue, libfile-checktree,
-  libi18n-collate, libpod-plainer.
-
-* libpng 1.2 isn't provided; oe-core includes the latest release of libpng
-  instead.
-
-* pax (POSIX standard archive) tool is not provided.

README.OE-Core

@@ -0,0 +1,29 @@
+OpenEmbedded-Core
+=================
+
+OpenEmbedded-Core is a layer containing the core metadata for current versions
+of OpenEmbedded. It is distro-less (can build a functional image with
+DISTRO = "nodistro") and contains only emulated machine support.
+
+For information about OpenEmbedded, see the OpenEmbedded website:
+    http://www.openembedded.org/
+
+The Yocto Project has extensive documentation about OE including a reference manual
+which can be found at:
+    http://yoctoproject.org/documentation
+
+
+Contributing
+------------
+
+Please refer to
+http://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
+for guidelines on how to submit patches.
+
+Mailing list:
+
+    http://lists.openembedded.org/mailman/listinfo/openembedded-core
+
+Source code:
+
+    http://git.openembedded.org/openembedded-core/

bitbake/.gitattributes

@@ -0,0 +1,2 @@
+*min.js binary
+*min.css binary

bitbake/COPYING

@@ -1,339 +0,0 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
-
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-			    Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-			    NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-		     END OF TERMS AND CONDITIONS
-
-	    How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) year name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.

bitbake/HEADER

@@ -1,19 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# <one line to give the program's name and a brief idea of what it does.>
-# Copyright (C) <year>  <name of author>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-

bitbake/LICENSE

@@ -1,4 +1,13 @@
-BitBake is licensed under the GNU General Public License version 2.0. See COPYING for further details.
+BitBake is licensed under the GNU General Public License version 2.0. See 
+LICENSE.GPL-2.0-only for further details.
+
+Individual files contain the following style tags instead of the full license text:
+
+    SPDX-License-Identifier:	GPL-2.0-only
+
+This enables machine processing of license information based on the SPDX
+License Identifiers that are here available: http://spdx.org/licenses/
+
 
 The following external components are distributed with this software:
 
@@ -17,3 +26,4 @@ Foundation and individual contributors.
 * Font Awesome fonts redistributed under the SIL Open Font License 1.1
 
 * simplediff is distributed under the zlib license.
+

bitbake/LICENSE.GPL-2.0-only

@@ -0,0 +1,288 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+Note:
+Individual files contain the following tag instead of the full license text.
+
+    SPDX-License-Identifier: GPL-2.0-only
+
+This enables machine processing of license information based on the SPDX
+License Identifiers that are here available: http://spdx.org/licenses/

bitbake/LICENSE.MIT

@@ -0,0 +1,25 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Note:
+Individual files contain the following tag instead of the full license text.
+
+    SPDX-License-Identifier: MIT
+
+This enables machine processing of license information based on the SPDX
+License Identifiers that are here available: http://spdx.org/licenses/

bitbake/bin/bitbake

@@ -1,6 +1,4 @@
 #!/usr/bin/env python3
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -9,18 +7,8 @@
 # Copyright (C) 2005        ROAD GmbH
 # Copyright (C) 2006        Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
 import sys
@@ -38,7 +26,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "1.37.0"
+__version__ = "1.46.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-diffsigs

@@ -1,27 +1,16 @@
 #!/usr/bin/env python3
 
-# bitbake-diffsigs
-# BitBake task signature data comparison utility
+# bitbake-diffsigs / bitbake-dumpsig
+# BitBake task signature data dump and comparison utility
 #
 # Copyright (C) 2012-2013, 2017 Intel Corporation
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
 import sys
 import warnings
-import fnmatch
 import argparse
 import logging
 import pickle
@@ -32,7 +21,10 @@ import bb.tinfoil
 import bb.siggen
 import bb.msg
 
-logger = bb.msg.logger_create('bitbake-diffsigs')
+myname = os.path.basename(sys.argv[0])
+logger = bb.msg.logger_create(myname)
+
+is_dump = myname == 'bitbake-dumpsig'
 
 def find_siginfo(tinfoil, pn, taskname, sigs=None):
     result = None
@@ -59,8 +51,8 @@ def find_siginfo(tinfoil, pn, taskname, sigs=None):
         sys.exit(2)
     return result
 
-def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False):
-    """ Find the most recent signature files for the specified PN/task and compare them """
+def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
+    """ Find the most recent signature files for the specified PN/task """
 
     if not taskname.startswith('do_'):
         taskname = 'do_%s' % taskname
@@ -79,73 +71,81 @@ def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False
         latestfiles = [sigfiles[sig1], sigfiles[sig2]]
     else:
         filedates = find_siginfo(bbhandler, pn, taskname)
-        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:]
+        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
         if not latestfiles:
             logger.error('No sigdata files found matching %s %s' % (pn, taskname))
             sys.exit(1)
-        elif len(latestfiles) < 2:
-            logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
-            sys.exit(1)
 
-    # Define recursion callback
-    def recursecb(key, hash1, hash2):
-        hashes = [hash1, hash2]
-        hashfiles = find_siginfo(bbhandler, key, None, hashes)
+    return latestfiles
 
-        recout = []
-        if len(hashfiles) == 0:
-            recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
-        elif not hash1 in hashfiles:
-            recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
-        elif not hash2 in hashfiles:
-            recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
-        else:
-            out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
-            for change in out2:
-                for line in change.splitlines():
-                    recout.append('  ' + line)
 
-        return recout
+# Define recursion callback
+def recursecb(key, hash1, hash2):
+    hashes = [hash1, hash2]
+    hashfiles = find_siginfo(tinfoil, key, None, hashes)
 
-    # Recurse into signature comparison
-    logger.debug("Signature file (previous): %s" % latestfiles[-2])
-    logger.debug("Signature file (latest): %s" % latestfiles[-1])
-    output = bb.siggen.compare_sigfiles(latestfiles[-2], latestfiles[-1], recursecb, color=color)
-    if output:
-        print('\n'.join(output))
-    sys.exit(0)
+    recout = []
+    if len(hashfiles) == 0:
+        recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
+    elif not hash1 in hashfiles:
+        recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
+    elif not hash2 in hashfiles:
+        recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
+    else:
+        out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
+        for change in out2:
+            for line in change.splitlines():
+                recout.append('    ' + line)
 
+    return recout
 
 
 parser = argparse.ArgumentParser(
-    description="Compares siginfo/sigdata files written out by BitBake")
+    description=("Dumps" if is_dump else "Compares") + " siginfo/sigdata files written out by BitBake")
 
-parser.add_argument('-d', '--debug',
+parser.add_argument('-D', '--debug',
                     help='Enable debug output',
                     action='store_true')
 
-parser.add_argument('--color',
-        help='Colorize output (where %(metavar)s is %(choices)s)',
-        choices=['auto', 'always', 'never'], default='auto', metavar='color')
+if is_dump:
+    parser.add_argument("-t", "--task",
+            help="find the signature data file for the last run of the specified task",
+            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
-parser.add_argument("-t", "--task",
-        help="find the signature data files for last two runs of the specified task and compare them",
-        action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+    parser.add_argument("sigdatafile1",
+            help="Signature file to dump. Not used when using -t/--task.",
+            action="store", nargs='?', metavar="sigdatafile")
+else:
+    parser.add_argument('-c', '--color',
+            help='Colorize the output (where %(metavar)s is %(choices)s)',
+            choices=['auto', 'always', 'never'], default='auto', metavar='color')
 
-parser.add_argument("-s", "--signature",
-        help="With -t/--task, specify the signatures to look for instead of taking the last two",
-        action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
+    parser.add_argument('-d', '--dump',
+            help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
+            action='store_true')
 
-parser.add_argument("sigdatafile1",
-        help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
-        action="store", nargs='?')
+    parser.add_argument("-t", "--task",
+            help="find the signature data files for the last two runs of the specified task and compare them",
+            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
-parser.add_argument("sigdatafile2",
-        help="Second signature file to compare",
-        action="store", nargs='?')
+    parser.add_argument("-s", "--signature",
+            help="With -t/--task, specify the signatures to look for instead of taking the last two",
+            action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
 
+    parser.add_argument("sigdatafile1",
+            help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
+            action="store", nargs='?')
+
+    parser.add_argument("sigdatafile2",
+            help="Second signature file to compare",
+            action="store", nargs='?')
 
 options = parser.parse_args()
+if is_dump:
+    options.color = 'never'
+    options.dump = True
+    options.sigdatafile2 = None
+    options.sigargs = None
 
 if options.debug:
     logger.setLevel(logging.DEBUG)
@@ -155,17 +155,32 @@ color = (options.color == 'always' or (options.color == 'auto' and sys.stdout.is
 if options.taskargs:
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=True)
-        if options.sigargs:
-            find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1], color=color)
+        if not options.dump and options.sigargs:
+            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1])
         else:
-            find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], color=color)
+            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
+
+        if options.dump:
+            logger.debug("Signature file: %s" % files[-1])
+            output = bb.siggen.dump_sigfile(files[-1])
+        else:
+            if len(files) < 2:
+                logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (options.taskargs[0], options.taskargs[1]))
+                sys.exit(1)
+
+            # Recurse into signature comparison
+            logger.debug("Signature file (previous): %s" % files[-2])
+            logger.debug("Signature file (latest): %s" % files[-1])
+            output = bb.siggen.compare_sigfiles(files[-2], files[-1], recursecb, color=color)
 else:
     if options.sigargs:
         logger.error('-s/--signature can only be used together with -t/--task')
         sys.exit(1)
     try:
-        if options.sigdatafile1 and options.sigdatafile2:
-            output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, color=color)
+        if not options.dump and options.sigdatafile1 and options.sigdatafile2:
+            with bb.tinfoil.Tinfoil() as tinfoil:
+                tinfoil.prepare(config_only=True)
+                output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, recursecb, color=color)
         elif options.sigdatafile1:
             output = bb.siggen.dump_sigfile(options.sigdatafile1)
         else:
@@ -179,5 +194,5 @@ else:
         logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files')
         sys.exit(1)
 
-    if output:
-        print('\n'.join(output))
+if output:
+    print('\n'.join(output))

bitbake/bin/bitbake-dumpsig

@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-
-# bitbake-dumpsig
-# BitBake task signature dump utility
-#
-# Copyright (C) 2013 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import os
-import sys
-import warnings
-import optparse
-import logging
-import pickle
-
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
-
-import bb.tinfoil
-import bb.siggen
-import bb.msg
-
-logger = bb.msg.logger_create('bitbake-dumpsig')
-
-def find_siginfo_task(bbhandler, pn, taskname):
-    """ Find the most recent signature file for the specified PN/task """
-
-    if not hasattr(bb.siggen, 'find_siginfo'):
-        logger.error('Metadata does not support finding signature data files')
-        sys.exit(1)
-
-    if not taskname.startswith('do_'):
-        taskname = 'do_%s' % taskname
-
-    filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
-    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-1:]
-    if not latestfiles:
-        logger.error('No sigdata files found matching %s %s' % (pn, taskname))
-        sys.exit(1)
-
-    return latestfiles[0]
-
-parser = optparse.OptionParser(
-    description = "Dumps siginfo/sigdata files written out by BitBake",
-    usage = """
-  %prog -t recipename taskname
-  %prog sigdatafile""")
-
-parser.add_option("-D", "--debug",
-        help = "enable debug",
-        action = "store_true", dest="debug", default = False)
-
-parser.add_option("-t", "--task",
-        help = "find the signature data file for the specified task",
-        action="store", dest="taskargs", nargs=2, metavar='recipename taskname')
-
-options, args = parser.parse_args(sys.argv)
-
-if options.debug:
-    logger.setLevel(logging.DEBUG)
-
-if options.taskargs:
-    tinfoil = bb.tinfoil.Tinfoil()
-    tinfoil.prepare(config_only = True)
-    file = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
-    logger.debug("Signature file: %s" % file)
-elif len(args) == 1:
-    parser.print_help()
-    sys.exit(0)
-else:
-    file = args[1]
-
-try:
-    output = bb.siggen.dump_sigfile(file)
-except IOError as e:
-    logger.error(str(e))
-    sys.exit(1)
-except (pickle.UnpicklingError, EOFError):
-    logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file')
-    sys.exit(1)
-
-if output:
-    print('\n'.join(output))

bitbake/bin/bitbake-dumpsig

@@ -0,0 +1 @@
+bitbake-diffsigs

bitbake/bin/bitbake-hashclient

@@ -0,0 +1,170 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2019 Garmin Ltd.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import argparse
+import hashlib
+import logging
+import os
+import pprint
+import sys
+import threading
+import time
+
+try:
+    import tqdm
+    ProgressBar = tqdm.tqdm
+except ImportError:
+    class ProgressBar(object):
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args, **kwargs):
+            pass
+
+        def update(self):
+            pass
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
+
+import hashserv
+
+DEFAULT_ADDRESS = 'unix://./hashserve.sock'
+METHOD = 'stress.test.method'
+
+
+def main():
+    def handle_stats(args, client):
+        if args.reset:
+            s = client.reset_stats()
+        else:
+            s = client.get_stats()
+        pprint.pprint(s)
+        return 0
+
+    def handle_stress(args, client):
+        def thread_main(pbar, lock):
+            nonlocal found_hashes
+            nonlocal missed_hashes
+            nonlocal max_time
+
+            client = hashserv.create_client(args.address)
+
+            for i in range(args.requests):
+                taskhash = hashlib.sha256()
+                taskhash.update(args.taskhash_seed.encode('utf-8'))
+                taskhash.update(str(i).encode('utf-8'))
+
+                start_time = time.perf_counter()
+                l = client.get_unihash(METHOD, taskhash.hexdigest())
+                elapsed = time.perf_counter() - start_time
+
+                with lock:
+                    if l:
+                        found_hashes += 1
+                    else:
+                        missed_hashes += 1
+
+                    max_time = max(elapsed, max_time)
+                    pbar.update()
+
+        max_time = 0
+        found_hashes = 0
+        missed_hashes = 0
+        lock = threading.Lock()
+        total_requests = args.clients * args.requests
+        start_time = time.perf_counter()
+        with ProgressBar(total=total_requests) as pbar:
+            threads = [threading.Thread(target=thread_main, args=(pbar, lock), daemon=False) for _ in range(args.clients)]
+            for t in threads:
+                t.start()
+
+            for t in threads:
+                t.join()
+
+        elapsed = time.perf_counter() - start_time
+        with lock:
+            print("%d requests in %.1fs. %.1f requests per second" % (total_requests, elapsed, total_requests / elapsed))
+            print("Average request time %.8fs" % (elapsed / total_requests))
+            print("Max request time was %.8fs" % max_time)
+            print("Found %d hashes, missed %d" % (found_hashes, missed_hashes))
+
+        if args.report:
+            with ProgressBar(total=args.requests) as pbar:
+                for i in range(args.requests):
+                    taskhash = hashlib.sha256()
+                    taskhash.update(args.taskhash_seed.encode('utf-8'))
+                    taskhash.update(str(i).encode('utf-8'))
+
+                    outhash = hashlib.sha256()
+                    outhash.update(args.outhash_seed.encode('utf-8'))
+                    outhash.update(str(i).encode('utf-8'))
+
+                    client.report_unihash(taskhash.hexdigest(), METHOD, outhash.hexdigest(), taskhash.hexdigest())
+
+                    with lock:
+                        pbar.update()
+
+    parser = argparse.ArgumentParser(description='Hash Equivalence Client')
+    parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")')
+    parser.add_argument('--log', default='WARNING', help='Set logging level')
+
+    subparsers = parser.add_subparsers()
+
+    stats_parser = subparsers.add_parser('stats', help='Show server stats')
+    stats_parser.add_argument('--reset', action='store_true',
+                              help='Reset server stats')
+    stats_parser.set_defaults(func=handle_stats)
+
+    stress_parser = subparsers.add_parser('stress', help='Run stress test')
+    stress_parser.add_argument('--clients', type=int, default=10,
+                               help='Number of simultaneous clients')
+    stress_parser.add_argument('--requests', type=int, default=1000,
+                               help='Number of requests each client will perform')
+    stress_parser.add_argument('--report', action='store_true',
+                               help='Report new hashes')
+    stress_parser.add_argument('--taskhash-seed', default='',
+                               help='Include string in taskhash')
+    stress_parser.add_argument('--outhash-seed', default='',
+                               help='Include string in outhash')
+    stress_parser.set_defaults(func=handle_stress)
+
+    args = parser.parse_args()
+
+    logger = logging.getLogger('hashserv')
+
+    level = getattr(logging, args.log.upper(), None)
+    if not isinstance(level, int):
+        raise ValueError('Invalid log level: %s' % args.log)
+
+    logger.setLevel(level)
+    console = logging.StreamHandler()
+    console.setLevel(level)
+    logger.addHandler(console)
+
+    func = getattr(args, 'func', None)
+    if func:
+        client = hashserv.create_client(args.address)
+        # Try to establish a connection to the server now to detect failures
+        # early
+        client.connect()
+
+        return func(args, client)
+
+    return 0
+
+
+if __name__ == '__main__':
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc()
+    sys.exit(ret)

bitbake/bin/bitbake-hashserv

@@ -0,0 +1,62 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2018 Garmin Ltd.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import os
+import sys
+import logging
+import argparse
+import sqlite3
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
+
+import hashserv
+
+VERSION = "1.0.0"
+
+DEFAULT_BIND = 'unix://./hashserve.sock'
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Hash Equivalence Reference Server. Version=%s' % VERSION,
+                                     epilog='''The bind address is the path to a unix domain socket if it is
+                                               prefixed with "unix://". Otherwise, it is an IP address
+                                               and port in form ADDRESS:PORT. To bind to all addresses, leave
+                                               the ADDRESS empty, e.g. "--bind :8686". To bind to a specific
+                                               IPv6 address, enclose the address in "[]", e.g.
+                                               "--bind [::1]:8686"'''
+                                     )
+
+    parser.add_argument('--bind', default=DEFAULT_BIND, help='Bind address (default "%(default)s")')
+    parser.add_argument('--database', default='./hashserv.db', help='Database file (default "%(default)s")')
+    parser.add_argument('--log', default='WARNING', help='Set logging level')
+
+    args = parser.parse_args()
+
+    logger = logging.getLogger('hashserv')
+
+    level = getattr(logging, args.log.upper(), None)
+    if not isinstance(level, int):
+        raise ValueError('Invalid log level: %s' % args.log)
+
+    logger.setLevel(level)
+    console = logging.StreamHandler()
+    console.setLevel(level)
+    logger.addHandler(console)
+
+    server = hashserv.create_server(args.bind, args.database)
+    server.serve_forever()
+    return 0
+
+
+if __name__ == '__main__':
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc()
+    sys.exit(ret)

bitbake/bin/bitbake-layers

@@ -7,18 +7,8 @@
 # Copyright (C) 2011 Mentor Graphics Corporation
 # Copyright (C) 2011-2015 Intel Corporation
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import logging
 import os
@@ -62,7 +52,9 @@ def main():
 
     # Need to re-run logger_create with color argument
     # (will be the same logger since it has the same name)
-    bb.msg.logger_create('bitbake-layers', output=sys.stdout, color=global_args.color)
+    bb.msg.logger_create('bitbake-layers', output=sys.stdout,
+                         color=global_args.color,
+                         level=logger.getEffectiveLevel())
 
     plugins = []
     tinfoil = bb.tinfoil.Tinfoil(tracking=True)

bitbake/bin/bitbake-prserv

@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
 import os
 import sys,logging
 import optparse

bitbake/bin/bitbake-selftest

@@ -2,18 +2,8 @@
 #
 # Copyright (C) 2012 Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
 import sys, logging
@@ -22,16 +12,25 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib
 import unittest
 try:
     import bb
+    import hashserv
+    import layerindexlib
 except RuntimeError as exc:
     sys.exit(str(exc))
 
 tests = ["bb.tests.codeparser",
+         "bb.tests.cooker",
          "bb.tests.cow",
          "bb.tests.data",
          "bb.tests.event",
          "bb.tests.fetch",
          "bb.tests.parse",
-         "bb.tests.utils"]
+         "bb.tests.persist_data",
+         "bb.tests.runqueue",
+         "bb.tests.utils",
+         "hashserv.tests",
+         "layerindexlib.tests.layerindexobj",
+         "layerindexlib.tests.restapi",
+         "layerindexlib.tests.cooker"]
 
 for t in tests:
     t = '.'.join(t.split('.')[:3])

bitbake/bin/bitbake-worker

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
 
 import os
 import sys
@@ -62,7 +65,6 @@ if 0:
     format_str = "%(levelname)s: %(message)s"
     conlogformat = bb.msg.BBLogFormatter(format_str)
     consolelog = logging.FileHandler(logfilename)
-    bb.msg.addDefaultlogFilter(consolelog)
     consolelog.setFormatter(conlogformat)
     logger.addHandler(consolelog)
 
@@ -136,7 +138,7 @@ def sigterm_handler(signum, frame):
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
+def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -192,9 +194,6 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
             global worker_pipe_lock
             pipein.close()
 
-            signal.signal(signal.SIGTERM, sigterm_handler)
-            # Let SIGHUP exit as SIGTERM
-            signal.signal(signal.SIGHUP, sigterm_handler)
             bb.utils.signal_on_parent_exit("SIGTERM")
 
             # Save out the PID so that the event can include it the
@@ -209,6 +208,11 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
             # This ensures signals sent to the controlling terminal like Ctrl+C
             # don't stop the child processes.
             os.setsid()
+
+            signal.signal(signal.SIGTERM, sigterm_handler)
+            # Let SIGHUP exit as SIGTERM
+            signal.signal(signal.SIGHUP, sigterm_handler)
+
             # No stdin
             newsi = os.open(os.devnull, os.O_RDWR)
             os.dup2(newsi, sys.stdin.fileno())
@@ -231,10 +235,13 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
                     the_data.setVar(varname, value)
 
                 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
+                if "newhashes" in workerdata:
+                    bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
                 ret = 0
 
                 the_data = bb_cache.loadDataFull(fn, appends)
-                the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task])
+                the_data.setVar('BB_TASKHASH', taskhash)
+                the_data.setVar('BB_UNIHASH', unihash)
 
                 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
 
@@ -373,6 +380,7 @@ class BitbakeWorker(object):
                 self.handle_item(b"cookerconfig", self.handle_cookercfg)
                 self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
                 self.handle_item(b"workerdata", self.handle_workerdata)
+                self.handle_item(b"newtaskhashes", self.handle_newtaskhashes)
                 self.handle_item(b"runtask", self.handle_runtask)
                 self.handle_item(b"finishnow", self.handle_finishnow)
                 self.handle_item(b"ping", self.handle_ping)
@@ -405,12 +413,16 @@ class BitbakeWorker(object):
 
     def handle_workerdata(self, data):
         self.workerdata = pickle.loads(data)
-        bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"]
+        bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"]
         bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"]
         bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"]
         bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
         for mc in self.databuilder.mcdata:
             self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
+            self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
+
+    def handle_newtaskhashes(self, data):
+        self.workerdata["newhashes"] = pickle.loads(data)
 
     def handle_ping(self, _):
         workerlog_write("Handling ping\n")
@@ -425,10 +437,10 @@ class BitbakeWorker(object):
         sys.exit(0)
 
     def handle_runtask(self, data):
-        fn, task, taskname, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
+        fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
 
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)

bitbake/bin/bitdoc

@@ -1,21 +1,9 @@
 #!/usr/bin/env python3
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2005 Holger Hans Peter Freyther
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import optparse, os, sys
 

bitbake/bin/git-make-shallow

@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
 """git-make-shallow: make the current git repository shallow
 
 Remove the history of the specified revisions, then optionally filter the

bitbake/bin/toaster

@@ -3,26 +3,18 @@
 # toaster - shell script to start Toaster
 
 # Copyright (C) 2013-2015 Intel Corp.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# SPDX-License-Identifier: GPL-2.0-or-later
 #
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see http://www.gnu.org/licenses/.
 
 HELP="
-Usage: source toaster start|stop [webport=<address:port>] [noweb] [nobuild]
+Usage 1: source toaster start|stop [webport=<address:port>] [noweb] [nobuild] [toasterdir]
     Optional arguments:
         [nobuild] Setup the environment for capturing builds with toaster but disable managed builds
         [noweb] Setup the environment for capturing builds with toaster but don't start the web server
         [webport] Set the development server (default: localhost:8000)
+        [toasterdir] Set absolute path to be used as TOASTER_DIR (default: BUILDDIR/../)
+Usage 2: source toaster manage [createsuperuser|lsupdates|migrate|makemigrations|checksettings|collectstatic|...]
 "
 
 custom_extention()
@@ -160,7 +152,9 @@ fi
 
 export BBBASEDIR=`dirname $TOASTER`/..
 MANAGE="python3 $BBBASEDIR/lib/toaster/manage.py"
-OE_ROOT=`dirname $TOASTER`/../..
+if [ -z "$OE_ROOT" ]; then
+    OE_ROOT=`dirname $TOASTER`/../..
+fi
 
 # this is the configuraton file we are using for toaster
 # we are using the same logic that oe-setup-builddir uses
@@ -186,6 +180,7 @@ unset OE_ROOT
 WEBSERVER=1
 export TOASTER_BUILDSERVER=1
 ADDR_PORT="localhost:8000"
+TOASTERDIR=`dirname $BUILDDIR`
 unset CMD
 for param in $*; do
     case $param in
@@ -211,13 +206,24 @@ for param in $*; do
                 ADDR_PORT="localhost:$PORT"
             fi
     ;;
+    toasterdir=*)
+            TOASTERDIR="${param#*=}"
+    ;;
+    manage )
+            CMD=$param
+            manage_cmd=""
+    ;;
     --help)
             echo "$HELP"
             return 0
     ;;
     *)
-            echo "$HELP"
-            return 1
+            if [ "manage" == "$CMD" ] ; then
+                manage_cmd="$manage_cmd $param"
+            else
+                echo "$HELP"
+                exit 1
+            fi
     ;;
 
     esac
@@ -241,7 +247,7 @@ fi
 # 2) the build dir (in build)
 # 3) the sqlite db if that is being used.
 # 4) pid's we need to clean up on exit/shutdown
-export TOASTER_DIR=`dirname $BUILDDIR`
+export TOASTER_DIR=$TOASTERDIR
 export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
 
 # Determine the action. If specified by arguments, fine, if not, toggle it
@@ -309,6 +315,10 @@ case $CMD in
         stop_system
         echo "Successful ${CMD}."
     ;;
+    manage )
+        cd $BBBASEDIR/lib/toaster
+        $MANAGE $manage_cmd
+    ;;
 esac
 custom_extention toaster_postpend $CMD $ADDR_PORT
 

bitbake/bin/toaster-eventreplay

@@ -1,25 +1,12 @@
 #!/usr/bin/env python3
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2014        Alex Damian
 #
+# SPDX-License-Identifier: GPL-2.0-only
+#
 # This file re-uses code spread throughout other Bitbake source files.
 # As such, all other copyrights belong to their own right holders.
 #
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 """
 This command takes a filename as a single parameter. The filename is read

bitbake/contrib/autobuilderlog.json

@@ -0,0 +1,13 @@
+{
+    "version": 1,
+    "loggers": {
+        "BitBake.SigGen.HashEquiv": {
+            "level": "VERBOSE",
+            "handlers": ["BitBake.verbconsole"]
+        },
+        "BitBake.RunQueue.HashEquiv": {
+            "level": "VERBOSE",
+            "handlers": ["BitBake.verbconsole"]
+        }
+    }
+}

bitbake/contrib/dump_cache.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python3
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2012, 2018 Wind River Systems, Inc.
 #

bitbake/contrib/vim/indent/bitbake.vim

@@ -0,0 +1,343 @@
+" Vim indent file
+" Language:             BitBake
+" Copyright:            Copyright (C) 2019 Agilent Technologies, Inc.
+" Maintainer:           Chris Laplante <chris.laplante@agilent.com>
+" License:              You may redistribute this under the same terms as Vim itself
+
+
+if exists("b:did_indent")
+    finish
+endif
+
+if exists("*BitbakeIndent")
+    finish
+endif
+
+runtime! indent/sh.vim
+unlet b:did_indent
+
+setlocal indentexpr=BitbakeIndent(v:lnum)
+setlocal autoindent nolisp
+
+function s:is_bb_python_func_def(lnum)
+    let stack = synstack(a:lnum, 1)
+    if len(stack) == 0
+        return 0
+    endif
+
+    let top = synIDattr(stack[0], "name")
+    echo top
+
+    return synIDattr(stack[0], "name") == "bbPyFuncDef"
+endfunction
+
+"""" begin modified from indent/python.vim, upstream commit 7a9bd7c1e0ce1baf5a02daf36eeae3638aa315c7
+"""" This copied code is licensed the same as Vim itself.
+setlocal indentkeys+=<:>,=elif,=except
+
+let s:keepcpo= &cpo
+set cpo&vim
+
+let s:maxoff = 50	" maximum number of lines to look backwards for ()
+
+function GetPythonIndent(lnum)
+
+  " If this line is explicitly joined: If the previous line was also joined,
+  " line it up with that one, otherwise add two 'shiftwidth'
+  if getline(a:lnum - 1) =~ '\\$'
+    if a:lnum > 1 && getline(a:lnum - 2) =~ '\\$'
+      return indent(a:lnum - 1)
+    endif
+    return indent(a:lnum - 1) + (exists("g:pyindent_continue") ? eval(g:pyindent_continue) : (shiftwidth() * 2))
+  endif
+
+  " If the start of the line is in a string don't change the indent.
+  if has('syntax_items')
+	\ && synIDattr(synID(a:lnum, 1, 1), "name") =~ "String$"
+    return -1
+  endif
+
+  " Search backwards for the previous non-empty line.
+  let plnum = prevnonblank(v:lnum - 1)
+
+  if plnum == 0
+    " This is the first non-empty line, use zero indent.
+    return 0
+  endif
+
+  call cursor(plnum, 1)
+
+  " Identing inside parentheses can be very slow, regardless of the searchpair()
+  " timeout, so let the user disable this feature if he doesn't need it
+  let disable_parentheses_indenting = get(g:, "pyindent_disable_parentheses_indenting", 0)
+
+  if disable_parentheses_indenting == 1
+    let plindent = indent(plnum)
+    let plnumstart = plnum
+  else
+    " searchpair() can be slow sometimes, limit the time to 150 msec or what is
+    " put in g:pyindent_searchpair_timeout
+    let searchpair_stopline = 0
+    let searchpair_timeout = get(g:, 'pyindent_searchpair_timeout', 150)
+
+    " If the previous line is inside parenthesis, use the indent of the starting
+    " line.
+    " Trick: use the non-existing "dummy" variable to break out of the loop when
+    " going too far back.
+    let parlnum = searchpair('(\|{\|\[', '', ')\|}\|\]', 'nbW',
+            \ "line('.') < " . (plnum - s:maxoff) . " ? dummy :"
+            \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+            \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+            \ searchpair_stopline, searchpair_timeout)
+    if parlnum > 0
+      " We may have found the opening brace of a BitBake Python task, e.g. 'python do_task {'
+      " If so, ignore it here - it will be handled later.
+      if s:is_bb_python_func_def(parlnum)
+        let parlnum = 0
+        let plindent = indent(plnum)
+        let plnumstart = plnum
+      else
+        let plindent = indent(parlnum)
+        let plnumstart = parlnum
+      endif
+    else
+      let plindent = indent(plnum)
+      let plnumstart = plnum
+    endif
+
+    " When inside parenthesis: If at the first line below the parenthesis add
+    " two 'shiftwidth', otherwise same as previous line.
+    " i = (a
+    "       + b
+    "       + c)
+    call cursor(a:lnum, 1)
+    let p = searchpair('(\|{\|\[', '', ')\|}\|\]', 'bW',
+            \ "line('.') < " . (a:lnum - s:maxoff) . " ? dummy :"
+            \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+            \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+            \ searchpair_stopline, searchpair_timeout)
+    if p > 0
+        if s:is_bb_python_func_def(p)
+          " Handle first non-empty line inside a BB Python task
+          if p == plnum
+              return shiftwidth()
+          endif
+
+          " Handle the user actually trying to close a BitBake Python task
+          let line = getline(a:lnum)
+          if line =~ '^\s*}'
+              return -2
+          endif
+
+          " Otherwise ignore the brace
+          let p = 0
+        else
+          if p == plnum
+            " When the start is inside parenthesis, only indent one 'shiftwidth'.
+            let pp = searchpair('(\|{\|\[', '', ')\|}\|\]', 'bW',
+                \ "line('.') < " . (a:lnum - s:maxoff) . " ? dummy :"
+                \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+                \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+                \ searchpair_stopline, searchpair_timeout)
+            if pp > 0
+              return indent(plnum) + (exists("g:pyindent_nested_paren") ? eval(g:pyindent_nested_paren) : shiftwidth())
+            endif
+            return indent(plnum) + (exists("g:pyindent_open_paren") ? eval(g:pyindent_open_paren) : (shiftwidth() * 2))
+          endif
+          if plnumstart == p
+            return indent(plnum)
+          endif
+          return plindent
+        endif
+    endif
+
+  endif
+
+
+  " Get the line and remove a trailing comment.
+  " Use syntax highlighting attributes when possible.
+  let pline = getline(plnum)
+  let pline_len = strlen(pline)
+  if has('syntax_items')
+    " If the last character in the line is a comment, do a binary search for
+    " the start of the comment.  synID() is slow, a linear search would take
+    " too long on a long line.
+    if synIDattr(synID(plnum, pline_len, 1), "name") =~ "\\(Comment\\|Todo\\)$"
+      let min = 1
+      let max = pline_len
+      while min < max
+	let col = (min + max) / 2
+	if synIDattr(synID(plnum, col, 1), "name") =~ "\\(Comment\\|Todo\\)$"
+	  let max = col
+	else
+	  let min = col + 1
+	endif
+      endwhile
+      let pline = strpart(pline, 0, min - 1)
+    endif
+  else
+    let col = 0
+    while col < pline_len
+      if pline[col] == '#'
+	let pline = strpart(pline, 0, col)
+	break
+      endif
+      let col = col + 1
+    endwhile
+  endif
+
+  " If the previous line ended with a colon, indent this line
+  if pline =~ ':\s*$'
+    return plindent + shiftwidth()
+  endif
+
+  " If the previous line was a stop-execution statement...
+  " TODO: utilize this logic to deindent when ending a bbPyDefRegion
+  if getline(plnum) =~ '^\s*\(break\|continue\|raise\|return\|pass\|bb\.fatal\)\>'
+    " See if the user has already dedented
+    if indent(a:lnum) > indent(plnum) - shiftwidth()
+      " If not, recommend one dedent
+      return indent(plnum) - shiftwidth()
+    endif
+    " Otherwise, trust the user
+    return -1
+  endif
+
+  " If the current line begins with a keyword that lines up with "try"
+  if getline(a:lnum) =~ '^\s*\(except\|finally\)\>'
+    let lnum = a:lnum - 1
+    while lnum >= 1
+      if getline(lnum) =~ '^\s*\(try\|except\)\>'
+	let ind = indent(lnum)
+	if ind >= indent(a:lnum)
+	  return -1	" indent is already less than this
+	endif
+	return ind	" line up with previous try or except
+      endif
+      let lnum = lnum - 1
+    endwhile
+    return -1		" no matching "try"!
+  endif
+
+  " If the current line begins with a header keyword, dedent
+  if getline(a:lnum) =~ '^\s*\(elif\|else\)\>'
+
+    " Unless the previous line was a one-liner
+    if getline(plnumstart) =~ '^\s*\(for\|if\|try\)\>'
+      return plindent
+    endif
+
+    " Or the user has already dedented
+    if indent(a:lnum) <= plindent - shiftwidth()
+      return -1
+    endif
+
+    return plindent - shiftwidth()
+  endif
+
+  " When after a () construct we probably want to go back to the start line.
+  " a = (b
+  "       + c)
+  " here
+  if parlnum > 0
+    return plindent
+  endif
+
+  return -1
+
+endfunction
+
+let &cpo = s:keepcpo
+unlet s:keepcpo
+
+""" end of stuff from indent/python.vim
+
+
+let b:did_indent = 1
+setlocal indentkeys+=0\"
+
+
+function BitbakeIndent(lnum)
+    if !has('syntax_items')
+        return -1
+    endif
+
+    let stack = synstack(a:lnum, 1)
+    if len(stack) == 0
+        return -1
+    endif
+
+    let name = synIDattr(stack[0], "name")
+
+    " TODO: support different styles of indentation for assignments. For now,
+    " we only support like this:
+    " VAR = " \
+    "     value1 \
+    "     value2 \
+    " "
+    "
+    " i.e. each value indented by shiftwidth(), with the final quote " completely unindented.
+    if name == "bbVarValue"
+        " Quote handling is tricky. kernel.bbclass has this line for instance:
+        "     EXTRA_OEMAKE = " HOSTCC="${BUILD_CC} ${BUILD_CFLAGS} ${BUILD_LDFLAGS}" " HOSTCPP="${BUILD_CPP}""
+        " Instead of trying to handle crazy cases like that, just assume that a
+        " double-quote on a line by itself (following an assignment) means the
+        " user is closing the assignment, and de-dent.
+        if getline(a:lnum) =~ '^\s*"$'
+            return 0
+        endif
+
+        let prevstack = synstack(a:lnum - 1, 1)
+        if len(prevstack) == 0
+            return -1
+        endif
+
+        let prevname = synIDattr(prevstack[0], "name")
+
+        " Only indent if there was actually a continuation character on
+        " the previous line, to avoid misleading indentation.
+        let prevlinelastchar = synIDattr(synID(a:lnum - 1, col([a:lnum - 1, "$"]) - 1, 1), "name")
+        let prev_continued = prevlinelastchar == "bbContinue"
+
+        " Did the previous line introduce an assignment?
+        if index(["bbVarDef", "bbVarFlagDef"], prevname) != -1
+            if prev_continued
+                return shiftwidth()
+            endif
+        endif
+
+        if !prev_continued
+            return 0
+        endif
+
+        " Autoindent can take it from here
+        return -1
+    endif
+
+    if index(["bbPyDefRegion", "bbPyFuncRegion"], name) != -1
+        let ret = GetPythonIndent(a:lnum)
+        " Should normally always be indented by at least one shiftwidth; but allow
+        " return of -1 (defer to autoindent) or -2 (force indent to 0)
+        if ret == 0
+            return shiftwidth()
+        elseif ret == -2
+            return 0
+        endif
+        return ret
+    endif
+
+    " TODO: GetShIndent doesn't detect tasks prepended with 'fakeroot'
+    " Need to submit a patch upstream to Vim to provide an extension point.
+    " Unlike the Python indenter, the Sh indenter is way too large to copy and
+    " modify here.
+    if name == "bbShFuncRegion"
+        return GetShIndent()
+    endif
+
+    " TODO:
+    "   + heuristics for de-denting out of a bbPyDefRegion? e.g. when the user
+    "       types an obvious BB keyword like addhandler or addtask, or starts
+    "       writing a shell task. Maybe too hard to implement...
+
+    return -1
+endfunction

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml

@@ -31,7 +31,7 @@
             <para>
                 Prior to executing BitBake, you should take advantage of available
                 parallel thread execution on your build host by setting the
-                <link linkend='var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
+                <link linkend='var-bb-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
                 variable in your project's <filename>local.conf</filename>
                 configuration file.
             </para>
@@ -87,9 +87,9 @@
         <para>
             The <filename>layer.conf</filename> files are used to
             construct key variables such as
-            <link linkend='var-BBPATH'><filename>BBPATH</filename></link>
+            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
             and
-            <link linkend='var-BBFILES'><filename>BBFILES</filename></link>.
+            <link linkend='var-bb-BBFILES'><filename>BBFILES</filename></link>.
             <filename>BBPATH</filename> is used to search for
             configuration and class files under the
             <filename>conf</filename> and <filename>classes</filename>
@@ -113,23 +113,23 @@
         </para>
 
         <para>
-            Prior to parsing configuration files, Bitbake looks
+            Prior to parsing configuration files, BitBake looks
             at certain variables, including:
             <itemizedlist>
                 <listitem><para>
-                    <link linkend='var-BB_ENV_WHITELIST'><filename>BB_ENV_WHITELIST</filename></link>
+                    <link linkend='var-bb-BB_ENV_WHITELIST'><filename>BB_ENV_WHITELIST</filename></link>
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_ENV_EXTRAWHITE'><filename>BB_ENV_EXTRAWHITE</filename></link>
+                    <link linkend='var-bb-BB_ENV_EXTRAWHITE'><filename>BB_ENV_EXTRAWHITE</filename></link>
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_PRESERVE_ENV'><filename>BB_PRESERVE_ENV</filename></link>
+                    <link linkend='var-bb-BB_PRESERVE_ENV'><filename>BB_PRESERVE_ENV</filename></link>
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_ORIGENV'><filename>BB_ORIGENV</filename></link>
+                    <link linkend='var-bb-BB_ORIGENV'><filename>BB_ORIGENV</filename></link>
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BITBAKE_UI'><filename>BITBAKE_UI</filename></link>
+                    <link linkend='var-bb-BITBAKE_UI'><filename>BITBAKE_UI</filename></link>
                     </para></listitem>
             </itemizedlist>
             The first four variables in this list relate to how BitBake treats shell
@@ -156,7 +156,7 @@
             BitBake first searches the current working directory for an
             optional <filename>conf/bblayers.conf</filename> configuration file.
             This file is expected to contain a
-            <link linkend='var-BBLAYERS'><filename>BBLAYERS</filename></link>
+            <link linkend='var-bb-BBLAYERS'><filename>BBLAYERS</filename></link>
             variable that is a space-delimited list of 'layer' directories.
             Recall that if BitBake cannot find a <filename>bblayers.conf</filename>
             file, then it is assumed the user has set the <filename>BBPATH</filename>
@@ -166,10 +166,10 @@
         <para>
             For each directory (layer) in this list, a <filename>conf/layer.conf</filename>
             file is located and parsed with the
-            <link linkend='var-LAYERDIR'><filename>LAYERDIR</filename></link>
+            <link linkend='var-bb-LAYERDIR'><filename>LAYERDIR</filename></link>
             variable being set to the directory where the layer was found.
             The idea is these files automatically set up
-            <link linkend='var-BBPATH'><filename>BBPATH</filename></link>
+            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
             and other variables correctly for a given build directory.
         </para>
 
@@ -189,7 +189,7 @@
             depending on the environment variables previously
             mentioned or set in the configuration files.
             The
-            "<link linkend='ref-variables-glos'>Variables Glossary</link>"
+            "<link linkend='ref-bb-variables-glos'>Variables Glossary</link>"
             chapter presents a full list of variables.
         </para>
 
@@ -204,7 +204,7 @@
         <para>
             The <filename>base.bbclass</filename> file is always included.
             Other classes that are specified in the configuration using the
-            <link linkend='var-INHERIT'><filename>INHERIT</filename></link>
+            <link linkend='var-bb-INHERIT'><filename>INHERIT</filename></link>
             variable are also included.
             BitBake searches for class files in a
             <filename>classes</filename> subdirectory under
@@ -270,7 +270,7 @@
 
         <para>
             During the configuration phase, BitBake will have set
-            <link linkend='var-BBFILES'><filename>BBFILES</filename></link>.
+            <link linkend='var-bb-BBFILES'><filename>BBFILES</filename></link>.
             BitBake now uses it to construct a list of recipes to parse,
             along with any append files (<filename>.bbappend</filename>)
             to apply.
@@ -292,7 +292,7 @@
             Any inherit statements cause BitBake to find and
             then parse class files (<filename>.bbclass</filename>)
             using
-            <link linkend='var-BBPATH'><filename>BBPATH</filename></link>
+            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
             as the search path.
             Finally, BitBake parses in order any append files found in
             <filename>BBFILES</filename>.
@@ -303,8 +303,8 @@
             pieces of metadata.
             For example, in <filename>bitbake.conf</filename> the recipe
             name and version are used to set the variables
-            <link linkend='var-PN'><filename>PN</filename></link> and
-            <link linkend='var-PV'><filename>PV</filename></link>:
+            <link linkend='var-bb-PN'><filename>PN</filename></link> and
+            <link linkend='var-bb-PV'><filename>PV</filename></link>:
             <literallayout class='monospaced'>
      PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
      PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
@@ -336,10 +336,10 @@
             recipe information.
             The validity of this cache is determined by first computing a
             checksum of the base configuration data (see
-            <link linkend='var-BB_HASHCONFIG_WHITELIST'><filename>BB_HASHCONFIG_WHITELIST</filename></link>)
+            <link linkend='var-bb-BB_HASHCONFIG_WHITELIST'><filename>BB_HASHCONFIG_WHITELIST</filename></link>)
             and then checking if the checksum matches.
             If that checksum matches what is in the cache and the recipe
-            and class files have not changed, Bitbake is able to use
+            and class files have not changed, BitBake is able to use
             the cache.
             BitBake then reloads the cached information about the recipe
             instead of reparsing it from scratch.
@@ -384,9 +384,9 @@
             the recipe can be known.
             Each recipe's <filename>PROVIDES</filename> list is created
             implicitly through the recipe's
-            <link linkend='var-PN'><filename>PN</filename></link> variable
+            <link linkend='var-bb-PN'><filename>PN</filename></link> variable
             and explicitly through the recipe's
-            <link linkend='var-PROVIDES'><filename>PROVIDES</filename></link>
+            <link linkend='var-bb-PROVIDES'><filename>PROVIDES</filename></link>
             variable, which is optional.
         </para>
 
@@ -427,9 +427,9 @@
      PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
             </literallayout>
             The default
-            <link linkend='var-PREFERRED_PROVIDER'><filename>PREFERRED_PROVIDER</filename></link>
+            <link linkend='var-bb-PREFERRED_PROVIDER'><filename>PREFERRED_PROVIDER</filename></link>
             is the provider with the same name as the target.
-            Bitbake iterates through each target it needs to build and
+            BitBake iterates through each target it needs to build and
             resolves them and their dependencies using this process.
         </para>
 
@@ -439,10 +439,10 @@
             BitBake defaults to the highest version of a provider.
             Version comparisons are made using the same method as Debian.
             You can use the
-            <link linkend='var-PREFERRED_VERSION'><filename>PREFERRED_VERSION</filename></link>
+            <link linkend='var-bb-PREFERRED_VERSION'><filename>PREFERRED_VERSION</filename></link>
             variable to specify a particular version.
             You can influence the order by using the
-            <link linkend='var-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
+            <link linkend='var-bb-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
             variable.
         </para>
 
@@ -464,7 +464,7 @@
             BitBake defaults to selecting the most recent
             version, unless otherwise specified.
             If the recipe in question has a
-            <link linkend='var-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
+            <link linkend='var-bb-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
             set lower than the other recipes (default is 0), then
             it will not be selected.
             This allows the person or persons maintaining
@@ -475,9 +475,9 @@
 
         <para>
             If the first recipe is named <filename>a_1.1.bb</filename>, then the
-            <link linkend='var-PN'><filename>PN</filename></link> variable
+            <link linkend='var-bb-PN'><filename>PN</filename></link> variable
             will be set to “a”, and the
-            <link linkend='var-PV'><filename>PV</filename></link>
+            <link linkend='var-bb-PV'><filename>PV</filename></link>
             variable will be set to 1.1.
         </para>
 
@@ -532,11 +532,11 @@
         <para>
             Dependencies are defined through several variables.
             You can find information about variables BitBake uses in
-            the <link linkend='ref-variables-glos'>Variables Glossary</link>
+            the <link linkend='ref-bb-variables-glos'>Variables Glossary</link>
             near the end of this manual.
             At a basic level, it is sufficient to know that BitBake uses the
-            <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link> and
-            <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link> variables when
+            <link linkend='var-bb-DEPENDS'><filename>DEPENDS</filename></link> and
+            <link linkend='var-bb-RDEPENDS'><filename>RDEPENDS</filename></link> variables when
             calculating dependencies.
         </para>
 
@@ -560,7 +560,7 @@
 
         <para>
             The build now starts with BitBake forking off threads up to the limit set in the
-            <link linkend='var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
+            <link linkend='var-bb-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
             variable.
             BitBake continues to fork threads as long as there are tasks ready to run,
             those tasks have all their dependencies met, and the thread threshold has not been
@@ -574,7 +574,7 @@
 
         <para>
             As each task completes, a timestamp is written to the directory specified by the
-            <link linkend='var-STAMP'><filename>STAMP</filename></link> variable.
+            <link linkend='var-bb-STAMP'><filename>STAMP</filename></link> variable.
             On subsequent runs, BitBake looks in the build directory within
             <filename>tmp/stamps</filename> and does not rerun
             tasks that are already completed unless a timestamp is found to be invalid.
@@ -618,12 +618,12 @@
         <para>
             Tasks can be either a shell task or a Python task.
             For shell tasks, BitBake writes a shell script to
-            <filename>${</filename><link linkend='var-T'><filename>T</filename></link><filename>}/run.do_taskname.pid</filename>
+            <filename>${</filename><link linkend='var-bb-T'><filename>T</filename></link><filename>}/run.do_taskname.<replaceable>pid</replaceable></filename>
             and then executes the script.
             The generated shell script contains all the exported variables,
             and the shell functions with all variables expanded.
             Output from the shell script goes to the file
-            <filename>${T}/log.do_taskname.pid</filename>.
+            <filename>${T}/log.do_taskname.<replaceable>pid</replaceable></filename>.
             Looking at the expanded shell functions in the run file and
             the output in the log files is a useful debugging technique.
         </para>
@@ -645,10 +645,10 @@
             behavior:
             <itemizedlist>
                 <listitem><para>
-                    <link linkend='var-BB_SCHEDULER'><filename>BB_SCHEDULER</filename></link>
+                    <link linkend='var-bb-BB_SCHEDULER'><filename>BB_SCHEDULER</filename></link>
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_SCHEDULERS'><filename>BB_SCHEDULERS</filename></link>
+                    <link linkend='var-bb-BB_SCHEDULERS'><filename>BB_SCHEDULERS</filename></link>
                     </para></listitem>
             </itemizedlist>
             It is possible to have functions run before and after a task's main
@@ -684,7 +684,7 @@
             The simplistic approach for excluding the working directory is to set
             it to some fixed value and create the checksum for the "run" script.
             BitBake goes one step better and uses the
-            <link linkend='var-BB_HASHBASE_WHITELIST'><filename>BB_HASHBASE_WHITELIST</filename></link>
+            <link linkend='var-bb-BB_HASHBASE_WHITELIST'><filename>BB_HASHBASE_WHITELIST</filename></link>
             variable to define a list of variables that should never be included
             when generating the signatures.
         </para>
@@ -795,7 +795,7 @@
             This results in any metadata change that changes the task hash, automatically
             causing the task to be run again.
             This removes the need to bump
-            <link linkend='var-PR'><filename>PR</filename></link>
+            <link linkend='var-bb-PR'><filename>PR</filename></link>
             values, and changes to metadata automatically ripple across the build.
         </para>
 
@@ -821,7 +821,7 @@
 
         <para>
             It is worth noting that BitBake's "-S" option lets you
-            debug Bitbake's processing of signatures.
+            debug BitBake's processing of signatures.
             The options passed to -S allow different debugging modes
             to be used, either using BitBake's own debug functions
             or possibly those defined in the metadata/signature handler
@@ -884,7 +884,7 @@
 
         <para>
             BitBake first calls the function defined by the
-            <link linkend='var-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></link>
+            <link linkend='var-bb-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></link>
             variable with a list of tasks and corresponding
             hashes it wants to build.
             This function is designed to be fast and returns a list
@@ -908,7 +908,7 @@
             For example, it is pointless to obtain a compiler if you
             already have the compiled binary.
             To handle this, BitBake calls the
-            <link linkend='var-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></link>
+            <link linkend='var-bb-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></link>
             function for each successful setscene task to know whether or not it needs
             to obtain the dependencies of that task.
         </para>
@@ -916,7 +916,7 @@
         <para>
             Finally, after all the setscene tasks have executed, BitBake calls the
             function listed in
-            <link linkend='var-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></link>
+            <link linkend='var-bb-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></link>
             with the list of tasks BitBake thinks has been "covered".
             The metadata can then ensure that this list is correct and can
             inform BitBake that it wants specific tasks to be run regardless
@@ -929,4 +929,101 @@
             section.
         </para>
     </section>
+
+    <section id="logging">
+        <title>Logging</title>
+        <para>
+            In addition to the standard command line option to control how
+            verbose builds are when execute, bitbake also supports user defined
+            configuration of the
+            <ulink url='https://docs.python.org/3/library/logging.html'>Python logging</ulink>
+            facilities through the
+            <link linkend="var-bb-BB_LOGCONFIG"><filename>BB_LOGCONFIG</filename></link>
+            variable. This variable defines a json or yaml
+            <ulink url='https://docs.python.org/3/library/logging.config.html'>logging configuration</ulink>
+            that will be intelligently merged into the default configuration.
+            The logging configuration is merged using the following rules:
+            <itemizedlist>
+                <listitem><para>
+                    The user defined configuration will completely replace the default
+                    configuration if top level key
+                    <filename>bitbake_merge</filename> is set to the value
+                    <filename>False</filename>. In this case, all other rules
+                    are ignored.
+                </para></listitem>
+                <listitem><para>
+                    The user configuration must have a top level
+                    <filename>version</filename> which must match the value of
+                    the default configuration.
+                </para></listitem>
+                <listitem><para>
+                    Any keys defined in the <filename>handlers</filename>,
+                    <filename>formatters</filename>, or <filename>filters</filename>,
+                    will be merged into the same section in the default
+                    configuration, with the user specified keys taking
+                    replacing a default one if there is a conflict. In
+                    practice, this means that if both the default configuration
+                    and user configuration specify a handler named
+                    <filename>myhandler</filename>, the user defined one will
+                    replace the default. To prevent the user from inadvertently
+                    replacing a default handler, formatter, or filter, all of
+                    the default ones are named with a prefix of
+                    "<filename>BitBake.</filename>"
+                </para></listitem>
+                <listitem><para>
+                    If a logger is defined by the user with the key
+                    <filename>bitbake_merge</filename> set to
+                    <filename>False</filename>, that logger will be completely
+                    replaced by user configuration. In this case, no other
+                    rules will apply to that logger.
+                </para></listitem>
+                <listitem><para>
+                    All user defined <filename>filter</filename> and
+                    <filename>handlers</filename> properties for a given logger
+                    will be merged with corresponding properties from the
+                    default logger. For example, if the user configuration adds
+                    a filter called <filename>myFilter</filename> to the
+                    <filename>BitBake.SigGen</filename>, and the default
+                    configuration adds a filter called
+                    <filename>BitBake.defaultFilter</filename>, both filters
+                    will be applied to the logger
+                </para></listitem>
+            </itemizedlist>
+        </para>
+
+        <para>
+            As an example, consider the following user logging configuration
+            file which logs all Hash Equivalence related messages of VERBOSE or
+            higher to a file called <filename>hashequiv.log</filename>
+            <literallayout class='monospaced'>
+    {
+        "version": 1,
+        "handlers": {
+            "autobuilderlog": {
+                "class": "logging.FileHandler",
+                "formatter": "logfileFormatter",
+                "level": "DEBUG",
+                "filename": "hashequiv.log",
+                "mode": "w"
+            }
+        },
+        "formatters": {
+                "logfileFormatter": {
+                    "format": "%(name)s: %(levelname)s: %(message)s"
+                }
+        },
+        "loggers": {
+            "BitBake.SigGen.HashEquiv": {
+                "level": "VERBOSE",
+                "handlers": ["autobuilderlog"]
+            },
+            "BitBake.RunQueue.HashEquiv": {
+                "level": "VERBOSE",
+                "handlers": ["autobuilderlog"]
+            }
+        }
+    }
+            </literallayout>
+        </para>
+    </section>
 </chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -44,7 +44,7 @@
             </literallayout>
             This code sets up an instance of the fetch class.
             The instance uses a space-separated list of URLs from the
-            <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>
+            <link linkend='var-bb-SRC_URI'><filename>SRC_URI</filename></link>
             variable and then calls the <filename>download</filename>
             method to download the files.
         </para>
@@ -78,7 +78,7 @@
                 <listitem><para><emphasis>Pre-mirror Sites:</emphasis>
                     BitBake first uses pre-mirrors to try and find source files.
                     These locations are defined using the
-                    <link linkend='var-PREMIRRORS'><filename>PREMIRRORS</filename></link>
+                    <link linkend='var-bb-PREMIRRORS'><filename>PREMIRRORS</filename></link>
                     variable.
                     </para></listitem>
                 <listitem><para><emphasis>Source URI:</emphasis>
@@ -88,7 +88,7 @@
                 <listitem><para><emphasis>Mirror Sites:</emphasis>
                     If fetch failures occur, BitBake next uses mirror locations as
                     defined by the
-                    <link linkend='var-MIRRORS'><filename>MIRRORS</filename></link>
+                    <link linkend='var-bb-MIRRORS'><filename>MIRRORS</filename></link>
                     variable.
                     </para></listitem>
             </itemizedlist>
@@ -139,12 +139,12 @@
         </para>
 
         <para>
-            Since network accesses are slow, Bitbake maintains a
+            Since network accesses are slow, BitBake maintains a
             cache of files downloaded from the network.
             Any source files that are not local (i.e.
             downloaded from the Internet) are placed into the download
             directory, which is specified by the
-            <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link>
+            <link linkend='var-bb-DL_DIR'><filename>DL_DIR</filename></link>
             variable.
         </para>
 
@@ -184,11 +184,11 @@
 
         <para>
             If
-            <link linkend='var-BB_STRICT_CHECKSUM'><filename>BB_STRICT_CHECKSUM</filename></link>
+            <link linkend='var-bb-BB_STRICT_CHECKSUM'><filename>BB_STRICT_CHECKSUM</filename></link>
             is set, any download without a checksum triggers an
             error message.
             The
-            <link linkend='var-BB_NO_NETWORK'><filename>BB_NO_NETWORK</filename></link>
+            <link linkend='var-bb-BB_NO_NETWORK'><filename>BB_NO_NETWORK</filename></link>
             variable can be used to make any attempted network access a fatal
             error, which is useful for checking that mirrors are complete
             as well as other things.
@@ -265,11 +265,11 @@
                 The filename you specify within the URL can be
                 either an absolute or relative path to a file.
                 If the filename is relative, the contents of the
-                <link linkend='var-FILESPATH'><filename>FILESPATH</filename></link>
+                <link linkend='var-bb-FILESPATH'><filename>FILESPATH</filename></link>
                 variable is used in the same way
                 <filename>PATH</filename> is used to find executables.
                 If the file cannot be found, it is assumed that it is available in
-                <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link>
+                <link linkend='var-bb-DL_DIR'><filename>DL_DIR</filename></link>
                 by the time the <filename>download()</filename> method is called.
             </para>
 
@@ -304,7 +304,7 @@
                 allows the name of the downloaded file to be specified.
                 Specifying the name of the downloaded file is useful
                 for avoiding collisions in
-                <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link>
+                <link linkend='var-bb-DL_DIR'><filename>DL_DIR</filename></link>
                 when dealing with multiple files that have the same name.
             </para>
 
@@ -355,7 +355,7 @@
                         A special value of "now" causes the checkout to
                         be updated on every build.
                         </para></listitem>
-                    <listitem><para><emphasis><link linkend='var-CVSDIR'><filename>CVSDIR</filename></link>:</emphasis>
+                    <listitem><para><emphasis><link linkend='var-bb-CVSDIR'><filename>CVSDIR</filename></link>:</emphasis>
                         Specifies where a temporary checkout is saved.
                         The location is often <filename>DL_DIR/cvs</filename>.
                         </para></listitem>
@@ -395,7 +395,7 @@
                     <listitem><para><emphasis>"date":</emphasis>
                         Specifies a date.
                         If no "date" is specified, the
-                        <link linkend='var-SRCDATE'><filename>SRCDATE</filename></link>
+                        <link linkend='var-bb-SRCDATE'><filename>SRCDATE</filename></link>
                         of the configuration is used to checkout a specific date.
                         The special value of "now" causes the checkout to be
                         updated on every build.
@@ -406,7 +406,7 @@
                         to which the module is unpacked.
                         You are forcing the module into a special
                         directory relative to
-                        <link linkend='var-CVSDIR'><filename>CVSDIR</filename></link>.
+                        <link linkend='var-bb-CVSDIR'><filename>CVSDIR</filename></link>.
                         </para></listitem>
                     <listitem><para><emphasis>"rsh"</emphasis>
                         Used in conjunction with the "method" parameter.
@@ -448,7 +448,7 @@
                 <filename>FETCHCMD_svn</filename>, which defaults
                 to "svn".
                 The fetcher's temporary working directory is set by
-                <link linkend='var-SVNDIR'><filename>SVNDIR</filename></link>,
+                <link linkend='var-bb-SVNDIR'><filename>SVNDIR</filename></link>,
                 which is usually <filename>DL_DIR/svn</filename>.
             </para>
 
@@ -509,7 +509,7 @@
                 source control system.
                 The fetcher works by creating a bare clone of the
                 remote into
-                <link linkend='var-GITDIR'><filename>GITDIR</filename></link>,
+                <link linkend='var-bb-GITDIR'><filename>GITDIR</filename></link>,
                 which is usually <filename>DL_DIR/git2</filename>.
                 This bare clone is then cloned into the work directory during the
                 unpack stage when a specific tree is checked out.
@@ -588,6 +588,14 @@
                         The name of the path in which to place the checkout.
                         By default, the path is <filename>git/</filename>.
                         </para></listitem>
+                    <listitem><para><emphasis>"usehead":</emphasis>
+                        Enables local <filename>git://</filename> URLs to use the
+                        current branch HEAD as the revision for use with
+                        <filename>AUTOREV</filename>.
+                        The "usehead" parameter implies no branch and only works
+                        when the transfer protocol is
+                        <filename>file://</filename>.
+                        </para></listitem>
                 </itemizedlist>
                 Here are some example URLs:
                 <literallayout class='monospaced'>
@@ -604,7 +612,7 @@
                 This fetcher submodule inherits from the
                 <link linkend='git-fetcher'>Git fetcher</link> and extends
                 that fetcher's behavior by fetching a repository's submodules.
-                <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>
+                <link linkend='var-bb-SRC_URI'><filename>SRC_URI</filename></link>
                 is passed to the Git fetcher as described in the
                 "<link linkend='git-fetcher'>Git Fetcher (<filename>git://</filename>)</link>"
                 section.
@@ -639,9 +647,9 @@
 
             <para>
                 To use this fetcher, make sure your recipe has proper
-                <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>,
-                <link linkend='var-SRCREV'><filename>SRCREV</filename></link>, and
-                <link linkend='var-PV'><filename>PV</filename></link> settings.
+                <link linkend='var-bb-SRC_URI'><filename>SRC_URI</filename></link>,
+                <link linkend='var-bb-SRCREV'><filename>SRCREV</filename></link>, and
+                <link linkend='var-bb-PV'><filename>PV</filename></link> settings.
                 Here is an example:
                 <literallayout class='monospaced'>
      SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
@@ -726,15 +734,15 @@
                 <filename>FETCHCMD_p4</filename>, which defaults
                 to "p4".
                 The fetcher's temporary working directory is set by
-                <link linkend='var-P4DIR'><filename>P4DIR</filename></link>,
+                <link linkend='var-bb-P4DIR'><filename>P4DIR</filename></link>,
                 which defaults to "DL_DIR/p4".
             </para>
 
             <para>
                 To use this fetcher, make sure your recipe has proper
-                <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>,
-                <link linkend='var-SRCREV'><filename>SRCREV</filename></link>, and
-                <link linkend='var-PV'><filename>PV</filename></link> values.
+                <link linkend='var-bb-SRC_URI'><filename>SRC_URI</filename></link>,
+                <link linkend='var-bb-SRCREV'><filename>SRCREV</filename></link>, and
+                <link linkend='var-bb-PV'><filename>PV</filename></link> values.
                 The p4 executable is able to use the config file defined by your
                 system's <filename>P4CONFIG</filename> environment variable in
                 order to define the Perforce server URL and port, username, and
@@ -785,9 +793,9 @@
                 <filename>google-repo</filename> source control system.
                 The fetcher works by initiating and syncing sources of the
                 repository into
-                <link linkend='var-REPODIR'><filename>REPODIR</filename></link>,
+                <link linkend='var-bb-REPODIR'><filename>REPODIR</filename></link>,
                 which is usually
-                <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link><filename>/repo</filename>.
+                <link linkend='var-bb-DL_DIR'><filename>DL_DIR</filename></link><filename>/repo</filename>.
             </para>
 
             <para>
@@ -824,7 +832,13 @@
                         Bazaar (<filename>bzr://</filename>)
                         </para></listitem>
                     <listitem><para>
-                        Trees using Git Annex (<filename>gitannex://</filename>)
+                        Mercurial (<filename>hg://</filename>)
+                        </para></listitem>
+                    <listitem><para>
+                        npm (<filename>npm://</filename>)
+                        </para></listitem>
+                    <listitem><para>
+                        OSC (<filename>osc://</filename>)
                         </para></listitem>
                     <listitem><para>
                         Secure FTP (<filename>sftp://</filename>)
@@ -833,10 +847,7 @@
                         Secure Shell (<filename>ssh://</filename>)
                         </para></listitem>
                     <listitem><para>
-                        OSC (<filename>osc://</filename>)
-                        </para></listitem>
-                    <listitem><para>
-                        Mercurial (<filename>hg://</filename>)
+                        Trees using Git Annex (<filename>gitannex://</filename>)
                         </para></listitem>
                 </itemizedlist>
                 No documentation currently exists for these lesser used

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml

@@ -166,7 +166,7 @@
                 Having a project directory is a good way to isolate your
                 project.
                 </para></listitem>
-            <listitem><para><emphasis>Run Bitbake:</emphasis>
+            <listitem><para><emphasis>Run BitBake:</emphasis>
                 At this point, you have nothing but a project directory.
                 Run the <filename>bitbake</filename> command and see what
                 it does:
@@ -194,10 +194,10 @@
                 <para>
                 When you run BitBake, it begins looking for metadata files.
                 The
-                <link linkend='var-BBPATH'><filename>BBPATH</filename></link>
+                <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
                 variable is what tells BitBake where to look for those files.
                 <filename>BBPATH</filename> is not set and you need to set it.
-                Without <filename>BBPATH</filename>, Bitbake cannot
+                Without <filename>BBPATH</filename>, BitBake cannot
                 find any configuration files (<filename>.conf</filename>)
                 or recipe files (<filename>.bb</filename>) at all.
                 BitBake also cannot find the <filename>bitbake.conf</filename>
@@ -225,7 +225,7 @@
                     as the shell would.
                 </note>
                 </para></listitem>
-            <listitem><para><emphasis>Run Bitbake:</emphasis>
+            <listitem><para><emphasis>Run BitBake:</emphasis>
                 Now that you have <filename>BBPATH</filename> defined, run
                 the <filename>bitbake</filename> command again:
                 <literallayout class='monospaced'>
@@ -273,14 +273,14 @@
                 some editor to create the <filename>bitbake.conf</filename>
                 so that it contains the following:
                 <literallayout class='monospaced'>
-     <link linkend='var-PN'>PN</link>  = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+     <link linkend='var-bb-PN'>PN</link>  = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
                 </literallayout>
                 <literallayout class='monospaced'>
-     TMPDIR  = "${<link linkend='var-TOPDIR'>TOPDIR</link>}/tmp"
-     <link linkend='var-CACHE'>CACHE</link>   = "${TMPDIR}/cache"
-     <link linkend='var-STAMP'>STAMP</link>   = "${TMPDIR}/${PN}/stamps"
-     <link linkend='var-T'>T</link>       = "${TMPDIR}/${PN}/work"
-     <link linkend='var-B'>B</link>       = "${TMPDIR}/${PN}"
+     TMPDIR  = "${<link linkend='var-bb-TOPDIR'>TOPDIR</link>}/tmp"
+     <link linkend='var-bb-CACHE'>CACHE</link>   = "${TMPDIR}/cache"
+     <link linkend='var-bb-STAMP'>STAMP</link>   = "${TMPDIR}/${PN}/stamps"
+     <link linkend='var-bb-T'>T</link>       = "${TMPDIR}/${PN}/work"
+     <link linkend='var-bb-B'>B</link>       = "${TMPDIR}/${PN}"
                 </literallayout>
                 <note>
                     Without a value for <filename>PN</filename>, the
@@ -313,7 +313,7 @@
                 example, click on the links to take you to the definitions in
                 the glossary.
                 </para></listitem>
-            <listitem><para><emphasis>Run Bitbake:</emphasis>
+            <listitem><para><emphasis>Run BitBake:</emphasis>
                 After making sure that the <filename>conf/bitbake.conf</filename>
                 file exists, you can run the <filename>bitbake</filename>
                 command again:
@@ -364,7 +364,7 @@
                 more depending on which build environments BitBake is
                 supporting.
                 </para></listitem>
-            <listitem><para><emphasis>Run Bitbake:</emphasis>
+            <listitem><para><emphasis>Run BitBake:</emphasis>
                 After making sure that the <filename>classes/base.bbclass</filename>
                 file exists, you can run the <filename>bitbake</filename>
                 command again:
@@ -402,12 +402,12 @@
                 Move to the <filename>conf</filename> directory and create a
                 <filename>layer.conf</filename> file that has the following:
                 <literallayout class='monospaced'>
-     BBPATH .= ":${<link linkend='var-LAYERDIR'>LAYERDIR</link>}"
+     BBPATH .= ":${<link linkend='var-bb-LAYERDIR'>LAYERDIR</link>}"
 
-     <link linkend='var-BBFILES'>BBFILES</link> += "${LAYERDIR}/*.bb"
+     <link linkend='var-bb-BBFILES'>BBFILES</link> += "${LAYERDIR}/*.bb"
 
-     <link linkend='var-BBFILE_COLLECTIONS'>BBFILE_COLLECTIONS</link> += "mylayer"
-     <link linkend='var-BBFILE_PATTERN'>BBFILE_PATTERN_mylayer</link> := "^${LAYERDIR_RE}/"
+     <link linkend='var-bb-BBFILE_COLLECTIONS'>BBFILE_COLLECTIONS</link> += "mylayer"
+     <link linkend='var-bb-BBFILE_PATTERN'>BBFILE_PATTERN_mylayer</link> := "^${LAYERDIR_RE}/"
                 </literallayout>
                 For information on these variables, click the links
                 to go to the definitions in the glossary.</para>
@@ -416,9 +416,9 @@
                 a recipe file named <filename>printhello.bb</filename> that
                 has the following:
                 <literallayout class='monospaced'>
-     <link linkend='var-DESCRIPTION'>DESCRIPTION</link> = "Prints Hello World"
-     <link linkend='var-PN'>PN</link> = 'printhello'
-     <link linkend='var-PV'>PV</link> = '1'
+     <link linkend='var-bb-DESCRIPTION'>DESCRIPTION</link> = "Prints Hello World"
+     <link linkend='var-bb-PN'>PN</link> = 'printhello'
+     <link linkend='var-bb-PV'>PV</link> = '1'
 
      python do_build() {
         bb.plain("********************");
@@ -434,7 +434,7 @@
                 For more information on these variables, follow the links
                 to the glossary.
                 </para></listitem>
-            <listitem><para><emphasis>Run Bitbake With a Target:</emphasis>
+            <listitem><para><emphasis>Run BitBake With a Target:</emphasis>
                 Now that a BitBake target exists, run the command and provide
                 that target:
                 <literallayout class='monospaced'>
@@ -468,7 +468,7 @@
                 You need to provide your own information for
                 <filename>you</filename> in the file.
                 </para></listitem>
-            <listitem><para><emphasis>Run Bitbake With a Target:</emphasis>
+            <listitem><para><emphasis>Run BitBake With a Target:</emphasis>
                 Now that you have supplied the <filename>bblayers.conf</filename>
                 file, run the <filename>bitbake</filename> command and provide
                 the target:

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -127,7 +127,7 @@
                     (e.g. Cygwin, the BSDs, and so forth).
                     </para></listitem>
                 <listitem><para>
-                    Be self contained, rather than tightly
+                    Be self-contained, rather than tightly
                     integrated into the build machine's root
                     filesystem.
                     </para></listitem>
@@ -221,6 +221,8 @@
                         them</para></listitem>
                     <listitem><para>How to configure and compile the
                         source code</para></listitem>
+                    <listitem><para>How to assemble the generated artifacts into
+                        one or more installable packages</para></listitem>
                     <listitem><para>Where on the target machine to install the
                         package or packages created</para></listitem>
                 </itemizedlist>
@@ -229,7 +231,7 @@
             <para>
                 Within the context of BitBake, or any project utilizing BitBake
                 as its build system, files with the <filename>.bb</filename>
-                extension are referred to as recipes.
+                extension are referred to as <firstterm>recipes</firstterm>.
                 <note>
                     The term "package" is also commonly used to describe recipes.
                     However, since the same word is used to describe packaged
@@ -252,9 +254,9 @@
                 various configuration variables that govern the project's build
                 process.
                 These files fall into several areas that define
-                machine configuration options, distribution configuration
-                options, compiler tuning options, general common
-                configuration options, and user configuration options.
+                machine configuration, distribution configuration,
+                possible compiler tuning, general common
+                configuration, and user configuration.
                 The main configuration file is the sample
                 <filename>bitbake.conf</filename> file, which is
                 located within the BitBake source tree
@@ -292,7 +294,7 @@
                 Layers allow you to isolate different types of
                 customizations from each other.
                 While you might find it tempting to keep everything in one layer
-                when working on a single project, the more modular you organize
+                when working on a single project, the more modular
                 your metadata, the easier it is to cope with future changes.
             </para>
 
@@ -300,8 +302,8 @@
                 To illustrate how you can use layers to keep things modular,
                 consider customizations you might make to support a specific target machine.
                 These types of customizations typically reside in a special layer,
-                rather than a general layer, called a Board Support Package (BSP)
-                Layer.
+                rather than a general layer, called a <firstterm>Board Support Package</firstterm> (BSP)
+                layer.
                 Furthermore, the machine customizations should be isolated from
                 recipes and metadata that support a new GUI environment, for
                 example.
@@ -342,13 +344,14 @@
 
             <para>
                 When you name an append file, you can use the
-                wildcard character (%) to allow for matching recipe names.
+                "<filename>%</filename>" wildcard character to allow for matching
+                recipe names.
                 For example, suppose you have an append file named
                 as follows:
                 <literallayout class='monospaced'>
      busybox_1.21.%.bbappend
                 </literallayout>
-                That append file would match any <filename>busybox_1.21.x.bb</filename>
+                That append file would match any <filename>busybox_1.21.</filename><replaceable>x</replaceable><filename>.bb</filename>
                 version of the recipe.
                 So, the append file would match the following recipe names:
                 <literallayout class='monospaced'>
@@ -356,6 +359,14 @@
      busybox_1.21.2.bb
      busybox_1.21.3.bb
                 </literallayout>
+                <note><title>Important</title>
+                    The use of the "<filename>%</filename>" character
+                    is limited in that it only works directly in front of the
+                    <filename>.bbappend</filename> portion of the append file's
+                    name.
+                    You cannot use the wildcard character in any other
+                    location of the name.
+                </note>
                 If the <filename>busybox</filename> recipe was updated to
                 <filename>busybox_1.3.0.bb</filename>, the append name would not
                 match.
@@ -439,7 +450,7 @@
                 <listitem><para><emphasis>Using the BitBake that Comes With Your
                     Build Checkout:</emphasis>
                     A final possibility for getting a copy of BitBake is that it
-                    already comes with your checkout of a larger Bitbake-based build
+                    already comes with your checkout of a larger BitBake-based build
                     system, such as Poky.
                     Rather than manually checking out individual layers and
                     gluing them together yourself, you can check
@@ -683,14 +694,9 @@
                 </para>
 
                 <para>
-                    When you generate a dependency graph, BitBake writes three files
+                    When you generate a dependency graph, BitBake writes two files
                     to the current working directory:
                     <itemizedlist>
-                        <listitem><para>
-                            <emphasis><filename>recipe-depends.dot</filename>:</emphasis>
-                            Shows dependencies between recipes (i.e. a collapsed version of
-                            <filename>task-depends.dot</filename>).
-                            </para></listitem>
                         <listitem><para>
                             <emphasis><filename>task-depends.dot</filename>:</emphasis>
                             Shows dependencies between tasks.
@@ -723,6 +729,163 @@
                     </literallayout>
                 </para>
             </section>
+
+            <section id='executing-a-multiple-configuration-build'>
+                <title>Executing a Multiple Configuration Build</title>
+
+                <para>
+                    BitBake is able to build multiple images or packages
+                    using a single command where the different targets
+                    require different configurations (multiple configuration
+                    builds).
+                    Each target, in this scenario, is referred to as a
+                    "multiconfig".
+                </para>
+
+                <para>
+                    To accomplish a multiple configuration build, you must
+                    define each target's configuration separately using
+                    a parallel configuration file in the build directory.
+                    The location for these multiconfig configuration files
+                    is specific.
+                    They must reside in the current build directory in
+                    a sub-directory of <filename>conf</filename> named
+                    <filename>multiconfig</filename>.
+                    Following is an example for two separate targets:
+                    <imagedata fileref="figures/bb_multiconfig_files.png" align="center" width="4in" depth="3in" />
+                </para>
+
+                <para>
+                    The reason for this required file hierarchy
+                    is because the <filename>BBPATH</filename> variable
+                    is not constructed until the layers are parsed.
+                    Consequently, using the configuration file as a
+                    pre-configuration file is not possible unless it is
+                    located in the current working directory.
+                </para>
+
+                <para>
+                    Minimally, each configuration file must define the
+                    machine and the temporary directory BitBake uses
+                    for the build.
+                    Suggested practice dictates that you do not
+                    overlap the temporary directories used during the
+                    builds.
+                </para>
+
+                <para>
+                    Aside from separate configuration files for each
+                    target, you must also enable BitBake to perform multiple
+                    configuration builds.
+                    Enabling is accomplished by setting the
+                    <link linkend='var-bb-BBMULTICONFIG'><filename>BBMULTICONFIG</filename></link>
+                    variable in the <filename>local.conf</filename>
+                    configuration file.
+                    As an example, suppose you had configuration files
+                    for <filename>target1</filename> and
+                    <filename>target2</filename> defined in the build
+                    directory.
+                    The following statement in the
+                    <filename>local.conf</filename> file both enables
+                    BitBake to perform multiple configuration builds and
+                    specifies the two extra multiconfigs:
+                    <literallayout class='monospaced'>
+     BBMULTICONFIG = "target1 target2"
+                    </literallayout>
+                </para>
+
+                <para>
+                    Once the target configuration files are in place and
+                    BitBake has been enabled to perform multiple configuration
+                    builds, use the following command form to start the
+                    builds:
+                    <literallayout class='monospaced'>
+     $ bitbake [mc:<replaceable>multiconfigname</replaceable>:]<replaceable>target</replaceable> [[[mc:<replaceable>multiconfigname</replaceable>:]<replaceable>target</replaceable>] ... ]
+                    </literallayout>
+                    Here is an example for two extra multiconfigs:
+                    <filename>target1</filename> and
+                    <filename>target2</filename>:
+                    <literallayout class='monospaced'>
+     $ bitbake mc::<replaceable>target</replaceable> mc:target1:<replaceable>target</replaceable> mc:target2:<replaceable>target</replaceable>
+                    </literallayout>
+                </para>
+            </section>
+
+            <section id='bb-enabling-multiple-configuration-build-dependencies'>
+                <title>Enabling Multiple Configuration Build Dependencies</title>
+
+                <para>
+                    Sometimes dependencies can exist between targets
+                    (multiconfigs) in a multiple configuration build.
+                    For example, suppose that in order to build an image
+                    for a particular architecture, the root filesystem of
+                    another build for a different architecture needs to
+                    exist.
+                    In other words, the image for the first multiconfig depends
+                    on the root filesystem of the second multiconfig.
+                    This dependency is essentially that the task in the recipe
+                    that builds one multiconfig is dependent on the
+                    completion of the task in the recipe that builds
+                    another multiconfig.
+                </para>
+
+                <para>
+                    To enable dependencies in a multiple configuration
+                    build, you must declare the dependencies in the recipe
+                    using the following statement form:
+                    <literallayout class='monospaced'>
+     <replaceable>task_or_package</replaceable>[mcdepends] = "mc:<replaceable>from_multiconfig</replaceable>:<replaceable>to_multiconfig</replaceable>:<replaceable>recipe_name</replaceable>:<replaceable>task_on_which_to_depend</replaceable>"
+                    </literallayout>
+                    To better show how to use this statement, consider an
+                    example with two multiconfigs: <filename>target1</filename>
+                    and <filename>target2</filename>:
+                    <literallayout class='monospaced'>
+     <replaceable>image_task</replaceable>[mcdepends] = "mc:target1:target2:<replaceable>image2</replaceable>:<replaceable>rootfs_task</replaceable>"
+                    </literallayout>
+                    In this example, the
+                    <replaceable>from_multiconfig</replaceable> is "target1" and
+                    the <replaceable>to_multiconfig</replaceable> is "target2".
+                    The task on which the image whose recipe contains
+                    <replaceable>image_task</replaceable> depends on the
+                    completion of the <replaceable>rootfs_task</replaceable>
+                    used to build out <replaceable>image2</replaceable>, which
+                    is associated with the "target2" multiconfig.
+               </para>
+
+               <para>
+                   Once you set up this dependency, you can build the
+                   "target1" multiconfig using a BitBake command as follows:
+                   <literallayout class='monospaced'>
+     $ bitbake mc:target1:<replaceable>image1</replaceable>
+                   </literallayout>
+                   This command executes all the tasks needed to create
+                   <replaceable>image1</replaceable> for the "target1"
+                   multiconfig.
+                   Because of the dependency, BitBake also executes through
+                   the <replaceable>rootfs_task</replaceable> for the "target2"
+                   multiconfig build.
+               </para>
+
+               <para>
+                   Having a recipe depend on the root filesystem of another
+                   build might not seem that useful.
+                   Consider this change to the statement in the
+                   <replaceable>image1</replaceable> recipe:
+                   <literallayout class='monospaced'>
+     <replaceable>image_task</replaceable>[mcdepends] = "mc:target1:target2:<replaceable>image2</replaceable>:<replaceable>image_task</replaceable>"
+                   </literallayout>
+                   In this case, BitBake must create
+                   <replaceable>image2</replaceable> for the "target2"
+                   build since the "target1" build depends on it.
+               </para>
+
+               <para>
+                   Because "target1" and "target2" are enabled for multiple
+                   configuration builds and have separate configuration
+                   files, BitBake places the artifacts for each build in the
+                   respective temporary build directories.
+               </para>
+            </section>
         </section>
     </section>
 </chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -5,7 +5,7 @@
     <title>Syntax and Operators</title>
 
     <para>
-        Bitbake files have their own syntax.
+        BitBake files have their own syntax.
         The syntax has similarities to several
         other languages but also has some unique features.
         This section describes the available syntax and operators
@@ -61,6 +61,78 @@
             </para>
         </section>
 
+        <section id='modifying-existing-variables'>
+            <title>Modifying Existing Variables</title>
+
+            <para>
+                Sometimes you need to modify existing variables.
+                Following are some cases where you might find you want to
+                modify an existing variable:
+                <itemizedlist>
+                    <listitem><para>
+                        Customize a recipe that uses the variable.
+                        </para></listitem>
+                    <listitem><para>
+                        Change a variable's default value used in a
+                        <filename>*.bbclass</filename> file.
+                        </para></listitem>
+                    <listitem><para>
+                        Change the variable in a <filename>*.bbappend</filename>
+                        file to override the variable in the original recipe.
+                        </para></listitem>
+                    <listitem><para>
+                        Change the variable in a configuration file so that the
+                        value overrides an existing configuration.
+                        </para></listitem>
+                </itemizedlist>
+            </para>
+
+            <para>
+                Changing a variable value can sometimes depend on how the
+                value was originally assigned and also on the desired
+                intent of the change.
+                In particular, when you append a value to a variable that
+                has a default value, the resulting value might not be what
+                you expect.
+                In this case, the value you provide might replace the value
+                rather than append to the default value.
+            </para>
+
+            <para>
+                If after you have changed a variable's value and something
+                unexplained occurs, you can use BitBake to check the actual
+                value of the suspect variable.
+                You can make these checks for both configuration and recipe
+                level changes:
+                <itemizedlist>
+                    <listitem><para>
+                        For configuration changes, use the following:
+                        <literallayout class='monospaced'>
+     $ bitbake -e
+                        </literallayout>
+                        This command displays variable values after the
+                        configuration files (i.e. <filename>local.conf</filename>,
+                        <filename>bblayers.conf</filename>,
+                        <filename>bitbake.conf</filename> and so forth) have
+                        been parsed.
+                        <note>
+                            Variables that are exported to the environment are
+                            preceded by the string "export" in the command's
+                            output.
+                        </note>
+                        </para></listitem>
+                    <listitem><para>
+                        For recipe changes, use the following:
+                        <literallayout class='monospaced'>
+     $ bitbake <replaceable>recipe</replaceable> -e | grep VARIABLE="
+                        </literallayout>
+                        This command checks to see if the variable actually
+                        makes it into a specific recipe.
+                        </para></listitem>
+                </itemizedlist>
+            </para>
+        </section>
+
         <section id='line-joining'>
             <title>Line Joining</title>
 
@@ -222,17 +294,20 @@
                 rather than when the variable is actually used:
                 <literallayout class='monospaced'>
      T = "123"
-     A := "${B} ${A} test ${T}"
+     A := "test ${T}"
      T = "456"
-     B = "${T} bval"
+     B := "${T} ${C}"
      C = "cval"
      C := "${C}append"
                 </literallayout>
                 In this example, <filename>A</filename> contains
-                "test 123" because <filename>${B}</filename> and
-                <filename>${A}</filename> at the time of parsing are undefined,
-                which leaves "test 123".
-                And, the variable <filename>C</filename>
+                "test 123", even though the final value of <filename>T</filename>
+                is "456".
+                The variable <filename>B</filename> will end up containing "456 cvalappend".
+                This is because references to undefined variables are preserved as is
+                during (immediate)expansion. This is in contrast to GNU Make, where undefined
+                variables expand to nothing.
+                The variable <filename>C</filename>
                 contains "cvalappend" since <filename>${C}</filename> immediately
                 expands to "cval".
             </para>
@@ -297,9 +372,8 @@
 
             <para>
                 These operators differ from the ":=", ".=", "=.", "+=", and "=+"
-                operators in that their effects are deferred
-                until after parsing completes rather than being immediately
-                applied.
+                operators in that their effects are applied at variable
+                expansion time rather than being immediately applied.
                 Here are some examples:
                 <literallayout class='monospaced'>
      B = "bval"
@@ -342,23 +416,28 @@
 
             <para>
                 When you use this syntax, BitBake expects one or more strings.
-                Surrounding spaces are removed as well.
+                Surrounding spaces and spacing are preserved.
                 Here is an example:
                 <literallayout class='monospaced'>
      FOO = "123 456 789 123456 123 456 123 456"
      FOO_remove = "123"
      FOO_remove = "456"
-     FOO2 = "abc def ghi abcdef abc def abc def"
-     FOO2_remove = "abc def"
+     FOO2 = " abc  def ghi abcdef abc def abc  def def"
+     FOO2_remove = " \
+         def \
+         abc \
+         ghi \
+     "
                 </literallayout>
                 The variable <filename>FOO</filename> becomes
-                "789 123456" and <filename>FOO2</filename> becomes
-                "ghi abcdef".
+                "&nbsp;&nbsp;789&nbsp;123456&nbsp;&nbsp;&nbsp;&nbsp;"
+                and <filename>FOO2</filename> becomes
+                "&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;abcdef&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;".
             </para>
 
             <para>
                 Like "_append" and "_prepend", "_remove"
-                is deferred until after parsing completes.
+                is applied at variable expansion time.
             </para>
         </section>
 
@@ -502,7 +581,7 @@
         </section>
 
         <section id='unsetting-variables'>
-            <title>Unseting variables</title>
+            <title>Unsetting variables</title>
 
             <para>
                 It is possible to completely remove a variable or a variable flag
@@ -594,7 +673,7 @@
 
         <para>
             BitBake uses
-            <link linkend='var-OVERRIDES'><filename>OVERRIDES</filename></link>
+            <link linkend='var-bb-OVERRIDES'><filename>OVERRIDES</filename></link>
             to control what variables are overridden after BitBake
             parses recipes and configuration files.
             This section describes how you can use
@@ -704,7 +783,7 @@
 
                         <para>Internally, this is implemented by prepending
                         the task (e.g. "task-compile:") to the value of
-                        <link linkend='var-OVERRIDES'><filename>OVERRIDES</filename></link>
+                        <link linkend='var-bb-OVERRIDES'><filename>OVERRIDES</filename></link>
                         for the local datastore of the <filename>do_compile</filename>
                         task.</para>
 
@@ -723,17 +802,15 @@
             <title>Key Expansion</title>
 
             <para>
-                Key expansion happens when the BitBake datastore is finalized
-                just before BitBake expands overrides.
+                Key expansion happens when the BitBake datastore is finalized.
                 To better understand this, consider the following example:
                 <literallayout class='monospaced'>
      A${B} = "X"
      B = "2"
      A2 = "Y"
                 </literallayout>
-                In this case, after all the parsing is complete, and
-                before any overrides are handled, BitBake expands
-                <filename>${B}</filename> into "2".
+                In this case, after all the parsing is complete,
+                BitBake expands <filename>${B}</filename> into "2".
                 This expansion causes <filename>A2</filename>, which was
                 set to "Y" before the expansion, to become "X".
             </para>
@@ -867,7 +944,7 @@
 
             <para>
                 BitBake uses the
-                <link linkend='var-BBPATH'><filename>BBPATH</filename></link>
+                <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
                 variable to locate needed include and class files.
                 Additionally, BitBake searches the current directory for
                 <filename>include</filename> and <filename>require</filename>
@@ -1085,7 +1162,7 @@
             <para>
                 When creating a configuration file (<filename>.conf</filename>),
                 you can use the
-                <link linkend='var-INHERIT'><filename>INHERIT</filename></link>
+                <link linkend='var-bb-INHERIT'><filename>INHERIT</filename></link>
                 configuration directive to inherit a class.
                 BitBake only supports this directive when used within
                 a configuration file.
@@ -1340,7 +1417,7 @@
         </section>
 
         <section id='bitbake-style-python-functions-versus-python-functions'>
-            <title>Bitbake-Style Python Functions Versus Python Functions</title>
+            <title>BitBake-Style Python Functions Versus Python Functions</title>
 
             <para>
                 Following are some important differences between
@@ -1369,7 +1446,7 @@
                         </para></listitem>
                     <listitem><para>
                         BitBake-style Python functions generate a separate
-                        <filename>${</filename><link linkend='var-T'><filename>T</filename></link><filename>}/run.</filename><replaceable>function-name</replaceable><filename>.</filename><replaceable>pid</replaceable>
+                        <filename>${</filename><link linkend='var-bb-T'><filename>T</filename></link><filename>}/run.</filename><replaceable>function-name</replaceable><filename>.</filename><replaceable>pid</replaceable>
                         script that is executed to run the function, and also
                         generate a log file in
                         <filename>${T}/log.</filename><replaceable>function-name</replaceable><filename>.</filename><replaceable>pid</replaceable>
@@ -1772,7 +1849,7 @@
                     things exported or listed in its whitelist to ensure that the build
                     environment is reproducible and consistent.
                     You can prevent this "cleaning" by setting the
-                    <link linkend='var-BB_PRESERVE_ENV'><filename>BB_PRESERVE_ENV</filename></link>
+                    <link linkend='var-bb-BB_PRESERVE_ENV'><filename>BB_PRESERVE_ENV</filename></link>
                     variable.
                 </note>
                 Consequently, if you do want something to get passed into the
@@ -1782,15 +1859,15 @@
                         Tell BitBake to load what you want from the environment
                         into the datastore.
                         You can do so through the
-                        <link linkend='var-BB_ENV_WHITELIST'><filename>BB_ENV_WHITELIST</filename></link>
+                        <link linkend='var-bb-BB_ENV_WHITELIST'><filename>BB_ENV_WHITELIST</filename></link>
                         and
-                        <link linkend='var-BB_ENV_EXTRAWHITE'><filename>BB_ENV_EXTRAWHITE</filename></link>
+                        <link linkend='var-bb-BB_ENV_EXTRAWHITE'><filename>BB_ENV_EXTRAWHITE</filename></link>
                         variables.
                         For example, assume you want to prevent the build system from
                         accessing your <filename>$HOME/.ccache</filename>
                         directory.
                         The following command "whitelists" the environment variable
-                        <filename>CCACHE_DIR</filename> causing BitBack to allow that
+                        <filename>CCACHE_DIR</filename> causing BitBake to allow that
                         variable into the datastore:
                         <literallayout class='monospaced'>
      export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE CCACHE_DIR"
@@ -1821,9 +1898,9 @@
             <para>
                 Sometimes, it is useful to be able to obtain information
                 from the original execution environment.
-                Bitbake saves a copy of the original environment into
+                BitBake saves a copy of the original environment into
                 a special variable named
-                <link linkend='var-BB_ORIGENV'><filename>BB_ORIGENV</filename></link>.
+                <link linkend='var-bb-BB_ORIGENV'><filename>BB_ORIGENV</filename></link>.
             </para>
 
             <para>
@@ -1882,7 +1959,7 @@
                 <listitem><para><emphasis><filename>[depends]</filename>:</emphasis>
                     Controls inter-task dependencies.
                     See the
-                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                    <link linkend='var-bb-DEPENDS'><filename>DEPENDS</filename></link>
                     variable and the
                     "<link linkend='inter-task-dependencies'>Inter-Task Dependencies</link>"
                     section for more information.
@@ -1890,7 +1967,7 @@
                 <listitem><para><emphasis><filename>[deptask]</filename>:</emphasis>
                     Controls task build-time dependencies.
                     See the
-                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                    <link linkend='var-bb-DEPENDS'><filename>DEPENDS</filename></link>
                     variable and the
                     "<link linkend='build-dependencies'>Build Dependencies</link>"
                     section for more information.
@@ -1929,6 +2006,38 @@
                         not careful.
                     </note>
                     </para></listitem>
+                <listitem><para><emphasis><filename>[number_threads]</filename>:</emphasis>
+                    Limits tasks to a specific number of simultaneous threads
+                    during execution.
+                    This varflag is useful when your build host has a large number
+                    of cores but certain tasks need to be rate-limited due to various
+                    kinds of resource constraints (e.g. to avoid network throttling).
+                    <filename>number_threads</filename> works similarly to the
+                    <link linkend='var-bb-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
+                    variable but is task-specific.</para>
+
+                    <para>Set the value globally.
+                    For example, the following makes sure the
+                    <filename>do_fetch</filename> task uses no more than two
+                    simultaneous execution threads:
+                    <literallayout class='monospaced'>
+     do_fetch[number_threads] = "2"
+                    </literallayout>
+                    <note><title>Warnings</title>
+                        <itemizedlist>
+                            <listitem><para>
+                                Setting the varflag in individual recipes rather
+                                than globally can result in unpredictable behavior.
+                                </para></listitem>
+                            <listitem><para>
+                                Setting the varflag to a value greater than the
+                                value used in the <filename>BB_NUMBER_THREADS</filename>
+                                variable causes <filename>number_threads</filename>
+                                to have no effect.
+                                </para></listitem>
+                        </itemizedlist>
+                    </note>
+                    </para></listitem>
                 <listitem><para><emphasis><filename>[postfuncs]</filename>:</emphasis>
                     List of functions to call after the completion of the task.
                     </para></listitem>
@@ -1938,9 +2047,9 @@
                 <listitem><para><emphasis><filename>[rdepends]</filename>:</emphasis>
                     Controls inter-task runtime dependencies.
                     See the
-                    <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
+                    <link linkend='var-bb-RDEPENDS'><filename>RDEPENDS</filename></link>
                     variable, the
-                    <link linkend='var-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
+                    <link linkend='var-bb-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
                     variable, and the
                     "<link linkend='inter-task-dependencies'>Inter-Task Dependencies</link>"
                     section for more information.
@@ -1948,9 +2057,9 @@
                 <listitem><para><emphasis><filename>[rdeptask]</filename>:</emphasis>
                     Controls task runtime dependencies.
                     See the
-                    <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
+                    <link linkend='var-bb-RDEPENDS'><filename>RDEPENDS</filename></link>
                     variable, the
-                    <link linkend='var-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
+                    <link linkend='var-bb-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
                     variable, and the
                     "<link linkend='runtime-dependencies'>Runtime Dependencies</link>"
                     section for more information.
@@ -1963,9 +2072,9 @@
                 <listitem><para><emphasis><filename>[recrdeptask]</filename>:</emphasis>
                     Controls task recursive runtime dependencies.
                     See the
-                    <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
+                    <link linkend='var-bb-RDEPENDS'><filename>RDEPENDS</filename></link>
                     variable, the
-                    <link linkend='var-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
+                    <link linkend='var-bb-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
                     variable, and the
                     "<link linkend='recursive-dependencies'>Recursive Dependencies</link>"
                     section for more information.
@@ -2094,7 +2203,7 @@
                     Any given datastore only has one such event executed
                     against it, however.
                     If
-                    <link linkende='var-BB_INVALIDCONF'><filename>BB_INVALIDCONF</filename></link>
+                    <link linkende='var-bb-BB_INVALIDCONF'><filename>BB_INVALIDCONF</filename></link>
                     is set in the datastore by the event handler, the
                     configuration is reparsed and a new event triggered,
                     allowing the metadata to update configuration.
@@ -2223,17 +2332,17 @@
             from a single recipe file multiple incarnations of that
             recipe file where all incarnations are buildable.
             These features are enabled through the
-            <link linkend='var-BBCLASSEXTEND'><filename>BBCLASSEXTEND</filename></link>
+            <link linkend='var-bb-BBCLASSEXTEND'><filename>BBCLASSEXTEND</filename></link>
             and
-            <link linkend='var-BBVERSIONS'><filename>BBVERSIONS</filename></link>
+            <link linkend='var-bb-BBVERSIONS'><filename>BBVERSIONS</filename></link>
             variables.
             <note>
                 The mechanism for this class extension is extremely
                 specific to the implementation.
                 Usually, the recipe's
-                <link linkend='var-PROVIDES'><filename>PROVIDES</filename></link>,
-                <link linkend='var-PN'><filename>PN</filename></link>, and
-                <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                <link linkend='var-bb-PROVIDES'><filename>PROVIDES</filename></link>,
+                <link linkend='var-bb-PN'><filename>PN</filename></link>, and
+                <link linkend='var-bb-DEPENDS'><filename>DEPENDS</filename></link>
                 variables would need to be modified by the extension class.
                 For specific examples, see the OE-Core
                 <filename>native</filename>, <filename>nativesdk</filename>,
@@ -2254,7 +2363,7 @@
                     project from a single recipe file.
                     You can also specify conditional metadata
                     (using the
-                    <link linkend='var-OVERRIDES'><filename>OVERRIDES</filename></link>
+                    <link linkend='var-bb-OVERRIDES'><filename>OVERRIDES</filename></link>
                     mechanism) for a single version, or an optionally named range of versions.
                     Here is an example:
                     <literallayout class='monospaced'>
@@ -2273,7 +2382,7 @@
                     into overrides, but it is also made available for the metadata to use
                     in the variable that defines the base recipe versions for use in
                     <filename>file://</filename> search paths
-                    (<link linkend='var-FILESPATH'><filename>FILESPATH</filename></link>).
+                    (<link linkend='var-bb-FILESPATH'><filename>FILESPATH</filename></link>).
                     </para></listitem>
             </itemizedlist>
         </para>
@@ -2375,7 +2484,7 @@
 
             <para>
                 BitBake uses the
-                <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                <link linkend='var-bb-DEPENDS'><filename>DEPENDS</filename></link>
                 variable to manage build time dependencies.
                 The <filename>[deptask]</filename> varflag for tasks
                 signifies the task of each
@@ -2396,9 +2505,9 @@
 
             <para>
                 BitBake uses the
-                <link linkend='var-PACKAGES'><filename>PACKAGES</filename></link>,
-                <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>, and
-                <link linkend='var-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
+                <link linkend='var-bb-PACKAGES'><filename>PACKAGES</filename></link>,
+                <link linkend='var-bb-RDEPENDS'><filename>RDEPENDS</filename></link>, and
+                <link linkend='var-bb-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
                 variables to manage runtime dependencies.
             </para>
 
@@ -2417,6 +2526,9 @@
                 In the previous example, the <filename>do_packagedata</filename>
                 task of each item in <filename>RDEPENDS</filename> must have
                 completed before <filename>do_package_qa</filename> can execute.
+                Although <filename>RDEPENDS</filename> contains entries from the
+                runtime dependency namespace, BitBake knows how to map them back
+                to the build-time dependency namespace, in which the tasks are defined.
             </para>
         </section>
 
@@ -2453,15 +2565,17 @@
             </para>
 
             <para>
-                You might want to not only have BitBake look for
-                dependencies of those tasks, but also have BitBake look
-                for build-time and runtime dependencies of the dependent
-                tasks as well.
-                If that is the case, you need to reference the task name
-                itself in the task list:
+                BitBake allows a task to recursively depend on itself by
+                referencing itself in the task list:
                 <literallayout class='monospaced'>
      do_a[recrdeptask] = "do_a do_b"
                 </literallayout>
+                In the same way as before, this means that the <filename>do_a</filename>
+                and <filename>do_b</filename> tasks of the current recipe and all
+                recipes reachable (by way of dependencies) from the recipe
+                must run before the <filename>do_a</filename> task can run. In this
+                case BitBake will ignore the current recipe's <filename>do_a</filename>
+                task circular dependency on itself.
             </para>
         </section>
 
@@ -2510,7 +2624,7 @@
             <para>
                 It is often necessary to access variables in the
                 BitBake datastore using Python functions.
-                The Bitbake datastore has an API that allows you this
+                The BitBake datastore has an API that allows you this
                 access.
                 Here is a list of available operations:
             </para>
@@ -2653,7 +2767,7 @@
 
         <para>
             These checksums are stored in
-            <link linkend='var-STAMP'><filename>STAMP</filename></link>.
+            <link linkend='var-bb-STAMP'><filename>STAMP</filename></link>.
             You can examine the checksums using the following BitBake command:
             <literallayout class='monospaced'>
      $ bitbake-dumpsigs
@@ -2675,48 +2789,74 @@
             The following list describes related variables:
             <itemizedlist>
                 <listitem><para>
-                    <link linkend='var-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></link>:
+                    <link linkend='var-bb-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></link>:
                     Specifies the name of the function to call during
                     the "setscene" part of the task's execution in order
                     to validate the list of task hashes.
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></link>:
+                    <link linkend='var-bb-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></link>:
                     Specifies a function BitBake calls that determines
                     whether BitBake requires a setscene dependency to
                     be met.
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></link>:
+                    <link linkend='var-bb-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></link>:
                     Specifies a function to call that verifies the list of
                     planned task execution before the main task execution
                     happens.
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_STAMP_POLICY'><filename>BB_STAMP_POLICY</filename></link>:
+                    <link linkend='var-bb-BB_STAMP_POLICY'><filename>BB_STAMP_POLICY</filename></link>:
                     Defines the mode for comparing timestamps of stamp files.
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_STAMP_WHITELIST'><filename>BB_STAMP_WHITELIST</filename></link>:
+                    <link linkend='var-bb-BB_STAMP_WHITELIST'><filename>BB_STAMP_WHITELIST</filename></link>:
                     Lists stamp files that are looked at when the stamp policy
                     is "whitelist".
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-BB_TASKHASH'><filename>BB_TASKHASH</filename></link>:
+                    <link linkend='var-bb-BB_TASKHASH'><filename>BB_TASKHASH</filename></link>:
                     Within an executing task, this variable holds the hash
                     of the task as returned by the currently enabled
                     signature generator.
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-STAMP'><filename>STAMP</filename></link>:
+                    <link linkend='var-bb-STAMP'><filename>STAMP</filename></link>:
                     The base path to create stamp files.
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='var-STAMPCLEAN'><filename>STAMPCLEAN</filename></link>:
+                    <link linkend='var-bb-STAMPCLEAN'><filename>STAMPCLEAN</filename></link>:
                     Again, the base path to create stamp files but can use wildcards
                     for matching a range of files for clean operations.
                     </para></listitem>
             </itemizedlist>
         </para>
     </section>
+
+    <section id='wildcard-support-in-variables'>
+        <title>Wildcard Support in Variables</title>
+
+        <para>
+            Support for wildcard use in variables varies depending on the
+            context in which it is used.
+            For example, some variables and file names allow limited use of
+            wildcards through the "<filename>%</filename>" and
+            "<filename>*</filename>" characters.
+            Other variables or names support Python's
+            <ulink url='https://docs.python.org/3/library/glob.html'><filename>glob</filename></ulink>
+            syntax,
+            <ulink url='https://docs.python.org/3/library/fnmatch.html#module-fnmatch'><filename>fnmatch</filename></ulink>
+            syntax, or
+            <ulink url='https://docs.python.org/3/library/re.html#re'><filename>Regular Expression (re)</filename></ulink>
+            syntax.
+        </para>
+
+        <para>
+            For variables that have wildcard suport, the
+            documentation describes which form of wildcard, its
+            use, and its limitations.
+        </para>
+    </section>
+
 </chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml

@@ -56,7 +56,7 @@
 -->
 
         <copyright>
-            <year>2004-2017</year>
+            <year>2004-2018</year>
             <holder>Richard Purdie</holder>
             <holder>Chris Larson</holder>
             <holder>and Phil Blundell</holder>

bitbake/doc/poky.ent

@@ -17,13 +17,6 @@
 <!ENTITY OE_DOCS_URL "http://docs.openembedded.org">
 <!ENTITY OH_HOME_URL "http://o-hand.com">
 <!ENTITY BITBAKE_HOME_URL "http://developer.berlios.de/projects/bitbake/">
-<!ENTITY ECLIPSE_MAIN_URL "http://www.eclipse.org/downloads">
-<!ENTITY ECLIPSE_DL_URL "http://download.eclipse.org">
-<!ENTITY ECLIPSE_DL_PLUGIN_URL "&YOCTO_DL_URL;/releases/eclipse-plugin/&DISTRO;">
-<!ENTITY ECLIPSE_UPDATES_URL "&ECLIPSE_DL_URL;/tm/updates/3.3">
-<!ENTITY ECLIPSE_INDIGO_URL "&ECLIPSE_DL_URL;/releases/indigo">
-<!ENTITY ECLIPSE_JUNO_URL "&ECLIPSE_DL_URL;/releases/juno">
-<!ENTITY ECLIPSE_INDIGO_CDT_URL "&ECLIPSE_DL_URL;tools/cdt/releases/indigo">
 <!ENTITY YOCTO_DOCS_URL "&YOCTO_HOME_URL;/docs">
 <!ENTITY YOCTO_SOURCES_URL "&YOCTO_HOME_URL;/sources/">
 <!ENTITY YOCTO_AB_PORT_URL "&YOCTO_AB_URL;:8010">
@@ -31,7 +24,6 @@
 <!ENTITY YOCTO_POKY_URL "&YOCTO_DL_URL;/releases/poky/">
 <!ENTITY YOCTO_RELEASE_DL_URL "&YOCTO_DL_URL;/releases/yocto/yocto-&DISTRO;">
 <!ENTITY YOCTO_TOOLCHAIN_DL_URL "&YOCTO_RELEASE_DL_URL;/toolchain/">
-<!ENTITY YOCTO_ECLIPSE_DL_URL "&YOCTO_RELEASE_DL_URL;/eclipse-plugin/indigo;">
 <!ENTITY YOCTO_ADTINSTALLER_DL_URL "&YOCTO_RELEASE_DL_URL;/adt_installer">
 <!ENTITY YOCTO_POKY_DL_URL "&YOCTO_RELEASE_DL_URL;/&YOCTO_POKY;.tar.bz2">
 <!ENTITY YOCTO_MACHINES_DL_URL "&YOCTO_RELEASE_DL_URL;/machines">

bitbake/lib/bb/COW.py

@@ -1,23 +1,8 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # This is a copy on write dictionary and set which abuses classes to try and be nice and fast.
 #
 # Copyright (C) 2006 Tim Ansell
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
 #Please Note:
 # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW.
 # Assign a file to __warn__ to get warnings about slow operations.
@@ -25,7 +10,6 @@
 
 
 import copy
-import types
 ImmutableTypes = (
     bool,
     complex,
@@ -150,7 +134,7 @@ class COWDictMeta(COWMeta):
                 yield value
             if type == "items":
                 yield (key, value)
-        raise StopIteration()
+        return
 
     def iterkeys(cls):
         return cls.iter("keys")

bitbake/lib/bb/__init__.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # BitBake Build System Python Library
 #
@@ -8,24 +6,14 @@
 #
 # Based on Gentoo's portage.py.
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.37.0"
+__version__ = "1.46.0"
 
 import sys
-if sys.version_info < (3, 4, 0):
-    raise RuntimeError("Sorry, python 3.4.0 or later is required for this version of bitbake")
+if sys.version_info < (3, 5, 0):
+    raise RuntimeError("Sorry, python 3.5.0 or later is required for this version of bitbake")
 
 
 class BBHandledException(Exception):
@@ -55,7 +43,13 @@ class BBLogger(Logger):
         Logger.__init__(self, name)
 
     def bbdebug(self, level, msg, *args, **kwargs):
-        return self.log(logging.DEBUG - level + 1, msg, *args, **kwargs)
+        loglevel = logging.DEBUG - level + 1
+        if not bb.event.worker_pid:
+            if self.name in bb.msg.loggerDefaultDomains and loglevel > (bb.msg.loggerDefaultDomains[self.name]):
+                return
+            if loglevel > bb.msg.loggerDefaultLogLevel:
+                return
+        return self.log(loglevel, msg, *args, **kwargs)
 
     def plain(self, msg, *args, **kwargs):
         return self.log(logging.INFO + 1, msg, *args, **kwargs)
@@ -63,6 +57,10 @@ class BBLogger(Logger):
     def verbose(self, msg, *args, **kwargs):
         return self.log(logging.INFO - 1, msg, *args, **kwargs)
 
+    def verbnote(self, msg, *args, **kwargs):
+        return self.log(logging.INFO + 2, msg, *args, **kwargs)
+
+
 logging.raiseExceptions = False
 logging.setLoggerClass(BBLogger)
 
@@ -93,6 +91,18 @@ def debug(lvl, *args):
 def note(*args):
     mainlogger.info(''.join(args))
 
+#
+# A higher prioity note which will show on the console but isn't a warning
+#
+# Something is happening the user should be aware of but they probably did
+# something to make it happen
+#
+def verbnote(*args):
+    mainlogger.verbnote(''.join(args))
+
+#
+# Warnings - things the user likely needs to pay attention to and fix
+#
 def warn(*args):
     mainlogger.warning(''.join(args))
 

bitbake/lib/bb/build.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # BitBake 'Build' implementation
 #
@@ -10,25 +8,13 @@
 #
 # Based on Gentoo's portage.py.
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import os
 import sys
 import logging
-import shlex
 import glob
 import time
 import stat
@@ -41,8 +27,6 @@ from bb import data, event, utils
 bblogger = logging.getLogger('BitBake')
 logger = logging.getLogger('BitBake.Build')
 
-NULL = open(os.devnull, 'r+')
-
 __mtime_cache = {}
 
 def cached_mtime_noerror(f):
@@ -69,34 +53,20 @@ else:
 builtins['bb'] = bb
 builtins['os'] = os
 
-class FuncFailed(Exception):
-    def __init__(self, name = None, logfile = None):
-        self.logfile = logfile
-        self.name = name
-        if name:
-            self.msg = 'Function failed: %s' % name
-        else:
-            self.msg = "Function failed"
-
-    def __str__(self):
-        if self.logfile and os.path.exists(self.logfile):
-            msg = ("%s (log file is located at %s)" %
-                   (self.msg, self.logfile))
-        else:
-            msg = self.msg
-        return msg
-
 class TaskBase(event.Event):
     """Base class for task events"""
 
-    def __init__(self, t, logfile, d):
+    def __init__(self, t, fn, logfile, d):
         self._task = t
+        self._fn = fn
         self._package = d.getVar("PF")
         self._mc = d.getVar("BB_CURRENT_MC")
         self.taskfile = d.getVar("FILE")
         self.taskname = self._task
         self.logfile = logfile
         self.time = time.time()
+        self.pn = d.getVar("PN")
+        self.pv = d.getVar("PV")
         event.Event.__init__(self)
         self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName())
 
@@ -113,8 +83,8 @@ class TaskBase(event.Event):
 
 class TaskStarted(TaskBase):
     """Task execution started"""
-    def __init__(self, t, logfile, taskflags, d):
-        super(TaskStarted, self).__init__(t, logfile, d)
+    def __init__(self, t, fn, logfile, taskflags, d):
+        super(TaskStarted, self).__init__(t, fn, logfile, d)
         self.taskflags = taskflags
 
 class TaskSucceeded(TaskBase):
@@ -123,9 +93,9 @@ class TaskSucceeded(TaskBase):
 class TaskFailed(TaskBase):
     """Task execution failed"""
 
-    def __init__(self, task, logfile, metadata, errprinted = False):
+    def __init__(self, task, fn, logfile, metadata, errprinted = False):
         self.errprinted = errprinted
-        super(TaskFailed, self).__init__(task, logfile, metadata)
+        super(TaskFailed, self).__init__(task, fn, logfile, metadata)
 
 class TaskFailedSilent(TaskBase):
     """Task execution failed (silently)"""
@@ -135,8 +105,8 @@ class TaskFailedSilent(TaskBase):
 
 class TaskInvalid(TaskBase):
 
-    def __init__(self, task, metadata):
-        super(TaskInvalid, self).__init__(task, None, metadata)
+    def __init__(self, task, fn, metadata):
+        super(TaskInvalid, self).__init__(task, fn, None, metadata)
         self._message = "No such task '%s'" % task
 
 class TaskProgress(event.Event):
@@ -178,15 +148,33 @@ class LogTee(object):
 
     def __repr__(self):
         return '<LogTee {0}>'.format(self.name)
+
     def flush(self):
         self.outfile.flush()
 
-#
-# pythonexception allows the python exceptions generated to be raised
-# as the real exceptions (not FuncFailed) and without a backtrace at the 
-# origin of the failure.
-#
-def exec_func(func, d, dirs = None, pythonexception=False):
+
+class StdoutNoopContextManager:
+    """
+    This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods.
+    """
+    def __enter__(self):
+        return sys.stdout
+
+    def __exit__(self, *exc_info):
+        pass
+
+    def write(self, string):
+        return sys.stdout.write(string)
+
+    def flush(self):
+        sys.stdout.flush()
+
+    @property
+    def name(self):
+        return sys.stdout.name
+
+
+def exec_func(func, d, dirs = None):
     """Execute a BB 'function'"""
 
     try:
@@ -258,7 +246,7 @@ def exec_func(func, d, dirs = None, pythonexception=False):
 
     with bb.utils.fileslocked(lockfiles):
         if ispython:
-            exec_func_python(func, d, runfile, cwd=adir, pythonexception=pythonexception)
+            exec_func_python(func, d, runfile, cwd=adir)
         else:
             exec_func_shell(func, d, runfile, cwd=adir)
 
@@ -278,7 +266,7 @@ _functionfmt = """
 {function}(d)
 """
 logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-def exec_func_python(func, d, runfile, cwd=None, pythonexception=False):
+def exec_func_python(func, d, runfile, cwd=None):
     """Execute a python BB 'function'"""
 
     code = _functionfmt.format(function=func)
@@ -303,13 +291,7 @@ def exec_func_python(func, d, runfile, cwd=None, pythonexception=False):
         bb.methodpool.insert_method(func, text, fn, lineno - 1)
 
         comp = utils.better_compile(code, func, "exec_python_func() autogenerated")
-        utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated", pythonexception=pythonexception)
-    except (bb.parse.SkipRecipe, bb.build.FuncFailed):
-        raise
-    except:
-        if pythonexception:
-            raise
-        raise FuncFailed(func, None)
+        utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated")
     finally:
         bb.debug(2, "Python function %s finished" % func)
 
@@ -337,6 +319,42 @@ trap 'bb_exit_handler' 0
 set -e
 '''
 
+def create_progress_handler(func, progress, logfile, d):
+    if progress == 'percent':
+        # Use default regex
+        return bb.progress.BasicProgressHandler(d, outfile=logfile)
+    elif progress.startswith('percent:'):
+        # Use specified regex
+        return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+    elif progress.startswith('outof:'):
+        # Use specified regex
+        return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+    elif progress.startswith("custom:"):
+        # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__
+        import functools
+        from types import ModuleType
+
+        parts = progress.split(":", 2)
+        _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None
+        if cls:
+            def resolve(x, y):
+                if not x:
+                    return None
+                if isinstance(x, ModuleType):
+                    return getattr(x, y, None)
+                return x.get(y)
+            cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context)
+            if not cls_obj:
+                # Fall-back on __builtins__
+                cls_obj = functools.reduce(lambda x, y: x.get(y), cls.split("."), __builtins__)
+            if cls_obj:
+                return cls_obj(d, outfile=logfile, otherargs=otherargs)
+            bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls))
+    else:
+        bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
+
+    return logfile
+
 def exec_func_shell(func, d, runfile, cwd=None):
     """Execute a shell function from the metadata
 
@@ -374,23 +392,13 @@ exit $ret
             cmd = [fakerootcmd, runfile]
 
     if bb.msg.loggerDefaultVerbose:
-        logfile = LogTee(logger, sys.stdout)
+        logfile = LogTee(logger, StdoutNoopContextManager())
     else:
-        logfile = sys.stdout
+        logfile = StdoutNoopContextManager()
 
     progress = d.getVarFlag(func, 'progress')
     if progress:
-        if progress == 'percent':
-            # Use default regex
-            logfile = bb.progress.BasicProgressHandler(d, outfile=logfile)
-        elif progress.startswith('percent:'):
-            # Use specified regex
-            logfile = bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
-        elif progress.startswith('outof:'):
-            # Use specified regex
-            logfile = bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
-        else:
-            bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
+        logfile = create_progress_handler(func, progress, logfile, d)
 
     fifobuffer = bytearray()
     def readfifo(data):
@@ -409,6 +417,8 @@ exit $ret
                     bb.plain(value)
                 elif cmd == 'bbnote':
                     bb.note(value)
+                elif cmd == 'bbverbnote':
+                    bb.verbnote(value)
                 elif cmd == 'bbwarn':
                     bb.warn(value)
                 elif cmd == 'bberror':
@@ -438,13 +448,8 @@ exit $ret
     with open(fifopath, 'r+b', buffering=0) as fifo:
         try:
             bb.debug(2, "Executing shell function %s" % func)
-
-            try:
-                with open(os.devnull, 'r+') as stdin:
-                    bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
-            except bb.process.CmdError:
-                logfn = d.getVar('BB_LOGFILE')
-                raise FuncFailed(func, logfn)
+            with open(os.devnull, 'r+') as stdin, logfile:
+                bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
         finally:
             os.unlink(fifopath)
 
@@ -533,7 +538,6 @@ def _exec_task(fn, task, d, quieterr):
                 self.triggered = True
 
     # Handle logfiles
-    si = open('/dev/null', 'r')
     try:
         bb.utils.mkdirhier(os.path.dirname(logfn))
         logfile = open(logfn, 'w')
@@ -547,7 +551,8 @@ def _exec_task(fn, task, d, quieterr):
     ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
 
     # Replace those fds with our own
-    os.dup2(si.fileno(), osi[1])
+    with open('/dev/null', 'r') as si:
+        os.dup2(si.fileno(), osi[1])
     os.dup2(logfile.fileno(), oso[1])
     os.dup2(logfile.fileno(), ose[1])
 
@@ -569,12 +574,9 @@ def _exec_task(fn, task, d, quieterr):
 
     try:
         try:
-            event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
+            event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata)
         except (bb.BBHandledException, SystemExit):
             return 1
-        except FuncFailed as exc:
-            logger.error(str(exc))
-            return 1
 
         try:
             for func in (prefuncs or '').split():
@@ -582,16 +584,16 @@ def _exec_task(fn, task, d, quieterr):
             exec_func(task, localdata)
             for func in (postfuncs or '').split():
                 exec_func(func, localdata)
-        except FuncFailed as exc:
+        except bb.BBHandledException:
+            event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata)
+            return 1
+        except Exception as exc:
             if quieterr:
-                event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
+                event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
             else:
                 errprinted = errchk.triggered
                 logger.error(str(exc))
-                event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
-            return 1
-        except bb.BBHandledException:
-            event.fire(TaskFailed(task, logfn, localdata, True), localdata)
+                event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
             return 1
     finally:
         sys.stdout.flush()
@@ -608,14 +610,13 @@ def _exec_task(fn, task, d, quieterr):
         os.close(osi[0])
         os.close(oso[0])
         os.close(ose[0])
-        si.close()
 
         logfile.close()
         if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
             logger.debug(2, "Zero size logfn %s, removing", logfn)
             bb.utils.remove(logfn)
             bb.utils.remove(loglink)
-    event.fire(TaskSucceeded(task, logfn, localdata), localdata)
+    event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata)
 
     if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
         make_stamp(task, localdata)
@@ -803,6 +804,7 @@ def add_tasks(tasklist, d):
             if name in flags:
                 deptask = d.expand(flags[name])
                 task_deps[name][task] = deptask
+        getTask('mcdepends')
         getTask('depends')
         getTask('rdepends')
         getTask('deptask')
@@ -816,6 +818,9 @@ def add_tasks(tasklist, d):
         task_deps['parents'][task] = []
         if 'deps' in flags:
             for dep in flags['deps']:
+                # Check and warn for "addtask task after foo" while foo does not exist
+                #if not dep in tasklist:
+                #    bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task))
                 dep = d.expand(dep)
                 task_deps['parents'][task].append(dep)
 

bitbake/lib/bb/cache.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # BitBake Cache implementation
 #
@@ -15,29 +13,19 @@
 # Copyright (C) 2005        Holger Hans Peter Freyther
 # Copyright (C) 2005        ROAD GmbH
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
-import sys
 import logging
 import pickle
 from collections import defaultdict
 import bb.utils
+import re
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "151"
+__cache_version__ = "152"
 
 def getCacheFile(path, filename, data_hash):
     return os.path.join(path, filename + "." + data_hash)
@@ -95,20 +83,20 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.appends = self.listvar('__BBAPPEND', metadata)
         self.nocache = self.getvar('BB_DONT_CACHE', metadata)
 
-        self.skipreason = self.getvar('__SKIPPED', metadata)
-        if self.skipreason:
-            self.pn = self.getvar('PN', metadata) or bb.parse.BBHandler.vars_from_file(filename,metadata)[0]
-            self.skipped = True
-            self.provides  = self.depvar('PROVIDES', metadata)
-            self.rprovides = self.depvar('RPROVIDES', metadata)
-            return
-
-        self.tasks = metadata.getVar('__BBTASKS', False)
-
-        self.pn = self.getvar('PN', metadata)
+        self.provides  = self.depvar('PROVIDES', metadata)
+        self.rprovides = self.depvar('RPROVIDES', metadata)
+        self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0]
         self.packages = self.listvar('PACKAGES', metadata)
         if not self.packages:
             self.packages.append(self.pn)
+        self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
+
+        self.skipreason = self.getvar('__SKIPPED', metadata)
+        if self.skipreason:
+            self.skipped = True
+            return
+
+        self.tasks = metadata.getVar('__BBTASKS', False)
 
         self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
         self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
@@ -125,11 +113,8 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.stampclean = self.getvar('STAMPCLEAN', metadata)
         self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
         self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True)
-        self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
         self.depends          = self.depvar('DEPENDS', metadata)
-        self.provides         = self.depvar('PROVIDES', metadata)
         self.rdepends         = self.depvar('RDEPENDS', metadata)
-        self.rprovides        = self.depvar('RPROVIDES', metadata)
         self.rrecommends      = self.depvar('RRECOMMENDS', metadata)
         self.rprovides_pkg    = self.pkgvar('RPROVIDES', self.packages, metadata)
         self.rdepends_pkg     = self.pkgvar('RDEPENDS', self.packages, metadata)
@@ -223,10 +208,10 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         # Collect files we may need for possible world-dep
         # calculations
-        if self.not_world:
-            logger.debug(1, "EXCLUDE FROM WORLD: %s", fn)
-        else:
+        if not self.not_world:
             cachedata.possible_world.append(fn)
+        #else:
+        #    logger.debug(2, "EXCLUDE FROM WORLD: %s", fn)
 
         # create a collection of all targets for sanity checking
         # tasks, such as upstream versions, license, and tools for
@@ -235,7 +220,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         cachedata.hashfn[fn] = self.hashfilename
         for task, taskhash in self.basetaskhashes.items():
-            identifier = '%s.%s' % (fn, task)
+            identifier = '%s:%s' % (fn, task)
             cachedata.basetaskhash[identifier] = taskhash
 
         cachedata.inherits[fn] = self.inherits
@@ -249,7 +234,7 @@ def virtualfn2realfn(virtualfn):
     Convert a virtual file name to a real one + the associated subclass keyword
     """
     mc = ""
-    if virtualfn.startswith('multiconfig:'):
+    if virtualfn.startswith('mc:'):
         elems = virtualfn.split(':')
         mc = elems[1]
         virtualfn = ":".join(elems[2:])
@@ -270,7 +255,7 @@ def realfn2virtual(realfn, cls, mc):
     if cls:
         realfn = "virtual:" + cls + ":" + realfn
     if mc:
-        realfn = "multiconfig:" + mc + ":" + realfn
+        realfn = "mc:" + mc + ":" + realfn
     return realfn
 
 def variant2virtual(realfn, variant):
@@ -279,11 +264,11 @@ def variant2virtual(realfn, variant):
     """
     if variant == "":
         return realfn
-    if variant.startswith("multiconfig:"):
+    if variant.startswith("mc:"):
         elems = variant.split(":")
         if elems[2]:
-            return "multiconfig:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
-        return "multiconfig:" + elems[1] + ":" + realfn
+            return "mc:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
+        return "mc:" + elems[1] + ":" + realfn
     return "virtual:" + variant + ":" + realfn
 
 def parse_recipe(bb_data, bbfile, appends, mc=''):
@@ -361,7 +346,7 @@ class NoCache(object):
             bb_data = self.databuilder.mcdata[mc].createCopy()
             newstores = parse_recipe(bb_data, bbfile, appends, mc)
             for ns in newstores:
-                datastores["multiconfig:%s:%s" % (mc, ns)] = newstores[ns]
+                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
 
         return datastores
 
@@ -385,6 +370,7 @@ class Cache(NoCache):
         self.data_fn = None
         self.cacheclean = True
         self.data_hash = data_hash
+        self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
 
         if self.cachedir in [None, '']:
             self.has_cache = False
@@ -411,6 +397,15 @@ class Cache(NoCache):
         else:
             logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
 
+        # We don't use the symlink, its just for debugging convinience
+        symlink = os.path.join(self.cachedir, "bb_cache.dat")
+        if os.path.exists(symlink):
+            bb.utils.remove(symlink)
+        try:
+            os.symlink(os.path.basename(self.cachefile), symlink)
+        except OSError:
+            pass
+
     def load_cachefile(self):
         cachesize = 0
         previous_progress = 0
@@ -614,20 +609,12 @@ class Cache(NoCache):
         if hasattr(info_array[0], 'file_checksums'):
             for _, fl in info_array[0].file_checksums.items():
                 fl = fl.strip()
-                while fl:
-                    # A .split() would be simpler but means spaces or colons in filenames would break
-                    a = fl.find(":True")
-                    b = fl.find(":False")
-                    if ((a < 0) and b) or ((b > 0) and (b < a)):
-                        f = fl[:b+6]
-                        fl = fl[b+7:]
-                    elif ((b < 0) and a) or ((a > 0) and (a < b)):
-                        f = fl[:a+5]
-                        fl = fl[a+6:]
-                    else:
-                        break
-                    fl = fl.strip()
-                    if "*" in f:
+                if not fl:
+                    continue
+                # Have to be careful about spaces and colons in filenames
+                flist = self.filelist_regex.split(fl)
+                for f in flist:
+                    if not f or "*" in f:
                         continue
                     f, exist = f.split(":")
                     if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
@@ -889,3 +876,56 @@ class MultiProcessCache(object):
             p.dump([data, self.__class__.CACHE_VERSION])
 
         bb.utils.unlockfile(glf)
+
+
+class SimpleCache(object):
+    """
+    BitBake multi-process cache implementation
+
+    Used by the codeparser & file checksum caches
+    """
+
+    def __init__(self, version):
+        self.cachefile = None
+        self.cachedata = None
+        self.cacheversion = version
+
+    def init_cache(self, d, cache_file_name=None, defaultdata=None):
+        cachedir = (d.getVar("PERSISTENT_DIR") or
+                    d.getVar("CACHE"))
+        if not cachedir:
+            return defaultdata
+
+        bb.utils.mkdirhier(cachedir)
+        self.cachefile = os.path.join(cachedir,
+                                      cache_file_name or self.__class__.cache_file_name)
+        logger.debug(1, "Using cache in '%s'", self.cachefile)
+
+        glf = bb.utils.lockfile(self.cachefile + ".lock")
+
+        try:
+            with open(self.cachefile, "rb") as f:
+                p = pickle.Unpickler(f)
+                data, version = p.load()
+        except:
+            bb.utils.unlockfile(glf)
+            return defaultdata
+
+        bb.utils.unlockfile(glf)
+
+        if version != self.cacheversion:
+            return defaultdata
+
+        return data
+
+    def save(self, data):
+        if not self.cachefile:
+            return
+
+        glf = bb.utils.lockfile(self.cachefile + ".lock")
+
+        with open(self.cachefile, "wb") as f:
+            p = pickle.Pickler(f, -1)
+            p.dump([data, self.cacheversion])
+
+        bb.utils.unlockfile(glf)

bitbake/lib/bb/cache_extra.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Extra RecipeInfo will be all defined in this file. Currently,
 # Only Hob (Image Creator) Requests some extra fields. So
@@ -12,18 +10,8 @@
 
 # Copyright (C) 2011, Intel Corporation. All rights reserved.
 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from bb.cache import RecipeInfoCommon
 

bitbake/lib/bb/checksum.py

@@ -2,24 +2,13 @@
 #
 # Copyright (C) 2012 Intel Corporation
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import glob
 import operator
 import os
 import stat
-import pickle
 import bb.utils
 import logging
 from bb.cache import MultiProcessCache
@@ -84,7 +73,7 @@ class FileChecksumCache(MultiProcessCache):
             else:
                 dest[0][h] = source[0][h]
 
-    def get_checksums(self, filelist, pn):
+    def get_checksums(self, filelist, pn, localdirsexclude):
         """Get checksums for a list of files"""
 
         def checksum_file(f):
@@ -97,8 +86,11 @@ class FileChecksumCache(MultiProcessCache):
 
         def checksum_dir(pth):
             # Handle directories recursively
+            if pth == "/":
+                bb.fatal("Refusing to checksum /")
             dirchecksums = []
-            for root, dirs, files in os.walk(pth):
+            for root, dirs, files in os.walk(pth, topdown=True):
+                [dirs.remove(d) for d in list(dirs) if d in localdirsexclude]
                 for name in files:
                     fullpth = os.path.join(root, name)
                     checksum = checksum_file(fullpth)

bitbake/lib/bb/codeparser.py

@@ -1,3 +1,7 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
 """
 BitBake code parser
 
@@ -21,19 +25,17 @@ import ast
 import sys
 import codegen
 import logging
-import pickle
 import bb.pysh as pysh
-import os.path
 import bb.utils, bb.data
 import hashlib
 from itertools import chain
-from bb.pysh import pyshyacc, pyshlex, sherrors
+from bb.pysh import pyshyacc, pyshlex
 from bb.cache import MultiProcessCache
 
 logger = logging.getLogger('BitBake.CodeParser')
 
 def bbhash(s):
-    return hashlib.md5(s.encode("utf-8")).hexdigest()
+    return hashlib.sha256(s.encode("utf-8")).hexdigest()
 
 def check_indent(codestr):
     """If the code is indented, add a top level piece of code to 'remove' the indentation"""
@@ -54,30 +56,10 @@ def check_indent(codestr):
 
     return codestr
 
-
-# Basically pickle, in python 2.7.3 at least, does badly with data duplication 
-# upon pickling and unpickling. Combine this with duplicate objects and things
-# are a mess.
-#
-# When the sets are originally created, python calls intern() on the set keys
-# which significantly improves memory usage. Sadly the pickle/unpickle process
-# doesn't call intern() on the keys and results in the same strings being duplicated
-# in memory. This also means pickle will save the same string multiple times in
-# the cache file.
-#
-# By having shell and python cacheline objects with setstate/getstate, we force
-# the object creation through our own routine where we can call intern (via internSet).
-#
-# We also use hashable frozensets and ensure we use references to these so that
-# duplicates can be removed, both in memory and in the resulting pickled data.
-#
-# By playing these games, the size of the cache file shrinks dramatically
-# meaning faster load times and the reloaded cache files also consume much less
-# memory. Smaller cache files, faster load times and lower memory usage is good.
-#
 # A custom getstate/setstate using tuples is actually worth 15% cachesize by
 # avoiding duplication of the attribute names!
 
+
 class SetCache(object):
     def __init__(self):
         self.setcache = {}
@@ -140,7 +122,7 @@ class CodeParserCache(MultiProcessCache):
     # so that an existing cache gets invalidated. Additionally you'll need
     # to increment __cache_version__ in cache.py in order to ensure that old
     # recipe caches don't trigger "Taskhash mismatch" errors.
-    CACHE_VERSION = 9
+    CACHE_VERSION = 11
 
     def __init__(self):
         MultiProcessCache.__init__(self)
@@ -214,7 +196,7 @@ class BufferedLogger(Logger):
         self.buffer = []
 
 class PythonParser():
-    getvars = (".getVar", ".appendVar", ".prependVar")
+    getvars = (".getVar", ".appendVar", ".prependVar", "oe.utils.conditional")
     getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag")
     containsfuncs = ("bb.utils.contains", "base_contains")
     containsanyfuncs = ("bb.utils.contains_any",  "bb.utils.filter")
@@ -368,8 +350,9 @@ class ShellParser():
     def _parse_shell(self, value):
         try:
             tokens, _ = pyshyacc.parse(value, eof=True, debug=False)
-        except pyshlex.NeedMore:
-            raise sherrors.ShellSyntaxError("Unexpected EOF")
+        except Exception:
+            bb.error('Error during parse shell code, the last 5 lines are:\n%s' % '\n'.join(value.split('\n')[-5:]))
+            raise
 
         self.process_tokens(tokens)
 

bitbake/lib/bb/command.py

@@ -6,18 +6,8 @@ Provide an interface to interact with the bitbake server through 'commands'
 
 # Copyright (C) 2006-2007  Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 """
 The bitbake server takes 'commands' from its UI/commandline.
@@ -75,7 +65,7 @@ class Command:
             # Can run synchronous commands straight away
             command_method = getattr(self.cmds_sync, command)
             if ro_only:
-                if not hasattr(command_method, 'readonly') or False == getattr(command_method, 'readonly'):
+                if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'):
                     return None, "Not able to execute not readonly commands in readonly mode"
             try:
                 self.cooker.process_inotify_updates()
@@ -424,7 +414,11 @@ class CommandsSync:
     getAllAppends.readonly = True
 
     def findProviders(self, command, params):
-        return command.cooker.findProviders()
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.findProviders(mc)
     findProviders.readonly = True
 
     def findBestProvider(self, command, params):
@@ -456,54 +450,38 @@ class CommandsSync:
         return all_p, best
     getRuntimeProviders.readonly = True
 
-    def dataStoreConnectorFindVar(self, command, params):
+    def dataStoreConnectorCmd(self, command, params):
         dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        value, overridedata = datastore._findVar(name)
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-        if value:
-            content = value.get('_content', None)
-            if isinstance(content, bb.data_smart.DataSmart):
-                # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
-                idx = command.remotedatastores.check_store(content, True)
-                return {'_content': DataStoreConnectionHandle(idx),
-                        '_connector_origtype': 'DataStoreConnectionHandle',
-                        '_connector_overrides': overridedata}
-            elif isinstance(content, set):
-                return {'_content': list(content),
-                        '_connector_origtype': 'set',
-                        '_connector_overrides': overridedata}
-            else:
-                value['_connector_overrides'] = overridedata
-        else:
-            value = {}
-            value['_connector_overrides'] = overridedata
-        return value
-    dataStoreConnectorFindVar.readonly = True
+        d = command.remotedatastores[dsindex]
+        ret = getattr(d, method)(*args, **kwargs)
 
-    def dataStoreConnectorGetKeys(self, command, params):
+        if isinstance(ret, bb.data_smart.DataSmart):
+            idx = command.remotedatastores.store(ret)
+            return DataStoreConnectionHandle(idx)
+
+        return ret
+
+    def dataStoreConnectorVarHistCmd(self, command, params):
         dsindex = params[0]
-        datastore = command.remotedatastores[dsindex]
-        return list(datastore.keys())
-    dataStoreConnectorGetKeys.readonly = True
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-    def dataStoreConnectorGetVarHistory(self, command, params):
+        d = command.remotedatastores[dsindex].varhistory
+        return getattr(d, method)(*args, **kwargs)
+
+    def dataStoreConnectorIncHistCmd(self, command, params):
         dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        return datastore.varhistory.variable(name)
-    dataStoreConnectorGetVarHistory.readonly = True
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-    def dataStoreConnectorExpandPythonRef(self, command, params):
-        config_data_dict = params[0]
-        varname = params[1]
-        expr = params[2]
-
-        config_data = command.remotedatastores.receive_datastore(config_data_dict)
-
-        varparse = bb.data_smart.VariableParse(varname, config_data)
-        return varparse.python_sub(expr)
+        d = command.remotedatastores[dsindex].inchistory
+        return getattr(d, method)(*args, **kwargs)
 
     def dataStoreConnectorRelease(self, command, params):
         dsindex = params[0]
@@ -511,31 +489,6 @@ class CommandsSync:
             raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex)
         command.remotedatastores.release(dsindex)
 
-    def dataStoreConnectorSetVarFlag(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        flag = params[2]
-        value = params[3]
-        datastore = command.remotedatastores[dsindex]
-        datastore.setVarFlag(name, flag, value)
-
-    def dataStoreConnectorDelVar(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        if len(params) > 2:
-            flag = params[2]
-            datastore.delVarFlag(name, flag)
-        else:
-            datastore.delVar(name)
-
-    def dataStoreConnectorRenameVar(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        newname = params[2]
-        datastore = command.remotedatastores[dsindex]
-        datastore.renameVar(name, newname)
-
     def parseRecipeFile(self, command, params):
         """
         Parse the specified recipe file (with or without bbappends)
@@ -546,8 +499,7 @@ class CommandsSync:
         appends = params[1]
         appendlist = params[2]
         if len(params) > 3:
-            config_data_dict = params[3]
-            config_data = command.remotedatastores.receive_datastore(config_data_dict)
+            config_data = command.remotedatastores[params[3]]
         else:
             config_data = None
 

bitbake/lib/bb/compat.py

@@ -1,3 +1,7 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
 """Code pulled from future python versions, here for compatibility"""
 
 from collections import MutableMapping, KeysView, ValuesView, ItemsView, OrderedDict

bitbake/lib/bb/cooker.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -9,22 +6,10 @@
 # Copyright (C) 2005        ROAD GmbH
 # Copyright (C) 2006 - 2007 Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
 
 import sys, os, glob, os.path, re, time
-import atexit
 import itertools
 import logging
 import multiprocessing
@@ -32,19 +17,17 @@ import sre_constants
 import threading
 from io import StringIO, UnsupportedOperation
 from contextlib import closing
-from functools import wraps
 from collections import defaultdict, namedtuple
 import bb, bb.exceptions, bb.command
 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
 import queue
 import signal
-import subprocess
-import errno
 import prserv.serv
 import pyinotify
 import json
 import pickle
 import codecs
+import hashserv
 
 logger      = logging.getLogger("BitBake")
 collectlog  = logging.getLogger("BitBake.Collection")
@@ -175,27 +158,45 @@ class BBCooker:
 
         self.configuration = configuration
 
+        bb.debug(1, "BBCooker starting %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher = pyinotify.WatchManager()
-        self.configwatcher.bbseen = []
-        self.configwatcher.bbwatchedfiles = []
+        bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
+        sys.stdout.flush()
+
+        self.configwatcher.bbseen = set()
+        self.configwatcher.bbwatchedfiles = set()
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
+        bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
+        sys.stdout.flush()
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
                          pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
         self.watcher = pyinotify.WatchManager()
-        self.watcher.bbseen = []
-        self.watcher.bbwatchedfiles = []
+        bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
+        sys.stdout.flush()
+        self.watcher.bbseen = set()
+        self.watcher.bbwatchedfiles = set()
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
+        bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
+        sys.stdout.flush()
+
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
         bb.parse.clear_cache()
         bb.parse.BBHandler.cached_statements = {}
 
         self.ui_cmdline = None
+        self.hashserv = None
+        self.hashservaddr = None
 
         self.initConfigurationData()
 
+        bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
+        sys.stdout.flush()
+
         # we log all events to a file if so directed
         if self.configuration.writeeventlog:
             # register the log file writer as UI Handler
@@ -233,6 +234,9 @@ class BBCooker:
         # Let SIGHUP exit as SIGTERM
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
+        bb.debug(1, "BBCooker startup complete %s" % time.time())
+        sys.stdout.flush()
+
     def process_inotify_updates(self):
         for n in [self.confignotifier, self.notifier]:
             if n.check_events(timeout=0):
@@ -270,14 +274,14 @@ class BBCooker:
         if not watcher:
             watcher = self.watcher
         for i in deps:
-            watcher.bbwatchedfiles.append(i[0])
+            watcher.bbwatchedfiles.add(i[0])
             if dirs:
                 f = i[0]
             else:
                 f = os.path.dirname(i[0])
             if f in watcher.bbseen:
                 continue
-            watcher.bbseen.append(f)
+            watcher.bbseen.add(f)
             watchtarget = None
             while True:
                 # We try and add watches for files that don't exist but if they did, would influence
@@ -286,7 +290,7 @@ class BBCooker:
                 try:
                     watcher.add_watch(f, self.watchmask, quiet=False)
                     if watchtarget:
-                        watcher.bbwatchedfiles.append(watchtarget)
+                        watcher.bbwatchedfiles.add(watchtarget)
                     break
                 except pyinotify.WatchManagerError as e:
                     if 'ENOENT' in str(e):
@@ -294,7 +298,7 @@ class BBCooker:
                         f = os.path.dirname(f)
                         if f in watcher.bbseen:
                             break
-                        watcher.bbseen.append(f)
+                        watcher.bbseen.add(f)
                         continue
                     if 'ENOSPC' in str(e):
                         providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
@@ -363,17 +367,12 @@ class BBCooker:
 
         self.data.setVar('BB_CMDLINE', self.ui_cmdline)
 
-        #
-        # Copy of the data store which has been expanded.
-        # Used for firing events and accessing variables where expansion needs to be accounted for
-        #
-        bb.parse.init_parser(self.data)
-
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.disableDataTracking()
 
-        self.data.renameVar("__depends", "__base_depends")
-        self.add_filewatch(self.data.getVar("__base_depends", False), self.configwatcher)
+        for mc in self.databuilder.mcdata.values():
+            mc.renameVar("__depends", "__base_depends")
+            self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher)
 
         self.baseconfig_valid = True
         self.parsecache_valid = False
@@ -385,6 +384,22 @@ class BBCooker:
         except prserv.serv.PRServiceConfigError as e:
             bb.fatal("Unable to start PR Server, exitting")
 
+        if self.data.getVar("BB_HASHSERVE") == "auto":
+            # Create a new hash server bound to a unix domain socket
+            if not self.hashserv:
+                dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+                self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
+                self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False)
+                self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever)
+                self.hashserv.process.start()
+            self.data.setVar("BB_HASHSERVE", self.hashservaddr)
+            self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr)
+            self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr)
+            for mc in self.databuilder.mcdata:
+                self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
+
+        bb.parse.init_parser(self.data)
+
     def enableDataTracking(self):
         self.configuration.tracking = True
         if hasattr(self, "data"):
@@ -488,6 +503,7 @@ class BBCooker:
         """
         fn = None
         envdata = None
+        mc = ''
         if not pkgs_to_build:
             pkgs_to_build = []
 
@@ -496,6 +512,12 @@ class BBCooker:
             self.enableDataTracking()
             self.reset()
 
+        def mc_base(p):
+            if p.startswith('mc:'):
+                s = p.split(':')
+                if len(s) == 2:
+                    return s[1]
+            return None
 
         if buildfile:
             # Parse the configuration here. We need to do it explicitly here since
@@ -506,18 +528,16 @@ class BBCooker:
             fn = self.matchFile(fn)
             fn = bb.cache.realfn2virtual(fn, cls, mc)
         elif len(pkgs_to_build) == 1:
-            ignore = self.data.getVar("ASSUME_PROVIDED") or ""
-            if pkgs_to_build[0] in set(ignore.split()):
-                bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
+            mc = mc_base(pkgs_to_build[0])
+            if not mc:
+                ignore = self.data.getVar("ASSUME_PROVIDED") or ""
+                if pkgs_to_build[0] in set(ignore.split()):
+                    bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-            taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
+                taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
 
-            mc = runlist[0][0]
-            fn = runlist[0][3]
-        else:
-            envdata = self.data
-            data.expandKeys(envdata)
-            parse.ast.runAnonFuncs(envdata)
+                mc = runlist[0][0]
+                fn = runlist[0][3]
 
         if fn:
             try:
@@ -526,6 +546,12 @@ class BBCooker:
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
+        else:
+            if not mc in self.databuilder.mcdata:
+                bb.fatal('Not multiconfig named "%s" found' % mc)
+            envdata = self.databuilder.mcdata[mc]
+            data.expandKeys(envdata)
+            parse.ast.runAnonFuncs(envdata)
 
         # Display history
         with closing(StringIO()) as env:
@@ -565,10 +591,10 @@ class BBCooker:
         wildcard = False
 
         # Wild card expansion:
-        # Replace string such as "multiconfig:*:bash"
-        # into "multiconfig:A:bash multiconfig:B:bash bash"
+        # Replace string such as "mc:*:bash"
+        # into "mc:A:bash mc:B:bash bash"
         for k in targetlist:
-            if k.startswith("multiconfig:"):
+            if k.startswith("mc:"):
                 if wildcard:
                     bb.fatal('multiconfig conflict')
                 if k.split(":")[1] == "*":
@@ -601,7 +627,7 @@ class BBCooker:
         runlist = []
         for k in fulltargetlist:
             mc = ""
-            if k.startswith("multiconfig:"):
+            if k.startswith("mc:"):
                 mc = k.split(":")[1]
                 k = ":".join(k.split(":")[2:])
             ktask = task
@@ -620,6 +646,37 @@ class BBCooker:
             runlist.append([mc, k, ktask, fn])
             bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
 
+        havemc = False
+        for mc in self.multiconfigs:
+            if taskdata[mc].get_mcdepends():
+                havemc = True
+
+        # No need to do check providers if there are no mcdeps or not an mc build
+        if havemc or len(self.multiconfigs) > 1:
+            seen = set()
+            new = True
+            # Make sure we can provide the multiconfig dependency
+            while new:
+                mcdeps = set()
+                # Add unresolved first, so we can get multiconfig indirect dependencies on time
+                for mc in self.multiconfigs:
+                    taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
+                    mcdeps |= set(taskdata[mc].get_mcdepends())
+                new = False
+                for mc in self.multiconfigs:
+                    for k in mcdeps:
+                        if k in seen:
+                            continue
+                        l = k.split(':')
+                        depmc = l[2]
+                        if depmc not in self.multiconfigs:
+                            bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
+                        else:
+                            logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
+                            taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
+                            seen.add(k)
+                            new = True
+
         for mc in self.multiconfigs:
             taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
 
@@ -655,7 +712,7 @@ class BBCooker:
     @staticmethod
     def add_mc_prefix(mc, pn):
         if mc:
-            return "multiconfig:%s:%s" % (mc, pn)
+            return "mc:%s:%s" % (mc, pn)
         return pn
 
     def buildDependTree(self, rq, taskdata):
@@ -706,8 +763,8 @@ class BBCooker:
             if not dotname in depend_tree["tdepends"]:
                 depend_tree["tdepends"][dotname] = []
             for dep in rq.rqdata.runtaskentries[tid].depends:
-                (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
-                deppn = self.recipecaches[mc].pkg_fn[deptaskfn]
+                (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
+                deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)
@@ -854,6 +911,10 @@ class BBCooker:
             os.unlink('package-depends.dot')
         except FileNotFoundError:
             pass
+        try:
+            os.unlink('recipe-depends.dot')
+        except FileNotFoundError:
+            pass
 
         with open('task-depends.dot', 'w') as f:
             f.write("digraph depends {\n")
@@ -867,27 +928,6 @@ class BBCooker:
             f.write("}\n")
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
-        with open('recipe-depends.dot', 'w') as f:
-            f.write("digraph depends {\n")
-            pndeps = {}
-            for task in sorted(depgraph["tdepends"]):
-                (pn, taskname) = task.rsplit(".", 1)
-                if pn not in pndeps:
-                    pndeps[pn] = set()
-                for dep in sorted(depgraph["tdepends"][task]):
-                    (deppn, deptaskname) = dep.rsplit(".", 1)
-                    pndeps[pn].add(deppn)
-            for pn in sorted(pndeps):
-                fn = depgraph["pn"][pn]["filename"]
-                version = depgraph["pn"][pn]["version"]
-                f.write('"%s" [label="%s\\n%s\\n%s"]\n' % (pn, pn, version, fn))
-                for dep in sorted(pndeps[pn]):
-                    if dep == pn:
-                        continue
-                    f.write('"%s" -> "%s"\n' % (pn, dep))
-            f.write("}\n")
-        logger.info("Flattened recipe dependencies saved to 'recipe-depends.dot'")
-
     def show_appends_with_no_recipes(self):
         # Determine which bbappends haven't been applied
 
@@ -983,16 +1023,16 @@ class BBCooker:
             bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
 
     def findProviders(self, mc=''):
-        return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+        return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
 
     def findBestProvider(self, pn, mc=''):
         if pn in self.recipecaches[mc].providers:
             filenames = self.recipecaches[mc].providers[pn]
-            eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.data, self.recipecaches[mc])
+            eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
             filename = eligible[0]
             return None, None, None, filename
         elif pn in self.recipecaches[mc].pkg_pn:
-            return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
         else:
             return None, None, None, None
 
@@ -1164,14 +1204,14 @@ class BBCooker:
             for c in collection_list:
                 calc_layer_priority(c)
                 regex = self.data.getVar("BBFILE_PATTERN_%s" % c)
-                if regex == None:
+                if regex is None:
                     parselog.error("BBFILE_PATTERN_%s not defined" % c)
                     errors = True
                     continue
                 elif regex == "":
                     parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c)
+                    cre = re.compile('^NULL$')
                     errors = False
-                    continue
                 else:
                     try:
                         cre = re.compile(regex)
@@ -1270,7 +1310,7 @@ class BBCooker:
         self.parseConfiguration()
 
         # If we are told to do the None task then query the default task
-        if (task == None):
+        if task is None:
             task = self.configuration.cmd
         if not task.startswith("do_"):
             task = "do_%s" % task
@@ -1414,7 +1454,7 @@ class BBCooker:
         self.buildSetVars()
 
         # If we are told to do the None task then query the default task
-        if (task == None):
+        if task is None:
             task = self.configuration.cmd
 
         if not task.startswith("do_"):
@@ -1432,7 +1472,7 @@ class BBCooker:
         ntargets = []
         for target in runlist:
             if target[0]:
-                ntargets.append("multiconfig:%s:%s:%s" % (target[0], target[1], target[2]))
+                ntargets.append("mc:%s:%s:%s" % (target[0], target[1], target[2]))
             ntargets.append("%s:%s" % (target[1], target[2]))
 
         for mc in self.multiconfigs:
@@ -1552,9 +1592,12 @@ class BBCooker:
             raise NothingToBuild
 
         ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
-        for pkg in pkgs_to_build:
+        for pkg in pkgs_to_build.copy():
             if pkg in ignore:
                 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
+            if pkg.startswith("multiconfig:"):
+                pkgs_to_build.remove(pkg)
+                pkgs_to_build.append(pkg.replace("multiconfig:", "mc:"))
 
         if 'world' in pkgs_to_build:
             pkgs_to_build.remove('world')
@@ -1562,11 +1605,11 @@ class BBCooker:
                 bb.providers.buildWorldTargetList(self.recipecaches[mc], task)
                 for t in self.recipecaches[mc].world_target:
                     if mc:
-                        t = "multiconfig:" + mc + ":" + t
+                        t = "mc:" + mc + ":" + t
                     pkgs_to_build.append(t)
 
         if 'universe' in pkgs_to_build:
-            parselog.warning("The \"universe\" target is only intended for testing and may produce errors.")
+            parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.")
             parselog.debug(1, "collating packages for \"universe\"")
             pkgs_to_build.remove('universe')
             for mc in self.multiconfigs:
@@ -1581,7 +1624,7 @@ class BBCooker:
                             bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task))
                             continue
                     if mc:
-                        t = "multiconfig:" + mc + ":" + t
+                        t = "mc:" + mc + ":" + t
                     pkgs_to_build.append(t)
 
         return pkgs_to_build
@@ -1594,9 +1637,11 @@ class BBCooker:
 
     def post_serve(self):
         prserv.serv.auto_shutdown()
+        if self.hashserv:
+            self.hashserv.process.terminate()
+            self.hashserv.process.join()
         bb.event.fire(CookerExit(), self.data)
 
-
     def shutdown(self, force = False):
         if force:
             self.state = state.forceshutdown
@@ -1611,6 +1656,7 @@ class BBCooker:
 
     def reset(self):
         self.initConfigurationData()
+        self.handlePRServ()
 
     def clientComplete(self):
         """Called when the client is done using the server"""
@@ -1619,6 +1665,8 @@ class BBCooker:
         self.command.reset()
         self.databuilder.reset()
         self.data = self.databuilder.data
+        self.parsecache_valid = False
+        self.baseconfig_valid = False
 
 
 class CookerExit(bb.event.Event):
@@ -1633,12 +1681,15 @@ class CookerExit(bb.event.Event):
 class CookerCollectFiles(object):
     def __init__(self, priorities):
         self.bbappends = []
-        self.bbfile_config_priorities = priorities
+        # Priorities is a list of tupples, with the second element as the pattern.
+        # We need to sort the list with the longest pattern first, and so on to
+        # the shortest.  This allows nested layers to be properly evaluated.
+        self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
 
     def calc_bbfile_priority( self, filename, matched = None ):
         for _, _, regex, pri in self.bbfile_config_priorities:
             if regex.match(filename):
-                if matched != None:
+                if matched is not None:
                     if not regex in matched:
                         matched.add(regex)
                 return pri
@@ -1735,7 +1786,7 @@ class CookerCollectFiles(object):
                 # When constructing an older style single regex, it's possible for BBMASK
                 # to end up beginning with '|', which matches and masks _everything_.
                 if mask.startswith("|"):
-                    collectlog.warn("BBMASK contains regular expression beginning with '|', fixing: %s" % mask)
+                    collectlog.warning("BBMASK contains regular expression beginning with '|', fixing: %s" % mask)
                     mask = mask[1:]
                 try:
                     re.compile(mask)
@@ -1841,35 +1892,6 @@ class ParsingFailure(Exception):
         self.recipe = recipe
         Exception.__init__(self, realexception, recipe)
 
-class Feeder(multiprocessing.Process):
-    def __init__(self, jobs, to_parsers, quit):
-        self.quit = quit
-        self.jobs = jobs
-        self.to_parsers = to_parsers
-        multiprocessing.Process.__init__(self)
-
-    def run(self):
-        while True:
-            try:
-                quit = self.quit.get_nowait()
-            except queue.Empty:
-                pass
-            else:
-                if quit == 'cancel':
-                    self.to_parsers.cancel_join_thread()
-                break
-
-            try:
-                job = self.jobs.pop()
-            except IndexError:
-                break
-
-            try:
-                self.to_parsers.put(job, timeout=0.5)
-            except queue.Full:
-                self.jobs.insert(0, job)
-                continue
-
 class Parser(multiprocessing.Process):
     def __init__(self, jobs, results, quit, init, profile):
         self.jobs = jobs
@@ -1916,14 +1938,12 @@ class Parser(multiprocessing.Process):
                 result = pending.pop()
             else:
                 try:
-                    job = self.jobs.get(timeout=0.25)
-                except queue.Empty:
-                    continue
-
-                if job is None:
+                    job = self.jobs.pop()
+                except IndexError:
                     break
                 result = self.parse(*job)
-
+                # Clear the siggen cache after parsing to control memory usage, its huge
+                bb.parse.siggen.postparsing_clean_cache()
             try:
                 self.results.put(result, timeout=0.25)
             except queue.Full:
@@ -1931,6 +1951,7 @@ class Parser(multiprocessing.Process):
 
     def parse(self, filename, appends):
         try:
+            origfilter = bb.event.LogHandler.filter
             # Record the filename we're parsing into any events generated
             def parse_filter(self, record):
                 record.taskpid = bb.event.worker_pid
@@ -1953,6 +1974,8 @@ class Parser(multiprocessing.Process):
         # a SystemExit event for example.
         except BaseException as exc:
             return True, ParsingFailure(exc, filename)
+        finally:
+            bb.event.LogHandler.filter = origfilter
 
 class CookerParser(object):
     def __init__(self, cooker, filelist, masked):
@@ -2004,14 +2027,15 @@ class CookerParser(object):
                 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
                 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
 
-            self.feeder_quit = multiprocessing.Queue(maxsize=1)
             self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
-            self.jobs = multiprocessing.Queue(maxsize=self.num_processes)
             self.result_queue = multiprocessing.Queue()
-            self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit)
-            self.feeder.start()
+
+            def chunkify(lst,n):
+                return [lst[i::n] for i in range(n)]
+            self.jobs = chunkify(self.willparse, self.num_processes)
+
             for i in range(0, self.num_processes):
-                parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
+                parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
                 parser.start()
                 self.process_names.append(parser.name)
                 self.processes.append(parser)
@@ -2032,17 +2056,20 @@ class CookerParser(object):
                                             self.total)
 
             bb.event.fire(event, self.cfgdata)
-            self.feeder_quit.put(None)
             for process in self.processes:
                 self.parser_quit.put(None)
         else:
-            self.feeder_quit.put('cancel')
-
             self.parser_quit.cancel_join_thread()
             for process in self.processes:
                 self.parser_quit.put(None)
 
-            self.jobs.cancel_join_thread()
+        # Cleanup the queue before call process.join(), otherwise there might be
+        # deadlocks.
+        while True:
+            try:
+               self.result_queue.get(timeout=0.25)
+            except queue.Empty:
+                break
 
         for process in self.processes:
             if force:
@@ -2050,7 +2077,6 @@ class CookerParser(object):
                 process.terminate()
             else:
                 process.join()
-        self.feeder.join()
 
         sync = threading.Thread(target=self.bb_cache.sync)
         sync.start()

bitbake/lib/bb/cookerdata.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -9,23 +6,14 @@
 # Copyright (C) 2005        ROAD GmbH
 # Copyright (C) 2006        Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import logging
 import os
 import re
 import sys
+import hashlib
 from functools import wraps
 import bb
 from bb import data
@@ -134,6 +122,7 @@ class CookerConfiguration(object):
         self.profile = False
         self.nosetscene = False
         self.setsceneonly = False
+        self.skipsetscene = False
         self.invalidate_stamp = False
         self.dump_signatures = []
         self.dry_run = False
@@ -279,12 +268,13 @@ class CookerDataBuilder(object):
         self.mcdata = {}
 
     def parseBaseConfiguration(self):
+        data_hash = hashlib.sha256()
         try:
-            bb.parse.init_parser(self.basedata)
             self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
 
             if self.data.getVar("BB_WORKERCONTEXT", False) is None:
                 bb.fetch.fetcher_init(self.data)
+            bb.parse.init_parser(self.data)
             bb.codeparser.parser_cache_init(self.data)
 
             bb.event.fire(bb.event.ConfigParsed(), self.data)
@@ -302,7 +292,7 @@ class CookerDataBuilder(object):
                 bb.event.fire(bb.event.ConfigParsed(), self.data)
 
             bb.parse.init_parser(self.data)
-            self.data_hash = self.data.get_hash()
+            data_hash.update(self.data.get_hash().encode('utf-8'))
             self.mcdata[''] = self.data
 
             multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
@@ -310,9 +300,11 @@ class CookerDataBuilder(object):
                 mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
                 bb.event.fire(bb.event.ConfigParsed(), mcdata)
                 self.mcdata[config] = mcdata
+                data_hash.update(mcdata.get_hash().encode('utf-8'))
             if multiconfig:
                 bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
 
+            self.data_hash = data_hash.hexdigest()
         except (SyntaxError, bb.BBHandledException):
             raise bb.BBHandledException
         except bb.data_smart.ExpansionError as e:
@@ -354,14 +346,24 @@ class CookerDataBuilder(object):
             data = parse_config_file(layerconf, data)
 
             layers = (data.getVar('BBLAYERS') or "").split()
+            broken_layers = []
 
             data = bb.data.createCopy(data)
             approved = bb.utils.approved_variables()
+
+            # Check whether present layer directories exist
             for layer in layers:
                 if not os.path.isdir(layer):
-                    parselog.critical("Layer directory '%s' does not exist! "
-                                      "Please check BBLAYERS in %s" % (layer, layerconf))
-                    sys.exit(1)
+                    broken_layers.append(layer)
+
+            if broken_layers:
+                parselog.critical("The following layer directories do not exist:")
+                for layer in broken_layers:
+                    parselog.critical("   %s", layer)
+                parselog.critical("Please check BBLAYERS in %s" % (layerconf))
+                sys.exit(1)
+
+            for layer in layers:
                 parselog.debug(2, "Adding layer %s", layer)
                 if 'HOME' in approved and '~' in layer:
                     layer = os.path.expanduser(layer)
@@ -391,7 +393,11 @@ class CookerDataBuilder(object):
                 bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
 
             layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
+            collections_tmp = collections[:]
             for c in collections:
+                collections_tmp.remove(c)
+                if c in collections_tmp:
+                    bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c)
                 compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split())
                 if compat and not (compat & layerseries):
                     bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"

bitbake/lib/bb/daemonize.py

@@ -1,3 +1,7 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
 """
 Python Daemonizing helper
 
@@ -16,6 +20,10 @@ def createDaemon(function, logfile):
     background as a daemon, returning control to the caller.
     """
 
+    # Ensure stdout/stderror are flushed before forking to avoid duplicate output
+    sys.stdout.flush()
+    sys.stderr.flush()
+
     try:
         # Fork a child process so the parent can exit.  This returns control to
         # the command-line or shell.  It also guarantees that the child will not
@@ -49,8 +57,8 @@ def createDaemon(function, logfile):
             # exit() or _exit()?
             # _exit is like exit(), but it doesn't call any functions registered
             # with atexit (and on_exit) or any registered signal handlers.  It also
-            # closes any open file descriptors.  Using exit() may cause all stdio
-            # streams to be flushed twice and any temporary files may be unexpectedly
+            # closes any open file descriptors, but doesn't flush any buffered output.
+            # Using exit() may cause all any temporary files to be unexpectedly
             # removed.  It's therefore recommended that child branches of a fork()
             # and the parent branch(es) of a daemon use _exit().
             os._exit(0)
@@ -61,17 +69,19 @@ def createDaemon(function, logfile):
     # The second child.
 
     # Replace standard fds with our own
-    si = open('/dev/null', 'r')
-    os.dup2(si.fileno(), sys.stdin.fileno())
+    with open('/dev/null', 'r') as si:
+        os.dup2(si.fileno(), sys.stdin.fileno())
 
     try:
         so = open(logfile, 'a+')
-        se = so
         os.dup2(so.fileno(), sys.stdout.fileno())
-        os.dup2(se.fileno(), sys.stderr.fileno())
+        os.dup2(so.fileno(), sys.stderr.fileno())
     except io.UnsupportedOperation:
         sys.stdout = open(logfile, 'a+')
-        sys.stderr = sys.stdout
+
+    # Have stdout and stderr be the same so log output matches chronologically
+    # and there aren't two seperate buffers
+    sys.stderr = sys.stdout
 
     try:
         function()
@@ -79,4 +89,9 @@ def createDaemon(function, logfile):
         traceback.print_exc()
     finally:
         bb.event.print_ui_queue()
+        # os._exit() doesn't flush open files like os.exit() does. Manually flush
+        # stdout and stderr so that any logging output will be seen, particularly
+        # exception tracebacks.
+        sys.stdout.flush()
+        sys.stderr.flush()
         os._exit(0)

bitbake/lib/bb/data.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Data' implementations
 
@@ -22,22 +20,12 @@ the speed is more critical here.
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2005        Holger Hans Peter Freyther
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import sys, os, re
+import hashlib
 if sys.argv[0][-5:] == "pydoc":
     path = os.path.dirname(os.path.dirname(sys.argv[1]))
 else:
@@ -91,7 +79,7 @@ def expand(s, d, varname = None):
     return d.expand(s, varname)
 
 def expandKeys(alterdata, readdata = None):
-    if readdata == None:
+    if readdata is None:
         readdata = alterdata
 
     todolist = {}
@@ -142,7 +130,7 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
         if all:
             oval = d.getVar(var, False)
         val = d.getVar(var)
-    except (KeyboardInterrupt, bb.build.FuncFailed):
+    except (KeyboardInterrupt):
         raise
     except Exception as exc:
         o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
@@ -283,14 +271,12 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     try:
         if key[-1] == ']':
             vf = key[:-1].split('[')
-            value = d.getVarFlag(vf[0], vf[1], False)
-            parser = d.expandWithRefs(value, key)
+            value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
             return deps, value
         varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
         vardeps = varflags.get("vardeps")
-        value = d.getVarFlag(key, "_content", False)
 
         def handle_contains(value, contains, d):
             newvalue = ""
@@ -309,25 +295,34 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
                 return newvalue
             return value + newvalue
 
+        def handle_remove(value, deps, removes, d):
+            for r in sorted(removes):
+                r2 = d.expandWithRefs(r, None)
+                value += "\n_remove of %s" % r
+                deps |= r2.references
+                deps = deps | (keys & r2.execs)
+            return value
+
         if "vardepvalue" in varflags:
-           value = varflags.get("vardepvalue")
+            value = varflags.get("vardepvalue")
         elif varflags.get("func"):
             if varflags.get("python"):
+                value = d.getVarFlag(key, "_content", False)
                 parser = bb.codeparser.PythonParser(key, logger)
-                if value and "\t" in value:
-                    logger.warning("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE")))
                 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
                 deps = deps | parser.references
                 deps = deps | (keys & parser.execs)
                 value = handle_contains(value, parser.contains, d)
             else:
-                parsedvar = d.expandWithRefs(value, key)
+                value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True)
                 parser = bb.codeparser.ShellParser(key, logger)
                 parser.parse_shell(parsedvar.value)
                 deps = deps | shelldeps
                 deps = deps | parsedvar.references
                 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
                 value = handle_contains(value, parsedvar.contains, d)
+                if hasattr(parsedvar, "removes"):
+                    value = handle_remove(value, deps, parsedvar.removes, d)
             if vardeps is None:
                 parser.log.flush()
             if "prefuncs" in varflags:
@@ -337,10 +332,12 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
             if "exports" in varflags:
                 deps = deps | set(varflags["exports"].split())
         else:
-            parser = d.expandWithRefs(value, key)
+            value, parser = d.getVarFlag(key, "_content", False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
             value = handle_contains(value, parser.contains, d)
+            if hasattr(parser, "removes"):
+                value = handle_remove(value, deps, parser.removes, d)
 
         if "vardepvalueexclude" in varflags:
             exclude = varflags.get("vardepvalueexclude")
@@ -368,7 +365,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
     #d.setVarFlag(key, "vardeps", deps)
 
-def generate_dependencies(d):
+def generate_dependencies(d, whitelist):
 
     keys = set(key for key in d if not key.startswith("__"))
     shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
@@ -383,7 +380,7 @@ def generate_dependencies(d):
         newdeps = deps[task]
         seen = set()
         while newdeps:
-            nextdeps = newdeps
+            nextdeps = newdeps - whitelist
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
@@ -394,6 +391,43 @@ def generate_dependencies(d):
         #print "For %s: %s" % (task, str(deps[task]))
     return tasklist, deps, values
 
+def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
+    taskdeps = {}
+    basehash = {}
+
+    for task in tasklist:
+        data = lookupcache[task]
+
+        if data is None:
+            bb.error("Task %s from %s seems to be empty?!" % (task, fn))
+            data = ''
+
+        gendeps[task] -= whitelist
+        newdeps = gendeps[task]
+        seen = set()
+        while newdeps:
+            nextdeps = newdeps
+            seen |= nextdeps
+            newdeps = set()
+            for dep in nextdeps:
+                if dep in whitelist:
+                    continue
+                gendeps[dep] -= whitelist
+                newdeps |= gendeps[dep]
+            newdeps -= seen
+
+        alldeps = sorted(seen)
+        for dep in alldeps:
+            data = data + dep
+            var = lookupcache[dep]
+            if var is not None:
+                data = data + str(var)
+        k = fn + ":" + task
+        basehash[k] = hashlib.sha256(data.encode("utf-8")).hexdigest()
+        taskdeps[task] = alldeps
+
+    return taskdeps, basehash
+
 def inherits_class(klass, d):
     val = d.getVar('__inherit_cache', False) or []
     needle = os.path.join('classes', '%s.bbclass' % klass)

bitbake/lib/bb/data_smart.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake Smart Dictionary Implementation
 
@@ -14,18 +12,8 @@ BitBake build tools.
 # Copyright (C) 2005        Uli Luckas
 # Copyright (C) 2005        ROAD GmbH
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import copy, re, sys, traceback
@@ -39,9 +27,11 @@ from bb.COW  import COWDictBase
 logger = logging.getLogger("BitBake.Data")
 
 __setvar_keyword__ = ["_append", "_prepend", "_remove"]
-__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
-__expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
+__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
+__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
+__whitespace_split__ = re.compile(r'(\s)')
+__override_regexp__ = re.compile(r'[a-z0-9]+')
 
 def infer_caller_details(loginfo, parent = False, varval = True):
     """Save the caller the trouble of specifying everything."""
@@ -104,11 +94,7 @@ class VariableParse:
             if self.varname and key:
                 if self.varname == key:
                     raise Exception("variable %s references itself!" % self.varname)
-            if key in self.d.expand_cache:
-                varparse = self.d.expand_cache[key]
-                var = varparse.value
-            else:
-                var = self.d.getVarFlag(key, "_content")
+            var = self.d.getVarFlag(key, "_content")
             self.references.add(key)
             if var is not None:
                 return var
@@ -121,11 +107,11 @@ class VariableParse:
             else:
                 code = match.group()[3:-1]
 
-            if "_remote_data" in self.d:
-                connector = self.d["_remote_data"]
-                return connector.expandPythonRef(self.varname, code, self.d)
-
-            codeobj = compile(code.strip(), self.varname or "<expansion>", "eval")
+            if self.varname:
+                varname = 'Var <%s>' % self.varname
+            else:
+                varname = '<expansion>'
+            codeobj = compile(code.strip(), varname, "eval")
 
             parser = bb.codeparser.PythonParser(self.varname, logger)
             parser.parse_python(code)
@@ -267,13 +253,18 @@ class VariableHistory(object):
             return
         self.variables[var].append(loginfo.copy())
 
-    def variable(self, var):
-        remote_connector = self.dataroot.getVar('_remote_data', False)
-        if remote_connector:
-            varhistory = remote_connector.getVarHistory(var)
-        else:
-            varhistory = []
+    def rename_variable_hist(self, oldvar, newvar):
+        if not self.dataroot._tracking:
+            return
+        if oldvar not in self.variables:
+            return
+        if newvar not in self.variables:
+            self.variables[newvar] = []
+        for i in self.variables[oldvar]:
+            self.variables[newvar].append(i.copy())
 
+    def variable(self, var):
+        varhistory = []
         if var in self.variables:
             varhistory.extend(self.variables[var])
         return varhistory
@@ -338,11 +329,12 @@ class VariableHistory(object):
                 lines.append(line)
         return lines
 
-    def get_variable_items_files(self, var, d):
+    def get_variable_items_files(self, var):
         """
         Use variable history to map items added to a list variable and
         the files in which they were added.
         """
+        d = self.dataroot
         history = self.variable(var)
         finalitems = (d.getVar(var) or '').split()
         filemap = {}
@@ -401,9 +393,6 @@ class DataSmart(MutableMapping):
         if not isinstance(s, str): # sanity check
             return VariableParse(varname, self, s)
 
-        if varname and varname in self.expand_cache:
-            return self.expand_cache[varname]
-
         varparse = VariableParse(varname, self)
 
         while s.find('${') != -1:
@@ -423,13 +412,11 @@ class DataSmart(MutableMapping):
             except bb.parse.SkipRecipe:
                 raise
             except Exception as exc:
-                raise ExpansionError(varname, s, exc) from exc
+                tb = sys.exc_info()[2]
+                raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
 
         varparse.value = s
 
-        if varname:
-            self.expand_cache[varname] = varparse
-
         return varparse
 
     def expand(self, s, varname = None):
@@ -475,10 +462,6 @@ class DataSmart(MutableMapping):
             if var in dest:
                 return dest[var], self.overridedata.get(var, None)
 
-            if "_remote_data" in dest:
-                connector = dest["_remote_data"]["_content"]
-                return connector.getVar(var)
-
             if "_data" not in dest:
                 break
             dest = dest["_data"]
@@ -498,19 +481,14 @@ class DataSmart(MutableMapping):
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
+        self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
             parsing=True
 
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.setVar(var, value)
-            if not res:
-                return
-
         if 'op' not in loginfo:
             loginfo['op'] = "set"
-        self.expand_cache = {}
+
         match  = __setvar_regexp__.match(var)
         if match and match.group("keyword") in __setvar_keyword__:
             base = match.group('base')
@@ -590,7 +568,7 @@ class DataSmart(MutableMapping):
         # aka pay the cookie monster
         override = var[var.rfind('_')+1:]
         shortvar = var[:var.rfind('_')]
-        while override and override.islower():
+        while override and __override_regexp__.match(override):
             if shortvar not in self.overridedata:
                 self.overridedata[shortvar] = []
             if [var, override] not in self.overridedata[shortvar]:
@@ -611,14 +589,13 @@ class DataSmart(MutableMapping):
         """
         Rename the variable key to newkey
         """
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.renameVar(key, newkey)
-            if not res:
-                return
+        if key == newkey:
+            bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
+            return
 
         val = self.getVar(key, 0, parsing=True)
         if val is not None:
+            self.varhistory.rename_variable_hist(key, newkey)
             loginfo['variable'] = newkey
             loginfo['op'] = 'rename from %s' % key
             loginfo['detail'] = val
@@ -660,16 +637,11 @@ class DataSmart(MutableMapping):
         self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.delVar(var)
-            if not res:
-                return
+        self.expand_cache = {}
 
         loginfo['detail'] = ""
         loginfo['op'] = 'del'
         self.varhistory.record(**loginfo)
-        self.expand_cache = {}
         self.dict[var] = {}
         if var in self.overridedata:
             del self.overridedata[var]
@@ -692,13 +664,8 @@ class DataSmart(MutableMapping):
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.setVarFlag(var, flag, value)
-            if not res:
-                return
-
         self.expand_cache = {}
+
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         loginfo['flag'] = flag
@@ -719,9 +686,21 @@ class DataSmart(MutableMapping):
                 self.dict["__exportlist"]["_content"] = set()
             self.dict["__exportlist"]["_content"].add(var)
 
-    def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False):
+    def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
+        if flag == "_content":
+            cachename = var
+        else:
+            if not flag:
+                bb.warn("Calling getVarFlag with flag unset is invalid")
+                return None
+            cachename = var + "[" + flag + "]"
+
+        if expand and cachename in self.expand_cache:
+            return self.expand_cache[cachename].value
+
         local_var, overridedata = self._findVar(var)
         value = None
+        removes = set()
         if flag == "_content" and overridedata is not None and not parsing:
             match = False
             active = {}
@@ -748,7 +727,11 @@ class DataSmart(MutableMapping):
                             match = active[a]
                             del active[a]
             if match:
-                value = self.getVar(match, False)
+                value, subparser = self.getVarFlag(match, "_content", False, retparser=True)
+                if hasattr(subparser, "removes"):
+                    # We have to carry the removes from the overridden variable to apply at the
+                    # end of processing
+                    removes = subparser.removes
 
         if local_var is not None and value is None:
             if flag in local_var:
@@ -784,17 +767,13 @@ class DataSmart(MutableMapping):
                 if match:
                     value = r + value
 
-        if expand and value:
-            # Only getvar (flag == _content) hits the expand cache
-            cachename = None
-            if flag == "_content":
-                cachename = var
-            else:
-                cachename = var + "[" + flag + "]"
-            value = self.expand(value, cachename)
+        parser = None
+        if expand or retparser:
+            parser = self.expandWithRefs(value, cachename)
+        if expand:
+            value = parser.value
 
-        if value and flag == "_content" and local_var is not None and "_remove" in local_var:
-            removes = []
+        if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing:
             self.need_overrides()
             for (r, o) in local_var["_remove"]:
                 match = True
@@ -803,26 +782,40 @@ class DataSmart(MutableMapping):
                         if not o2 in self.overrides:
                             match = False                            
                 if match:
-                    removes.extend(self.expand(r).split())
+                    removes.add(r)
+
+        if value and flag == "_content" and not parsing:
+            if removes and parser:
+                expanded_removes = {}
+                for r in removes:
+                    expanded_removes[r] = self.expand(r).split()
+
+                parser.removes = set()
+                val = ""
+                for v in __whitespace_split__.split(parser.value):
+                    skip = False
+                    for r in removes:
+                        if v in expanded_removes[r]:
+                            parser.removes.add(r)
+                            skip = True
+                    if skip:
+                        continue
+                    val = val + v
+                parser.value = val
+                if expand:
+                    value = parser.value
+
+        if parser:
+            self.expand_cache[cachename] = parser
+
+        if retparser:
+            return value, parser
 
-            if removes:
-                filtered = filter(lambda v: v not in removes,
-                                  value.split())
-                value = " ".join(filtered)
-                if expand and var in self.expand_cache:
-                    # We need to ensure the expand cache has the correct value
-                    # flag == "_content" here
-                    self.expand_cache[var].value = value
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.delVarFlag(var, flag)
-            if not res:
-                return
-
         self.expand_cache = {}
+
         local_var, _ = self._findVar(var)
         if not local_var:
             return
@@ -939,7 +932,7 @@ class DataSmart(MutableMapping):
 
     def localkeys(self):
         for key in self.dict:
-            if key not in ['_data', '_remote_data']:
+            if key not in ['_data']:
                 yield key
 
     def __iter__(self):
@@ -948,7 +941,7 @@ class DataSmart(MutableMapping):
         def keylist(d):        
             klist = set()
             for key in d:
-                if key in ["_data", "_remote_data"]:
+                if key in ["_data"]:
                     continue
                 if key in deleted:
                     continue
@@ -962,13 +955,6 @@ class DataSmart(MutableMapping):
             if "_data" in d:
                 klist |= keylist(d["_data"])
 
-            if "_remote_data" in d:
-                connector = d["_remote_data"]["_content"]
-                for key in connector.getKeys():
-                    if key in deleted:
-                        continue
-                    klist.add(key)
-
             return klist
 
         self.need_overrides()
@@ -1014,7 +1000,10 @@ class DataSmart(MutableMapping):
                 continue
 
             value = d.getVar(key, False) or ""
-            data.update({key:value})
+            if type(value) is type(self):
+                data.update({key:value.get_hash()})
+            else:
+                data.update({key:value})
 
             varflags = d.getVarFlags(key, internalflags = True)
             if not varflags:
@@ -1034,4 +1023,4 @@ class DataSmart(MutableMapping):
                     data.update({i:value})
 
         data_str = str([(k, data[k]) for k in sorted(data.keys())])
-        return hashlib.md5(data_str.encode("utf-8")).hexdigest()
+        return hashlib.sha256(data_str.encode("utf-8")).hexdigest()

bitbake/lib/bb/event.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Event' implementation
 
@@ -9,21 +7,10 @@ BitBake build tools.
 
 # Copyright (C) 2003, 2004  Chris Larson
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import os, sys
-import warnings
+import sys
 import pickle
 import logging
 import atexit
@@ -136,11 +123,15 @@ def fire_class_handlers(event, d):
 ui_queue = []
 @atexit.register
 def print_ui_queue():
+    global ui_queue
     """If we're exiting before a UI has been spawned, display any queued
     LogRecords to the console."""
     logger = logging.getLogger("BitBake")
     if not _uiready:
         from bb.msg import BBLogFormatter
+        # Flush any existing buffered content
+        sys.stdout.flush()
+        sys.stderr.flush()
         stdout = logging.StreamHandler(sys.stdout)
         stderr = logging.StreamHandler(sys.stderr)
         formatter = BBLogFormatter("%(levelname)s: %(message)s")
@@ -177,6 +168,7 @@ def print_ui_queue():
             logger.removeHandler(stderr)
         else:
             logger.removeHandler(stdout)
+        ui_queue = []
 
 def fire_ui_handlers(event, d):
     global _thread_lock
@@ -354,7 +346,7 @@ def set_UIHmask(handlerNum, level, debug_domains, mask):
 
 def getName(e):
     """Returns the name of a class or class instance"""
-    if getattr(e, "__name__", None) == None:
+    if getattr(e, "__name__", None) is None:
         return e.__class__.__name__
     else:
         return e.__name__
@@ -395,7 +387,7 @@ class RecipeEvent(Event):
         Event.__init__(self)
 
 class RecipePreFinalise(RecipeEvent):
-    """ Recipe Parsing Complete but not yet finialised"""
+    """ Recipe Parsing Complete but not yet finalised"""
 
 class RecipeTaskPreProcess(RecipeEvent):
     """
@@ -411,23 +403,6 @@ class RecipeTaskPreProcess(RecipeEvent):
 class RecipeParsed(RecipeEvent):
     """ Recipe Parsing Complete """
 
-class StampUpdate(Event):
-    """Trigger for any adjustment of the stamp files to happen"""
-
-    def __init__(self, targets, stampfns):
-        self._targets = targets
-        self._stampfns = stampfns
-        Event.__init__(self)
-
-    def getStampPrefix(self):
-        return self._stampfns
-
-    def getTargets(self):
-        return self._targets
-
-    stampPrefix = property(getStampPrefix)
-    targets = property(getTargets)
-
 class BuildBase(Event):
     """Base class for bitbake build events"""
 
@@ -533,7 +508,7 @@ class NoProvider(Event):
         extra = ''
         if not self._reasons:
             if self._close_matches:
-                extra = ". Close matches:\n  %s" % '\n  '.join(self._close_matches)
+                extra = ". Close matches:\n  %s" % '\n  '.join(sorted(set(self._close_matches)))
 
         if self._dependees:
             msg = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s" % (r, self._item, ", ".join(self._dependees), r, extra)

bitbake/lib/bb/exceptions.py

@@ -1,3 +1,6 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
 
 import inspect
 import traceback

bitbake/lib/bb/fetch2/__init__.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' implementations
 
@@ -10,18 +8,7 @@ BitBake build tools.
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2012  Intel Corporation
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
@@ -46,6 +33,9 @@ _checksum_cache = bb.checksum.FileChecksumCache()
 
 logger = logging.getLogger("BitBake.Fetcher")
 
+CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ]
+SHOWN_CHECKSUM_LIST = ["sha256"]
+
 class BBFetchException(Exception):
     """Class all fetch exceptions inherit from"""
     def __init__(self, message):
@@ -144,10 +134,9 @@ class NonLocalMethod(Exception):
         Exception.__init__(self)
 
 class MissingChecksumEvent(bb.event.Event):
-    def __init__(self, url, md5sum, sha256sum):
+    def __init__(self, url, **checksums):
         self.url = url
-        self.checksums = {'md5sum': md5sum,
-                          'sha256sum': sha256sum}
+        self.checksums = checksums
         bb.event.Event.__init__(self)
 
 
@@ -256,7 +245,7 @@ class URI(object):
 
         # Identify if the URI is relative or not
         if urlp.scheme in self._relative_schemes and \
-           re.compile("^\w+:(?!//)").match(uri):
+           re.compile(r"^\w+:(?!//)").match(uri):
             self.relative = True
 
         if not self.relative:
@@ -383,7 +372,7 @@ def decodeurl(url):
         path = location
     else:
         host = location
-        path = ""
+        path = "/"
     if user:
         m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user)
         if m:
@@ -452,8 +441,8 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
             # Handle URL parameters
             if i:
                 # Any specified URL parameters must match
-                for k in uri_replace_decoded[loc]:
-                    if uri_decoded[loc][k] != uri_replace_decoded[loc][k]:
+                for k in uri_find_decoded[loc]:
+                    if uri_decoded[loc][k] != uri_find_decoded[loc][k]:
                         return None
             # Overwrite any specified replacement parameters
             for k in uri_replace_decoded[loc]:
@@ -497,17 +486,22 @@ def fetcher_init(d):
     Called to initialize the fetchers once the configuration data is known.
     Calls before this must not hit the cache.
     """
+
+    revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
+    try:
+        # fetcher_init is called multiple times, so make sure we only save the
+        # revs the first time it is called.
+        if not bb.fetch2.saved_headrevs:
+            bb.fetch2.saved_headrevs = dict(revs)
+    except:
+        pass
+
     # When to drop SCM head revisions controlled by user policy
     srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
     if srcrev_policy == "cache":
         logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
     elif srcrev_policy == "clear":
         logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
-        revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
-        try:
-            bb.fetch2.saved_headrevs = revs.items()
-        except:
-            pass
         revs.clear()
     else:
         raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
@@ -524,24 +518,14 @@ def fetcher_parse_save():
 def fetcher_parse_done():
     _checksum_cache.save_merge()
 
-def fetcher_compare_revisions():
+def fetcher_compare_revisions(d):
     """
-    Compare the revisions in the persistant cache with current values and
-    return true/false on whether they've changed.
+    Compare the revisions in the persistent cache with the saved values from
+    when bitbake was started and return true if they have changed.
     """
 
-    data = bb.persist_data.persist('BB_URI_HEADREVS', d).items()
-    data2 = bb.fetch2.saved_headrevs
-
-    changed = False
-    for key in data:
-        if key not in data2 or data2[key] != data[key]:
-            logger.debug(1, "%s changed", key)
-            changed = True
-            return True
-        else:
-            logger.debug(2, "%s did not change", key)
-    return False
+    headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d))
+    return headrevs != bb.fetch2.saved_headrevs
 
 def mirror_from_string(data):
     mirrors = (data or "").replace('\\n',' ').split()
@@ -565,71 +549,84 @@ def verify_checksum(ud, d, precomputed={}):
     downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
     """
 
-    _MD5_KEY = "md5"
-    _SHA256_KEY = "sha256"
-
     if ud.ignore_checksums or not ud.method.supports_checksum(ud):
         return {}
 
-    if _MD5_KEY in precomputed:
-        md5data = precomputed[_MD5_KEY]
-    else:
-        md5data = bb.utils.md5_file(ud.localpath)
+    def compute_checksum_info(checksum_id):
+        checksum_name = getattr(ud, "%s_name" % checksum_id)
 
-    if _SHA256_KEY in precomputed:
-        sha256data = precomputed[_SHA256_KEY]
-    else:
-        sha256data = bb.utils.sha256_file(ud.localpath)
+        if checksum_id in precomputed:
+            checksum_data = precomputed[checksum_id]
+        else:
+            checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath)
 
-    if ud.method.recommends_checksum(ud) and not ud.md5_expected and not ud.sha256_expected:
-        # If strict checking enabled and neither sum defined, raise error
+        checksum_expected = getattr(ud, "%s_expected" % checksum_id)
+
+        return {
+            "id": checksum_id,
+            "name": checksum_name,
+            "data": checksum_data,
+            "expected": checksum_expected
+        }
+
+    checksum_infos = []
+    for checksum_id in CHECKSUM_LIST:
+        checksum_infos.append(compute_checksum_info(checksum_id))
+
+    checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos}
+    checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos}
+
+    for ci in checksum_infos:
+        if ci["id"] in SHOWN_CHECKSUM_LIST:
+            checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
+
+    # If no checksum has been provided
+    if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
+        messages = []
         strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
-        if strict == "1":
-            logger.error('No checksum specified for %s, please add at least one to the recipe:\n'
-                             'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
-                             (ud.localpath, ud.md5_name, md5data,
-                              ud.sha256_name, sha256data))
-            raise NoChecksumError('Missing SRC_URI checksum', ud.url)
 
-        bb.event.fire(MissingChecksumEvent(ud.url, md5data, sha256data), d)
+        # If strict checking enabled and neither sum defined, raise error
+        if strict == "1":
+            messages.append("No checksum specified for '%s', please add at " \
+                            "least one to the recipe:" % ud.localpath)
+            messages.extend(checksum_lines)
+            logger.error("\n".join(messages))
+            raise NoChecksumError("Missing SRC_URI checksum", ud.url)
+
+        bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
 
         if strict == "ignore":
-            return {
-                _MD5_KEY: md5data,
-                _SHA256_KEY: sha256data
-            }
+            return checksum_dict
 
         # Log missing sums so user can more easily add them
-        logger.warning('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
-                       'SRC_URI[%s] = "%s"',
-                       ud.localpath, ud.md5_name, md5data)
-        logger.warning('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n'
-                       'SRC_URI[%s] = "%s"',
-                       ud.localpath, ud.sha256_name, sha256data)
+        messages.append("Missing checksum for '%s', consider adding at " \
+                        "least one to the recipe:" % ud.localpath)
+        messages.extend(checksum_lines)
+        logger.warning("\n".join(messages))
 
     # We want to alert the user if a checksum is defined in the recipe but
     # it does not match.
-    msg = ""
-    mismatch = False
-    if ud.md5_expected and ud.md5_expected != md5data:
-        msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'md5', md5data, ud.md5_expected)
-        mismatch = True;
+    messages = []
+    messages.append("Checksum mismatch!")
+    bad_checksum = None
 
-    if ud.sha256_expected and ud.sha256_expected != sha256data:
-        msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'sha256', sha256data, ud.sha256_expected)
-        mismatch = True;
+    for ci in checksum_infos:
+        if ci["expected"] and ci["expected"] != ci["data"]:
+            messages.append("File: '%s' has %s checksum %s when %s was " \
+                            "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
+            bad_checksum = ci["data"]
 
-    if mismatch:
-        msg = msg + '\nIf this change is expected (e.g. you have upgraded to a new version without updating the checksums) then you can use these lines within the recipe:\nSRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"\nOtherwise you should retry the download and/or check with upstream to determine if the file has become corrupted or otherwise unexpectedly modified.\n' % (ud.md5_name, md5data, ud.sha256_name, sha256data)
-
-    if len(msg):
-        raise ChecksumError('Checksum mismatch!%s' % msg, ud.url, md5data)
-
-    return {
-        _MD5_KEY: md5data,
-        _SHA256_KEY: sha256data
-    }
+    if bad_checksum:
+        messages.append("If this change is expected (e.g. you have upgraded " \
+                        "to a new version without updating the checksums) " \
+                        "then you can use these lines within the recipe:")
+        messages.extend(checksum_lines)
+        messages.append("Otherwise you should retry the download and/or " \
+                        "check with upstream to determine if the file has " \
+                        "become corrupted or otherwise unexpectedly modified.")
+        raise ChecksumError("\n".join(messages), ud.url, bad_checksum)
 
+    return checksum_dict
 
 def verify_donestamp(ud, d, origud=None):
     """
@@ -777,7 +774,8 @@ def get_srcrev(d, method_name='sortable_revision'):
     #
     format = d.getVar('SRCREV_FORMAT')
     if not format:
-        raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
+        raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.\n"\
+                         "The SCMs are:\n%s" % '\n'.join(scms))
 
     name_to_rev = {}
     seenautoinc = False
@@ -827,6 +825,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
                   'NO_PROXY', 'no_proxy',
                   'ALL_PROXY', 'all_proxy',
                   'GIT_PROXY_COMMAND',
+                  'GIT_SSH',
                   'GIT_SSL_CAINFO',
                   'GIT_SMART_HTTP',
                   'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
@@ -837,14 +836,16 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     if not cleanup:
         cleanup = []
 
-    # If PATH contains WORKDIR which contains PV which contains SRCPV we
+    # If PATH contains WORKDIR which contains PV-PR which contains SRCPV we
     # can end up in circular recursion here so give the option of breaking it
     # in a data store copy.
     try:
         d.getVar("PV")
+        d.getVar("PR")
     except bb.data_smart.ExpansionError:
         d = bb.data.createCopy(d)
         d.setVar("PV", "fetcheravoidrecurse")
+        d.setVar("PR", "fetcheravoidrecurse")
 
     origenv = d.getVar("BB_ORIGENV", False)
     for var in exportvars:
@@ -852,10 +853,18 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         if val:
             cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
 
+    # Ensure that a _PYTHON_SYSCONFIGDATA_NAME value set by a recipe
+    # (for example via python3native.bbclass since warrior) is not set for
+    # host Python (otherwise tools like git-make-shallow will fail)
+    cmd = 'unset _PYTHON_SYSCONFIGDATA_NAME; ' + cmd
+
     # Disable pseudo as it may affect ssh, potentially causing it to hang.
     cmd = 'export PSEUDO_DISABLED=1; ' + cmd
 
-    logger.debug(1, "Running %s", cmd)
+    if workdir:
+        logger.debug(1, "Running '%s' in %s" % (cmd, workdir))
+    else:
+        logger.debug(1, "Running %s", cmd)
 
     success = False
     error_message = ""
@@ -891,7 +900,7 @@ def check_network_access(d, info, url):
     log remote network access, and error if BB_NO_NETWORK is set or the given
     URI is untrusted
     """
-    if d.getVar("BB_NO_NETWORK") == "1":
+    if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
         raise NetworkAccess(url, info)
     elif not trusted_network(d, url):
         raise UntrustedUrl(url, info)
@@ -963,7 +972,8 @@ def rename_bad_checksum(ud, suffix):
 
     new_localpath = "%s_bad-checksum_%s" % (ud.localpath, suffix)
     bb.warn("Renaming %s to %s" % (ud.localpath, new_localpath))
-    bb.utils.movefile(ud.localpath, new_localpath)
+    if not bb.utils.movefile(ud.localpath, new_localpath):
+        bb.warn("Renaming %s to %s failed, grep movefile in log.do_fetch to see why" % (ud.localpath, new_localpath))
 
 
 def try_mirror_url(fetch, origud, ud, ld, check = False):
@@ -1016,16 +1026,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
                     origud.method.build_mirror_data(origud, ld)
             return origud.localpath
         # Otherwise the result is a local file:// and we symlink to it
-        if not os.path.exists(origud.localpath):
-            if os.path.islink(origud.localpath):
-                # Broken symbolic link
-                os.unlink(origud.localpath)
-
-            # As per above, in case two tasks end up here simultaneously.
-            try:
-                os.symlink(ud.localpath, origud.localpath)
-            except FileExistsError:
-                pass
+        ensure_symlink(ud.localpath, origud.localpath)
         update_stamp(origud, ld)
         return ud.localpath
 
@@ -1033,7 +1034,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
         raise
 
     except IOError as e:
-        if e.errno in [os.errno.ESTALE]:
+        if e.errno in [errno.ESTALE]:
             logger.warning("Stale Error Observed %s." % ud.url)
             return False
         raise
@@ -1059,6 +1060,22 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
             bb.utils.unlockfile(lf)
 
 
+def ensure_symlink(target, link_name):
+    if not os.path.exists(link_name):
+        if os.path.islink(link_name):
+            # Broken symbolic link
+            os.unlink(link_name)
+
+        # In case this is executing without any file locks held (as is
+        # the case for file:// URLs), two tasks may end up here at the
+        # same time, in which case we do not want the second task to
+        # fail when the link has already been created by the first task.
+        try:
+            os.symlink(target, link_name)
+        except FileExistsError:
+            pass
+
+
 def try_mirrors(fetch, d, origud, mirrors, check = False):
     """
     Try to use a mirrored version of the sources.
@@ -1074,7 +1091,7 @@ def try_mirrors(fetch, d, origud, mirrors, check = False):
 
     for index, uri in enumerate(uris):
         ret = try_mirror_url(fetch, origud, uds[index], ld, check)
-        if ret != False:
+        if ret:
             return ret
     return None
 
@@ -1084,11 +1101,13 @@ def trusted_network(d, url):
     BB_ALLOWED_NETWORKS is set globally or for a specific recipe.
     Note: modifies SRC_URI & mirrors.
     """
-    if d.getVar('BB_NO_NETWORK') == "1":
+    if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
         return True
 
     pkgname = d.expand(d.getVar('PN', False))
-    trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
+    trusted_hosts = None
+    if pkgname:
+        trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
 
     if not trusted_hosts:
         trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS')
@@ -1188,14 +1207,14 @@ def get_checksum_file_list(d):
 
     return " ".join(filelist)
 
-def get_file_checksums(filelist, pn):
+def get_file_checksums(filelist, pn, localdirsexclude):
     """Get a list of the checksums for a list of local files
 
     Returns the checksums for a list of local files, caching the results as
     it proceeds
 
     """
-    return _checksum_cache.get_checksums(filelist, pn)
+    return _checksum_cache.get_checksums(filelist, pn, localdirsexclude)
 
 
 class FetchData(object):
@@ -1221,24 +1240,26 @@ class FetchData(object):
             self.pswd = self.parm["pswd"]
         self.setup = False
 
-        if "name" in self.parm:
-            self.md5_name = "%s.md5sum" % self.parm["name"]
-            self.sha256_name = "%s.sha256sum" % self.parm["name"]
-        else:
-            self.md5_name = "md5sum"
-            self.sha256_name = "sha256sum"
-        if self.md5_name in self.parm:
-            self.md5_expected = self.parm[self.md5_name]
-        elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
-            self.md5_expected = None
-        else:
-            self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
-        if self.sha256_name in self.parm:
-            self.sha256_expected = self.parm[self.sha256_name]
-        elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
-            self.sha256_expected = None
-        else:
-            self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
+        def configure_checksum(checksum_id):
+            if "name" in self.parm:
+                checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id)
+            else:
+                checksum_name = "%ssum" % checksum_id
+
+            setattr(self, "%s_name" % checksum_id, checksum_name)
+
+            if checksum_name in self.parm:
+                checksum_expected = self.parm[checksum_name]
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
+                checksum_expected = None
+            else:
+                checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
+
+            setattr(self, "%s_expected" % checksum_id, checksum_expected)
+
+        for checksum_id in CHECKSUM_LIST:
+            configure_checksum(checksum_id)
+
         self.ignore_checksums = False
 
         self.names = self.parm.get("name",'default').split(',')
@@ -1342,7 +1363,7 @@ class FetchMethod(object):
         """
 
         # We cannot compute checksums for directories
-        if os.path.isdir(urldata.localpath) == True:
+        if os.path.isdir(urldata.localpath):
             return False
         if urldata.localpath.find("*") != -1:
             return False
@@ -1356,6 +1377,18 @@ class FetchMethod(object):
         """
         return False
 
+    def verify_donestamp(self, ud, d):
+        """
+        Verify the donestamp file
+        """
+        return verify_donestamp(ud, d)
+
+    def update_donestamp(self, ud, d):
+        """
+        Update the donestamp file
+        """
+        update_stamp(ud, d)
+
     def _strip_leading_slashes(self, relpath):
         """
         Remove leading slash as os.path.join can't cope
@@ -1391,7 +1424,7 @@ class FetchMethod(object):
         Fetch urls
         Assumes localpath was called first
         """
-        raise NoMethodError(url)
+        raise NoMethodError(urldata.url)
 
     def unpack(self, urldata, rootdir, data):
         iterate = False
@@ -1457,7 +1490,7 @@ class FetchMethod(object):
                 else:
                     cmd = 'rpm2cpio.sh %s | cpio -id' % (file)
             elif file.endswith('.deb') or file.endswith('.ipk'):
-                output = subprocess.check_output('ar -t %s' % file, preexec_fn=subprocess_setup, shell=True)
+                output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup)
                 datafile = None
                 if output:
                     for line in output.decode().splitlines():
@@ -1530,12 +1563,18 @@ class FetchMethod(object):
         """
         return True
 
+    def try_mirrors(self, fetch, urldata, d, mirrors, check=False):
+        """
+        Try to use a mirror
+        """
+        return bool(try_mirrors(fetch, d, urldata, mirrors, check))
+
     def checkstatus(self, fetch, urldata, d):
         """
         Check the status of a URL
         Assumes localpath was called first
         """
-        logger.info("URL %s could not be checked for status since no method exists.", url)
+        logger.info("URL %s could not be checked for status since no method exists.", urldata.url)
         return True
 
     def latest_revision(self, ud, d, name):
@@ -1543,7 +1582,7 @@ class FetchMethod(object):
         Look in the cache for the latest revision, if not present ask the SCM.
         """
         if not hasattr(self, "_latest_revision"):
-            raise ParameterError("The fetcher for this URL does not support _latest_revision", url)
+            raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url)
 
         revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
         key = self.generate_revision_key(ud, d, name)
@@ -1558,8 +1597,7 @@ class FetchMethod(object):
         return True, str(latest_rev)
 
     def generate_revision_key(self, ud, d, name):
-        key = self._revision_key(ud, d, name)
-        return "%s-%s" % (key, d.getVar("PN") or "")
+        return self._revision_key(ud, d, name)
 
     def latest_versionstring(self, ud, d):
         """
@@ -1569,6 +1607,16 @@ class FetchMethod(object):
         """
         return ('', '')
 
+    def done(self, ud, d):
+        """
+        Is the download done ?
+        """
+        if os.path.exists(ud.localpath):
+            return True
+        if ud.localpath.find("*") != -1:
+            return True
+        return False
+
 class Fetch(object):
     def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
         if localonly and cache:
@@ -1583,8 +1631,11 @@ class Fetch(object):
 
         fn = d.getVar('FILE')
         mc = d.getVar('__BBMULTICONFIG') or ""
-        if cache and fn and mc + fn in urldata_cache:
-            self.ud = urldata_cache[mc + fn]
+        key = None
+        if cache and fn:
+            key = mc + fn + str(id(d))
+        if key in urldata_cache:
+            self.ud = urldata_cache[key]
 
         for url in urls:
             if url not in self.ud:
@@ -1595,8 +1646,8 @@ class Fetch(object):
                         self.ud[url] = None
                         pass
 
-        if fn and cache:
-            urldata_cache[mc + fn] = self.ud
+        if key:
+            urldata_cache[key] = self.ud
 
     def localpath(self, url):
         if url not in self.urls:
@@ -1626,13 +1677,13 @@ class Fetch(object):
             urls = self.urls
 
         network = self.d.getVar("BB_NO_NETWORK")
-        premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY") == "1")
+        premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY"))
 
         for u in urls:
             ud = self.ud[u]
             ud.setup_localpath(self.d)
             m = ud.method
-            localpath = ""
+            done = False
 
             if ud.lockfile:
                 lf = bb.utils.lockfile(ud.lockfile)
@@ -1640,28 +1691,28 @@ class Fetch(object):
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
 
-                if verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
-                    localpath = ud.localpath
+                if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
+                    done = True
                 elif m.try_premirror(ud, self.d):
                     logger.debug(1, "Trying PREMIRRORS")
                     mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
-                    localpath = try_mirrors(self, self.d, ud, mirrors, False)
-                    if localpath:
+                    done = m.try_mirrors(self, ud, self.d, mirrors)
+                    if done:
                         try:
                             # early checksum verification so that if the checksum of the premirror
                             # contents mismatch the fetcher can still try upstream and mirrors
-                            update_stamp(ud, self.d)
+                            m.update_donestamp(ud, self.d)
                         except ChecksumError as e:
                             logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u)
                             logger.debug(1, str(e))
-                            localpath = ""
+                            done = False
 
                 if premirroronly:
                     self.d.setVar("BB_NO_NETWORK", "1")
 
                 firsterr = None
-                verified_stamp = verify_donestamp(ud, self.d)
-                if not localpath and (not verified_stamp or m.need_update(ud, self.d)):
+                verified_stamp = m.verify_donestamp(ud, self.d)
+                if not done and (not verified_stamp or m.need_update(ud, self.d)):
                     try:
                         if not trusted_network(self.d, ud.url):
                             raise UntrustedUrl(ud.url)
@@ -1669,10 +1720,10 @@ class Fetch(object):
                         m.download(ud, self.d)
                         if hasattr(m, "build_mirror_data"):
                             m.build_mirror_data(ud, self.d)
-                        localpath = ud.localpath
+                        done = True
                         # early checksum verify, so that if checksum mismatched,
                         # fetcher still have chance to fetch from mirror
-                        update_stamp(ud, self.d)
+                        m.update_donestamp(ud, self.d)
 
                     except bb.fetch2.NetworkAccess:
                         raise
@@ -1694,17 +1745,17 @@ class Fetch(object):
                             m.clean(ud, self.d)
                         logger.debug(1, "Trying MIRRORS")
                         mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
-                        localpath = try_mirrors(self, self.d, ud, mirrors)
+                        done = m.try_mirrors(self, ud, self.d, mirrors)
 
-                if not localpath or ((not os.path.exists(localpath)) and localpath.find("*") == -1):
+                if not done or not m.done(ud, self.d):
                     if firsterr:
                         logger.error(str(firsterr))
                     raise FetchError("Unable to fetch URL from any source.", u)
 
-                update_stamp(ud, self.d)
+                m.update_donestamp(ud, self.d)
 
             except IOError as e:
-                if e.errno in [os.errno.ESTALE]:
+                if e.errno in [errno.ESTALE]:
                     logger.error("Stale Error Observed %s." % u)
                     raise ChecksumError("Stale Error Detected")
 
@@ -1732,14 +1783,14 @@ class Fetch(object):
             logger.debug(1, "Testing URL %s", u)
             # First try checking uri, u, from PREMIRRORS
             mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
-            ret = try_mirrors(self, self.d, ud, mirrors, True)
+            ret = m.try_mirrors(self, ud, self.d, mirrors, True)
             if not ret:
                 # Next try checking from the original uri, u
                 ret = m.checkstatus(self, ud, self.d)
                 if not ret:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
-                    ret = try_mirrors(self, self.d, ud, mirrors, True)
+                    ret = m.try_mirrors(self, ud, self.d, mirrors, True)
 
             if not ret:
                 raise FetchError("URL %s doesn't work" % u, u)
@@ -1774,7 +1825,7 @@ class Fetch(object):
 
         for url in urls:
             if url not in self.ud:
-                self.ud[url] = FetchData(url, d)
+                self.ud[url] = FetchData(url, self.d)
             ud = self.ud[url]
             ud.setup_localpath(self.d)
 
@@ -1844,6 +1895,7 @@ from . import osc
 from . import repo
 from . import clearcase
 from . import npm
+from . import npmsw
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1862,3 +1914,4 @@ methods.append(osc.Osc())
 methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
 methods.append(npm.Npm())
+methods.append(npmsw.NpmShrinkWrap())

bitbake/lib/bb/fetch2/bzr.py

@@ -10,22 +10,10 @@ BitBake 'Fetch' implementation for bzr.
 #   BitBake build tools.
 #   Copyright (C) 2003, 2004  Chris Larson
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
-import sys
-import logging
 import bb
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
@@ -41,8 +29,9 @@ class Bzr(FetchMethod):
         init bzr specific variable within url data
         """
         # Create paths to bzr checkouts
+        bzrdir = d.getVar("BZRDIR") or (d.getVar("DL_DIR") + "/bzr")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.expand('${BZRDIR}'), ud.host, relpath)
+        ud.pkgdir = os.path.join(bzrdir, ud.host, relpath)
 
         ud.setup_revisions(d)
 
@@ -57,7 +46,7 @@ class Bzr(FetchMethod):
         command is "fetch", "update", "revno"
         """
 
-        basecmd = d.expand('${FETCHCMD_bzr}')
+        basecmd = d.getVar("FETCHCMD_bzr") or "/usr/bin/env bzr"
 
         proto =  ud.parm.get('protocol', 'http')
 

bitbake/lib/bb/fetch2/clearcase.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' clearcase implementation
 
@@ -47,29 +45,18 @@ User credentials:
 """
 # Copyright (C) 2014 Siemens AG
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 
 import os
-import sys
 import shutil
 import bb
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
+from   bb.fetch2 import MissingParameterError
+from   bb.fetch2 import ParameterError
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
-from   distutils import spawn
 
 class ClearCase(FetchMethod):
     """Class to fetch urls via 'clearcase'"""
@@ -93,7 +80,7 @@ class ClearCase(FetchMethod):
         if 'protocol' in ud.parm:
             ud.proto = ud.parm['protocol']
         if not ud.proto in ('http', 'https'):
-            raise fetch2.ParameterError("Invalid protocol type", ud.url)
+            raise ParameterError("Invalid protocol type", ud.url)
 
         ud.vob = ''
         if 'vob' in ud.parm:
@@ -107,7 +94,7 @@ class ClearCase(FetchMethod):
         else:
             ud.module = ""
 
-        ud.basecmd = d.getVar("FETCHCMD_ccrc") or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
+        ud.basecmd = d.getVar("FETCHCMD_ccrc") or "/usr/bin/env cleartool || rcleartool"
 
         if d.getVar("SRCREV") == "INVALID":
           raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")
@@ -157,18 +144,18 @@ class ClearCase(FetchMethod):
 
         basecmd = "%s %s" % (ud.basecmd, command)
 
-        if command is 'mkview':
+        if command == 'mkview':
             if not "rcleartool" in ud.basecmd:
                 # Cleartool needs a -snapshot view
                 options.append("-snapshot")
             options.append("-tag %s" % ud.viewname)
             options.append(ud.viewdir)
 
-        elif command is 'rmview':
+        elif command == 'rmview':
             options.append("-force")
             options.append("%s" % ud.viewdir)
 
-        elif command is 'setcs':
+        elif command == 'setcs':
             options.append("-overwrite")
             options.append(ud.configspecfile)
 
@@ -250,7 +237,7 @@ class ClearCase(FetchMethod):
 
         # Clean clearcase meta-data before tar
 
-        runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath])
+        runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath], workdir = ud.viewdir)
 
         # Clean up so we can create a new view next time
         self.clean(ud, d);

bitbake/lib/bb/fetch2/cvs.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' implementations
 
@@ -10,24 +8,12 @@ BitBake build tools.
 
 # Copyright (C) 2003, 2004  Chris Larson
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-#Based on functions from the base bb module, Copyright 2003 Holger Schurig
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
 #
 
 import os
-import logging
 import bb
 from bb.fetch2 import FetchMethod, FetchError, MissingParameterError, logger
 from bb.fetch2 import runfetchcmd
@@ -110,7 +96,7 @@ class Cvs(FetchMethod):
         if ud.tag:
             options.append("-r %s" % ud.tag)
 
-        cvsbasecmd = d.getVar("FETCHCMD_cvs")
+        cvsbasecmd = d.getVar("FETCHCMD_cvs") or "/usr/bin/env cvs"
         cvscmd = cvsbasecmd + " '-d" + cvsroot + "' co " + " ".join(options) + " " + ud.module
         cvsupdatecmd = cvsbasecmd + " '-d" + cvsroot + "' update -d -P " + " ".join(options)
 
@@ -121,7 +107,8 @@ class Cvs(FetchMethod):
         # create module directory
         logger.debug(2, "Fetch: checking for module directory")
         pkg = d.getVar('PN')
-        pkgdir = os.path.join(d.getVar('CVSDIR'), pkg)
+        cvsdir = d.getVar("CVSDIR") or (d.getVar("DL_DIR") + "/cvs")
+        pkgdir = os.path.join(cvsdir, pkg)
         moddir = os.path.join(pkgdir, localdir)
         workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):

bitbake/lib/bb/fetch2/git.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' git implementation
 
@@ -55,20 +53,10 @@ Supported SRC_URI options are:
 
 """
 
-#Copyright (C) 2005 Richard Purdie
+# Copyright (C) 2005 Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import collections
 import errno
@@ -199,7 +187,7 @@ class Git(FetchMethod):
             depth_default = 1
         ud.shallow_depths = collections.defaultdict(lambda: depth_default)
 
-        revs_default = d.getVar("BB_GIT_SHALLOW_REVS", True)
+        revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
         ud.shallow_revs = []
         ud.branches = {}
         for pos, name in enumerate(ud.names):
@@ -248,7 +236,7 @@ class Git(FetchMethod):
                     ud.unresolvedrev[name] = ud.revisions[name]
                 ud.revisions[name] = self.latest_revision(ud, d, name)
 
-        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))
+        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
         if gitsrcname.startswith('.'):
             gitsrcname = gitsrcname[1:]
 
@@ -261,7 +249,7 @@ class Git(FetchMethod):
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
 
         dl_dir = d.getVar("DL_DIR")
-        gitdir = d.getVar("GITDIR") or (dl_dir + "/git2/")
+        gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
         ud.clonedir = os.path.join(gitdir, gitsrcname)
         ud.localfile = ud.clonedir
 
@@ -299,21 +287,36 @@ class Git(FetchMethod):
         return ud.clonedir
 
     def need_update(self, ud, d):
+        return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
+
+    def clonedir_need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
             return True
+        if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
+            return True
         for name in ud.names:
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 return True
-        if ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow):
-            return True
-        if ud.write_tarballs and not os.path.exists(ud.fullmirror):
-            return True
         return False
 
+    def clonedir_need_shallow_revs(self, ud, d):
+        for rev in ud.shallow_revs:
+            try:
+                runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir)
+            except bb.fetch2.FetchError:
+                return rev
+        return None
+
+    def shallow_tarball_need_update(self, ud):
+        return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
+
+    def tarball_need_update(self, ud):
+        return ud.write_tarballs and not os.path.exists(ud.fullmirror)
+
     def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
-        if d.getVar("BB_FETCH_PREMIRRORONLY") is not None:
+        if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
             return True
         if os.path.exists(ud.clonedir):
             return False
@@ -322,16 +325,13 @@ class Git(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        no_clone = not os.path.exists(ud.clonedir)
-        need_update = no_clone or self.need_update(ud, d)
-
         # A current clone is preferred to either tarball, a shallow tarball is
         # preferred to an out of date clone, and a missing clone will use
         # either tarball.
-        if ud.shallow and os.path.exists(ud.fullshallow) and need_update:
+        if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
             ud.localpath = ud.fullshallow
             return
-        elif os.path.exists(ud.fullmirror) and no_clone:
+        elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
             bb.utils.mkdirhier(ud.clonedir)
             runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
 
@@ -342,25 +342,20 @@ class Git(FetchMethod):
             # We do this since git will use a "-l" option automatically for local urls where possible
             if repourl.startswith("file://"):
                 repourl = repourl[7:]
-            clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
+            clone_cmd = "LANG=C %s clone --bare --mirror \"%s\" %s --progress" % (ud.basecmd, repourl, ud.clonedir)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
             runfetchcmd(clone_cmd, d, log=progresshandler)
 
         # Update the checkout if needed
-        needupdate = False
-        for name in ud.names:
-            if not self._contains_ref(ud, d, name, ud.clonedir):
-                needupdate = True
-        if needupdate:
-            try: 
-                runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
-            except bb.fetch2.FetchError:
-                logger.debug(1, "No Origin")
+        if self.clonedir_need_update(ud, d):
+            output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
+            if "origin" in output:
+              runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
 
-            runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
-            fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
+            runfetchcmd("%s remote add --mirror=fetch origin \"%s\"" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
+            fetch_cmd = "LANG=C %s fetch -f --progress \"%s\" refs/*:refs/*" % (ud.basecmd, repourl)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
@@ -373,10 +368,16 @@ class Git(FetchMethod):
             except OSError as exc:
                 if exc.errno != errno.ENOENT:
                     raise
+
         for name in ud.names:
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
 
+        if ud.shallow and ud.write_shallow_tarballs:
+            missing_rev = self.clonedir_need_shallow_revs(ud, d)
+            if missing_rev:
+                raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
+
     def build_mirror_data(self, ud, d):
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):
@@ -472,14 +473,42 @@ class Git(FetchMethod):
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
 
-        if ud.shallow and (not os.path.exists(ud.clonedir) or self.need_update(ud, d)):
-            bb.utils.mkdirhier(destdir)
-            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
-        else:
-            runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
+        need_lfs = ud.parm.get("lfs", "1") == "1"
+
+        source_found = False
+        source_error = []
+
+        if not source_found:
+            clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
+            if clonedir_is_up_to_date:
+                runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
+                source_found = True
+            else:
+                source_error.append("clone directory not available or not up to date: " + ud.clonedir)
+
+        if not source_found:
+            if ud.shallow:
+                if os.path.exists(ud.fullshallow):
+                    bb.utils.mkdirhier(destdir)
+                    runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
+                    source_found = True
+                else:
+                    source_error.append("shallow clone not available: " + ud.fullshallow)
+            else:
+                source_error.append("shallow clone not enabled")
+
+        if not source_found:
+            raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
 
         repourl = self._get_repo_url(ud)
-        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)
+        runfetchcmd("%s remote set-url origin \"%s\"" % (ud.basecmd, repourl), d, workdir=destdir)
+
+        if self._contains_lfs(ud, d, destdir):
+            if need_lfs and not self._find_git_lfs(d):
+                raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
+            else:
+                bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
+
         if not ud.nocheckout:
             if subdir != "":
                 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
@@ -499,9 +528,17 @@ class Git(FetchMethod):
     def clean(self, ud, d):
         """ clean the git directory """
 
-        bb.utils.remove(ud.localpath, True)
-        bb.utils.remove(ud.fullmirror)
-        bb.utils.remove(ud.fullmirror + ".done")
+        to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"]
+        # The localpath is a symlink to clonedir when it is cloned from a
+        # mirror, so remove both of them.
+        if os.path.islink(ud.localpath):
+            clonedir = os.path.realpath(ud.localpath)
+            to_remove.append(clonedir)
+
+        for r in to_remove:
+            if os.path.exists(r):
+                bb.note('Removing %s' % r)
+                bb.utils.remove(r, True)
 
     def supports_srcrev(self):
         return True
@@ -522,6 +559,27 @@ class Git(FetchMethod):
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
+    def _contains_lfs(self, ud, d, wd):
+        """
+        Check if the repository has 'lfs' (large file) content
+        """
+        cmd = "%s grep lfs HEAD:.gitattributes | wc -l" % (
+                ud.basecmd)
+        try:
+            output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
+            if int(output) > 0:
+                return True
+        except (bb.fetch2.FetchError,ValueError):
+            pass
+        return False
+
+    def _find_git_lfs(self, d):
+        """
+        Return True if git-lfs can be found, False otherwise.
+        """
+        import shutil
+        return shutil.which("git-lfs", path=d.getVar('PATH')) is not None
+
     def _get_repo_url(self, ud):
         """
         Return the repository URL
@@ -536,7 +594,9 @@ class Git(FetchMethod):
         """
         Return a unique key for the url
         """
-        return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name]
+        # Collapse adjacent slashes
+        slash_re = re.compile(r"/+")
+        return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
 
     def _lsremote(self, ud, d, search):
         """
@@ -553,7 +613,7 @@ class Git(FetchMethod):
         d.setVar('_BB_GIT_IN_LSREMOTE', '1')
         try:
             repourl = self._get_repo_url(ud)
-            cmd = "%s ls-remote %s %s" % \
+            cmd = "%s ls-remote \"%s\" %s" % \
                 (ud.basecmd, repourl, search)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, cmd, repourl)
@@ -592,7 +652,7 @@ class Git(FetchMethod):
         """
         pupver = ('', '')
 
-        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or "(?P<pver>([0-9][\.|_]?)+)")
+        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
         try:
             output = self._lsremote(ud, d, "refs/tags/*")
         except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
@@ -607,13 +667,13 @@ class Git(FetchMethod):
 
             tag_head = line.split("/")[-1]
             # Ignore non-released branches
-            m = re.search("(alpha|beta|rc|final)+", tag_head)
+            m = re.search(r"(alpha|beta|rc|final)+", tag_head)
             if m:
                 continue
 
             # search for version in the line
             tag = tagregex.search(tag_head)
-            if tag == None:
+            if tag is None:
                 continue
 
             tag = tag.group('pver')

bitbake/lib/bb/fetch2/gitannex.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' git annex implementation
 """
@@ -7,24 +5,12 @@ BitBake 'Fetch' git annex implementation
 # Copyright (C) 2014 Otavio Salvador
 # Copyright (C) 2014 O.S. Systems Software LTDA.
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import os
 import bb
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
-from   bb.fetch2 import logger
 
 class GitANNEX(Git):
     def supports(self, ud, d):

bitbake/lib/bb/fetch2/gitsm.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' git submodules implementation
 
@@ -16,24 +14,18 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r
 
 # Copyright (C) 2013 Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
 import bb
+import copy
+import shutil
+import tempfile
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
+from   bb.fetch2 import Fetch
 
 class GitSM(Git):
     def supports(self, ud, d):
@@ -42,94 +34,192 @@ class GitSM(Git):
         """
         return ud.type in ['gitsm']
 
-    def uses_submodules(self, ud, d, wd):
+    def process_submodules(self, ud, workdir, function, d):
+        """
+        Iterate over all of the submodules in this repository and execute
+        the 'function' for each of them.
+        """
+
+        submodules = []
+        paths = {}
+        revision = {}
+        uris = {}
+        subrevision = {}
+
+        def parse_gitmodules(gitmodules):
+            modules = {}
+            module = ""
+            for line in gitmodules.splitlines():
+                if line.startswith('[submodule'):
+                    module = line.split('"')[1]
+                    modules[module] = {}
+                elif module and line.strip().startswith('path'):
+                    path = line.split('=')[1].strip()
+                    modules[module]['path'] = path
+                elif module and line.strip().startswith('url'):
+                    url = line.split('=')[1].strip()
+                    modules[module]['url'] = url
+            return modules
+
+        # Collect the defined submodules, and their attributes
         for name in ud.names:
             try:
-                runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=wd)
-                return True
-            except bb.fetch.FetchError:
-                pass
-        return False
+                gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
+            except:
+                # No submodules to update
+                continue
 
-    def _set_relative_paths(self, repopath):
-        """
-        Fix submodule paths to be relative instead of absolute,
-        so that when we move the repo it doesn't break
-        (In Git 1.7.10+ this is done automatically)
-        """
-        submodules = []
-        with open(os.path.join(repopath, '.gitmodules'), 'r') as f:
-            for line in f.readlines():
-                if line.startswith('[submodule'):
-                    submodules.append(line.split('"')[1])
+            for m, md in parse_gitmodules(gitmodules).items():
+                try:
+                    module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
+                except:
+                    # If the command fails, we don't have a valid file to check.  If it doesn't
+                    # fail -- it still might be a failure, see next check...
+                    module_hash = ""
+
+                if not module_hash:
+                    logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
+                    continue
+
+                submodules.append(m)
+                paths[m] = md['path']
+                revision[m] = ud.revisions[name]
+                uris[m] = md['url']
+                subrevision[m] = module_hash.split()[2]
+
+                # Convert relative to absolute uri based on parent uri
+                if uris[m].startswith('..'):
+                    newud = copy.copy(ud)
+                    newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
+                    uris[m] = Git._get_repo_url(self, newud)
 
         for module in submodules:
-            repo_conf = os.path.join(repopath, module, '.git')
-            if os.path.exists(repo_conf):
-                with open(repo_conf, 'r') as f:
-                    lines = f.readlines()
-                newpath = ''
-                for i, line in enumerate(lines):
-                    if line.startswith('gitdir:'):
-                        oldpath = line.split(': ')[-1].rstrip()
-                        if oldpath.startswith('/'):
-                            newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module
-                            lines[i] = 'gitdir: %s\n' % newpath
-                            break
-                if newpath:
-                    with open(repo_conf, 'w') as f:
-                        for line in lines:
-                            f.write(line)
+            # Translate the module url into a SRC_URI
 
-            repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config')
-            if os.path.exists(repo_conf2):
-                with open(repo_conf2, 'r') as f:
-                    lines = f.readlines()
-                newpath = ''
-                for i, line in enumerate(lines):
-                    if line.lstrip().startswith('worktree = '):
-                        oldpath = line.split(' = ')[-1].rstrip()
-                        if oldpath.startswith('/'):
-                            newpath = '../' * (module.count('/') + 3) + module
-                            lines[i] = '\tworktree = %s\n' % newpath
-                            break
-                if newpath:
-                    with open(repo_conf2, 'w') as f:
-                        for line in lines:
-                            f.write(line)
+            if "://" in uris[module]:
+                # Properly formated URL already
+                proto = uris[module].split(':', 1)[0]
+                url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+            else:
+                if ":" in uris[module]:
+                    # Most likely an SSH style reference
+                    proto = "ssh"
+                    if ":/" in uris[module]:
+                        # Absolute reference, easy to convert..
+                        url = "gitsm://" + uris[module].replace(':/', '/', 1)
+                    else:
+                        # Relative reference, no way to know if this is right!
+                        logger.warning("Submodule included by %s refers to relative ssh reference %s.  References may fail if not absolute." % (ud.url, uris[module]))
+                        url = "gitsm://" + uris[module].replace(':', '/', 1)
+                else:
+                    # This has to be a file reference
+                    proto = "file"
+                    url = "gitsm://" + uris[module]
 
-    def update_submodules(self, ud, d):
-        # We have to convert bare -> full repo, do the submodule bit, then convert back
-        tmpclonedir = ud.clonedir + ".tmp"
-        gitdir = tmpclonedir + os.sep + ".git"
-        bb.utils.remove(tmpclonedir, True)
-        os.mkdir(tmpclonedir)
-        os.rename(ud.clonedir, gitdir)
-        runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
-        runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)
-        runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
-        runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=tmpclonedir)
-        self._set_relative_paths(tmpclonedir)
-        runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)
-        os.rename(gitdir, ud.clonedir,)
-        bb.utils.remove(tmpclonedir, True)
+            url += ';protocol=%s' % proto
+            url += ";name=%s" % module
+            url += ";subpath=%s" % module
+
+            ld = d.createCopy()
+            # Not necessary to set SRC_URI, since we're passing the URI to
+            # Fetch.
+            #ld.setVar('SRC_URI', url)
+            ld.setVar('SRCREV_%s' % module, subrevision[module])
+
+            # Workaround for issues with SRCPV/SRCREV_FORMAT errors
+            # error refer to 'multiple' repositories.  Only the repository
+            # in the original SRC_URI actually matters...
+            ld.setVar('SRCPV', d.getVar('SRCPV'))
+            ld.setVar('SRCREV_FORMAT', module)
+
+            function(ud, url, module, paths[module], workdir, ld)
+
+        return submodules != []
+
+    def need_update(self, ud, d):
+        if Git.need_update(self, ud, d):
+            return True
+
+        try:
+            # Check for the nugget dropped by the download operation
+            known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
+                            (ud.basecmd), d, workdir=ud.clonedir)
+
+            if ud.revisions[ud.names[0]] not in known_srcrevs.split():
+                return True
+        except bb.fetch2.FetchError:
+            # No srcrev nuggets, so this is new and needs to be updated
+            return True
+
+        return False
 
     def download(self, ud, d):
+        def download_submodule(ud, url, module, modpath, workdir, d):
+            url += ";bareclone=1;nobranch=1"
+
+            # Is the following still needed?
+            #url += ";nocheckout=1"
+
+            try:
+                newfetch = Fetch([url], d, cache=False)
+                newfetch.download()
+                # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
+                runfetchcmd("%s config --add bitbake.srcrev %s" % \
+                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=workdir)
+            except Exception as e:
+                logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
+                raise
+
         Git.download(self, ud, d)
 
-        if not ud.shallow or ud.localpath != ud.fullshallow:
-            submodules = self.uses_submodules(ud, d, ud.clonedir)
-            if submodules:
-                self.update_submodules(ud, d)
-
-    def clone_shallow_local(self, ud, dest, d):
-        super(GitSM, self).clone_shallow_local(ud, dest, d)
-
-        runfetchcmd('cp -fpPRH "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d)
+        # If we're using a shallow mirror tarball it needs to be unpacked
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+            self.process_submodules(ud, tmpdir, download_submodule, d)
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, download_submodule, d)
 
     def unpack(self, ud, destdir, d):
+        def unpack_submodules(ud, url, module, modpath, workdir, d):
+            url += ";bareclone=1;nobranch=1"
+
+            # Figure out where we clone over the bare submodules...
+            if ud.bareclone:
+                repo_conf = ud.destdir
+            else:
+                repo_conf = os.path.join(ud.destdir, '.git')
+
+            try:
+                newfetch = Fetch([url], d, cache=False)
+                newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
+            except Exception as e:
+                logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
+                raise
+
+            local_path = newfetch.localpath(url)
+
+            # Correct the submodule references to the local download version...
+            runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
+
+            if ud.shallow:
+                runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
+
+            # Ensure the submodule repository is NOT set to bare, since we're checking it out...
+            try:
+                runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module))
+            except:
+                logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module))
+                raise
+
         Git.unpack(self, ud, destdir, d)
 
-        if self.uses_submodules(ud, d, ud.destdir):
-            runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir)
-            runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=ud.destdir)
+        ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
+
+        if not ud.bareclone and ret:
+            # All submodules should already be downloaded and configured in the tree.  This simply sets
+            # up the configuration and checks out the files.  The main project config should remain
+            # unmodified, and no download from the internet should occur.
+            runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)

bitbake/lib/bb/fetch2/hg.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' implementation for mercurial DRCS (hg).
 
@@ -9,24 +7,12 @@ BitBake 'Fetch' implementation for mercurial DRCS (hg).
 # Copyright (C) 2004        Marcin Juszkiewicz
 # Copyright (C) 2007        Robert Schuster
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
+#
 
 import os
-import sys
-import logging
 import bb
 import errno
 from bb.fetch2 import FetchMethod
@@ -66,13 +52,6 @@ class Hg(FetchMethod):
         else:
             ud.proto = "hg"
 
-        ud.setup_revisions(d)
-
-        if 'rev' in ud.parm:
-            ud.revision = ud.parm['rev']
-        elif not ud.revision:
-            ud.revision = self.latest_revision(ud, d)
-
         # Create paths to mercurial checkouts
         hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
                             ud.host, ud.path.replace('/', '.'))
@@ -80,12 +59,19 @@ class Hg(FetchMethod):
         ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
         ud.mirrortarballs = [mirrortarball]
 
-        hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/")
+        hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg")
         ud.pkgdir = os.path.join(hgdir, hgsrcname)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
         ud.localfile = ud.moddir
         ud.basecmd = d.getVar("FETCHCMD_hg") or "/usr/bin/env hg"
 
+        ud.setup_revisions(d)
+
+        if 'rev' in ud.parm:
+            ud.revision = ud.parm['rev']
+        elif not ud.revision:
+            ud.revision = self.latest_revision(ud, d)
+
         ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS")
 
     def need_update(self, ud, d):
@@ -99,7 +85,7 @@ class Hg(FetchMethod):
     def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
-        if d.getVar("BB_FETCH_PREMIRRORONLY") is not None:
+        if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
             return True
         if os.path.exists(ud.moddir):
             return False
@@ -151,7 +137,7 @@ class Hg(FetchMethod):
                 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto)
             else:
                 cmd = "%s pull" % (ud.basecmd)
-        elif command == "update":
+        elif command == "update" or command == "up":
             if ud.user and ud.pswd:
                 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options))
             else:
@@ -259,12 +245,19 @@ class Hg(FetchMethod):
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata != "nokeep":
+            proto = ud.parm.get('protocol', 'http')
             if not os.access(os.path.join(codir, '.hg'), os.R_OK):
                 logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
                 runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
             logger.debug(2, "Unpack: updating source in '" + codir + "'")
-            runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir)
-            runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
+            if ud.user and ud.pswd:
+                runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir)
+            else:
+                runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir)
+            if ud.user and ud.pswd:
+                runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" up -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, revflag), d, workdir=codir)
+            else:
+                runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
         else:
             logger.debug(2, "Unpack: extracting source to '" + codir + "'")
             runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)

bitbake/lib/bb/fetch2/local.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' implementations
 
@@ -10,20 +8,10 @@ BitBake build tools.
 
 # Copyright (C) 2003, 2004  Chris Larson
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
+#
 
 import os
 import urllib.request, urllib.parse, urllib.error

bitbake/lib/bb/fetch2/npm.py

@@ -1,309 +1,296 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+# Copyright (C) 2020 Savoir-Faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
 """
-BitBake 'Fetch' NPM implementation
+BitBake 'Fetch' npm implementation
 
-The NPM fetcher is used to retrieve files from the npmjs repository
+npm fetcher support the SRC_URI with format of:
+SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..."
 
-Usage in the recipe:
+Supported SRC_URI options are:
 
-    SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}"
-    Suported SRC_URI options are:
+- package
+   The npm package name. This is a mandatory parameter.
 
-    - name
-    - version
+- version
+    The npm package version. This is a mandatory parameter.
 
-    npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz  would become npm://registry.npmjs.org;name=${PN};version=${PV}
-    The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done
+- downloadfilename
+    Specifies the filename used when storing the downloaded file.
 
+- destsuffix
+    Specifies the directory to use to unpack the package (default: npm).
 """
 
-import os
-import sys
-import urllib.request, urllib.parse, urllib.error
+import base64
 import json
-import subprocess
-import signal
+import os
+import re
+import shlex
+import tempfile
 import bb
-from   bb.fetch2 import FetchMethod
-from   bb.fetch2 import FetchError
-from   bb.fetch2 import ChecksumError
-from   bb.fetch2 import runfetchcmd
-from   bb.fetch2 import logger
-from   bb.fetch2 import UnpackError
-from   bb.fetch2 import ParameterError
-from   distutils import spawn
+from bb.fetch2 import Fetch
+from bb.fetch2 import FetchError
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import MissingParameterError
+from bb.fetch2 import ParameterError
+from bb.fetch2 import URI
+from bb.fetch2 import check_network_access
+from bb.fetch2 import runfetchcmd
+from bb.utils import is_semver
 
-def subprocess_setup():
-    # Python installs a SIGPIPE handler by default. This is usually not what
-    # non-Python subprocesses expect.
-    # SIGPIPE errors are known issues with gzip/bash
-    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+def npm_package(package):
+    """Convert the npm package name to remove unsupported character"""
+    # Scoped package names (with the @) use the same naming convention
+    # as the 'npm pack' command.
+    if package.startswith("@"):
+        return re.sub("/", "-", package[1:])
+    return package
+
+def npm_filename(package, version):
+    """Get the filename of a npm package"""
+    return npm_package(package) + "-" + version + ".tgz"
+
+def npm_localfile(package, version):
+    """Get the local filename of a npm package"""
+    return os.path.join("npm2", npm_filename(package, version))
+
+def npm_integrity(integrity):
+    """
+    Get the checksum name and expected value from the subresource integrity
+        https://www.w3.org/TR/SRI/
+    """
+    algo, value = integrity.split("-", maxsplit=1)
+    return "%ssum" % algo, base64.b64decode(value).hex()
+
+def npm_unpack(tarball, destdir, d):
+    """Unpack a npm tarball"""
+    bb.utils.mkdirhier(destdir)
+    cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball)
+    cmd += " --no-same-owner"
+    cmd += " --strip-components=1"
+    runfetchcmd(cmd, d, workdir=destdir)
+
+class NpmEnvironment(object):
+    """
+    Using a npm config file seems more reliable than using cli arguments.
+    This class allows to create a controlled environment for npm commands.
+    """
+    def __init__(self, d, configs=None):
+        self.d = d
+        self.configs = configs
+
+    def run(self, cmd, args=None, configs=None, workdir=None):
+        """Run npm command in a controlled environment"""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            d = bb.data.createCopy(self.d)
+            d.setVar("HOME", tmpdir)
+
+            cfgfile = os.path.join(tmpdir, "npmrc")
+
+            if not workdir:
+                workdir = tmpdir
+
+            def _run(cmd):
+                cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd
+                cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd
+                return runfetchcmd(cmd, d, workdir=workdir)
+
+            if self.configs:
+                for key, value in self.configs:
+                    _run("npm config set %s %s" % (key, shlex.quote(value)))
+
+            if configs:
+                for key, value in configs:
+                    _run("npm config set %s %s" % (key, shlex.quote(value)))
+
+            if args:
+                for key, value in args:
+                    cmd += " --%s=%s" % (key, shlex.quote(value))
+
+            return _run(cmd)
 
 class Npm(FetchMethod):
-
-    """Class to fetch urls via 'npm'"""
-    def init(self, d):
-        pass
+    """Class to fetch a package from a npm registry"""
 
     def supports(self, ud, d):
-        """
-        Check to see if a given url can be fetched with npm
-        """
-        return ud.type in ['npm']
-
-    def debug(self, msg):
-        logger.debug(1, "NpmFetch: %s", msg)
-
-    def clean(self, ud, d):
-        logger.debug(2, "Calling cleanup %s" % ud.pkgname)
-        bb.utils.remove(ud.localpath, False)
-        bb.utils.remove(ud.pkgdatadir, True)
-        bb.utils.remove(ud.fullmirror, False)
+        """Check if a given url can be fetched with npm"""
+        return ud.type in ["npm"]
 
     def urldata_init(self, ud, d):
-        """
-        init NPM specific variable within url data
-        """
-        if 'downloadfilename' in ud.parm:
-            ud.basename = ud.parm['downloadfilename']
-        else:
-            ud.basename = os.path.basename(ud.path)
+        """Init npm specific variables within url data"""
+        ud.package = None
+        ud.version = None
+        ud.registry = None
+
+        # Get the 'package' parameter
+        if "package" in ud.parm:
+            ud.package = ud.parm.get("package")
+
+        if not ud.package:
+            raise MissingParameterError("Parameter 'package' required", ud.url)
+
+        # Get the 'version' parameter
+        if "version" in ud.parm:
+            ud.version = ud.parm.get("version")
 
-        # can't call it ud.name otherwise fetcher base class will start doing sha1stuff
-        # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps
-        ud.pkgname = ud.parm.get("name", None)
-        if not ud.pkgname:
-            raise ParameterError("NPM fetcher requires a name parameter", ud.url)
-        ud.version = ud.parm.get("version", None)
         if not ud.version:
-            raise ParameterError("NPM fetcher requires a version parameter", ud.url)
-        ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
-        ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-')
-        ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0]
-        prefixdir = "npm/%s" % ud.pkgname
-        ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
-        if not os.path.exists(ud.pkgdatadir):
-            bb.utils.mkdirhier(ud.pkgdatadir)
-        ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
+            raise MissingParameterError("Parameter 'version' required", ud.url)
 
-        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
-        ud.prefixdir = prefixdir
+        if not is_semver(ud.version) and not ud.version == "latest":
+            raise ParameterError("Invalid 'version' parameter", ud.url)
 
-        ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
-        mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
-        mirrortarball = mirrortarball.replace('/', '-')
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
-        ud.mirrortarballs = [mirrortarball]
+        # Extract the 'registry' part of the url
+        ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0])
+
+        # Using the 'downloadfilename' parameter as local filename
+        # or the npm package name.
+        if "downloadfilename" in ud.parm:
+            ud.localfile = d.expand(ud.parm["downloadfilename"])
+        else:
+            ud.localfile = npm_localfile(ud.package, ud.version)
+
+        # Get the base 'npm' command
+        ud.basecmd = d.getVar("FETCHCMD_npm") or "npm"
+
+        # This fetcher resolves a URI from a npm package name and version and
+        # then forwards it to a proxy fetcher. A resolve file containing the
+        # resolved URI is created to avoid unwanted network access (if the file
+        # already exists). The management of the donestamp file, the lockfile
+        # and the checksums are forwarded to the proxy fetcher.
+        ud.proxy = None
+        ud.needdonestamp = False
+        ud.resolvefile = self.localpath(ud, d) + ".resolved"
+
+    def _resolve_proxy_url(self, ud, d):
+        def _npm_view():
+            configs = []
+            configs.append(("json", "true"))
+            configs.append(("registry", ud.registry))
+            pkgver = shlex.quote(ud.package + "@" + ud.version)
+            cmd = ud.basecmd + " view %s" % pkgver
+            env = NpmEnvironment(d)
+            check_network_access(d, cmd, ud.registry)
+            view_string = env.run(cmd, configs=configs)
+
+            if not view_string:
+                raise FetchError("Unavailable package %s" % pkgver, ud.url)
+
+            try:
+                view = json.loads(view_string)
+
+                error = view.get("error")
+                if error is not None:
+                    raise FetchError(error.get("summary"), ud.url)
+
+                if ud.version == "latest":
+                    bb.warn("The npm package %s is using the latest " \
+                            "version available. This could lead to " \
+                            "non-reproducible builds." % pkgver)
+                elif ud.version != view.get("version"):
+                    raise ParameterError("Invalid 'version' parameter", ud.url)
+
+                return view
+
+            except Exception as e:
+                raise FetchError("Invalid view from npm: %s" % str(e), ud.url)
+
+        def _get_url(view):
+            tarball_url = view.get("dist", {}).get("tarball")
+
+            if tarball_url is None:
+                raise FetchError("Invalid 'dist.tarball' in view", ud.url)
+
+            uri = URI(tarball_url)
+            uri.params["downloadfilename"] = ud.localfile
+
+            integrity = view.get("dist", {}).get("integrity")
+            shasum = view.get("dist", {}).get("shasum")
+
+            if integrity is not None:
+                checksum_name, checksum_expected = npm_integrity(integrity)
+                uri.params[checksum_name] = checksum_expected
+            elif shasum is not None:
+                uri.params["sha1sum"] = shasum
+            else:
+                raise FetchError("Invalid 'dist.integrity' in view", ud.url)
+
+            return str(uri)
+
+        url = _get_url(_npm_view())
+
+        bb.utils.mkdirhier(os.path.dirname(ud.resolvefile))
+        with open(ud.resolvefile, "w") as f:
+            f.write(url)
+
+    def _setup_proxy(self, ud, d):
+        if ud.proxy is None:
+            if not os.path.exists(ud.resolvefile):
+                self._resolve_proxy_url(ud, d)
+
+            with open(ud.resolvefile, "r") as f:
+                url = f.read()
+
+            # Avoid conflicts between the environment data and:
+            # - the proxy url checksum
+            data = bb.data.createCopy(d)
+            data.delVarFlags("SRC_URI")
+            ud.proxy = Fetch([url], data)
+
+    def _get_proxy_method(self, ud, d):
+        self._setup_proxy(ud, d)
+        proxy_url = ud.proxy.urls[0]
+        proxy_ud = ud.proxy.ud[proxy_url]
+        proxy_d = ud.proxy.d
+        proxy_ud.setup_localpath(proxy_d)
+        return proxy_ud.method, proxy_ud, proxy_d
+
+    def verify_donestamp(self, ud, d):
+        """Verify the donestamp file"""
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        return proxy_m.verify_donestamp(proxy_ud, proxy_d)
+
+    def update_donestamp(self, ud, d):
+        """Update the donestamp file"""
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        proxy_m.update_donestamp(proxy_ud, proxy_d)
 
     def need_update(self, ud, d):
-        if os.path.exists(ud.localpath):
-            return False
-        return True
+        """Force a fetch, even if localpath exists ?"""
+        if not os.path.exists(ud.resolvefile):
+            return True
+        if ud.version == "latest":
+            return True
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        return proxy_m.need_update(proxy_ud, proxy_d)
 
-    def _runwget(self, ud, d, command, quiet):
-        logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
-        bb.fetch2.check_network_access(d, command, ud.url)
-        dldir = d.getVar("DL_DIR")
-        runfetchcmd(command, d, quiet, workdir=dldir)
-
-    def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
-        file = data[pkg]['tgz']
-        logger.debug(2, "file to extract is %s" % file)
-        if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-            cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file)
-        else:
-            bb.fatal("NPM package %s downloaded not a tarball!" % file)
-
-        # Change to subdir before executing command
-        if not os.path.exists(destdir):
-            os.makedirs(destdir)
-        path = d.getVar('PATH')
-        if path:
-            cmd = "PATH=\"%s\" %s" % (path, cmd)
-        bb.note("Unpacking %s to %s/" % (file, destdir))
-        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir)
-
-        if ret != 0:
-            raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
-
-        if 'deps' not in data[pkg]:
-            return
-        for dep in data[pkg]['deps']:
-            self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d)
-
-
-    def unpack(self, ud, destdir, d):
-        dldir = d.getVar("DL_DIR")
-        with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile:
-            workobj = json.load(datafile)
-        dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname)
-
-        if 'subdir' in ud.parm:
-            unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir'))
-        else:
-            unpackdir = '%s/npmpkg' % destdir
-
-        self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d)
-
-    def _parse_view(self, output):
-        '''
-        Parse the output of npm view --json; the last JSON result
-        is assumed to be the one that we're interested in.
-        '''
-        pdata = None
-        outdeps = {}
-        datalines = []
-        bracelevel = 0
-        for line in output.splitlines():
-            if bracelevel:
-                datalines.append(line)
-            elif '{' in line:
-                datalines = []
-                datalines.append(line)
-            bracelevel = bracelevel + line.count('{') - line.count('}')
-        if datalines:
-            pdata = json.loads('\n'.join(datalines))
-        return pdata
-
-    def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None):
-        if fetchedlist is None:
-            fetchedlist = []
-        pkgfullname = pkg
-        if version != '*' and not '/' in version:
-            pkgfullname += "@'%s'" % version
-        logger.debug(2, "Calling getdeps on %s" % pkg)
-        fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry)
-        output = runfetchcmd(fetchcmd, d, True)
-        pdata = self._parse_view(output)
-        if not pdata:
-            raise FetchError("The command '%s' returned no output" % fetchcmd)
-        if optional:
-            pkg_os = pdata.get('os', None)
-            if pkg_os:
-                if not isinstance(pkg_os, list):
-                    pkg_os = [pkg_os]
-                blacklist = False
-                for item in pkg_os:
-                    if item.startswith('!'):
-                        blacklist = True
-                        break
-                if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
-                    logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
-                    return
-        #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile))
-        outputurl = pdata['dist']['tarball']
-        data[pkg] = {}
-        data[pkg]['tgz'] = os.path.basename(outputurl)
-        if outputurl in fetchedlist:
-            return
-
-        self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
-        fetchedlist.append(outputurl)
-
-        dependencies = pdata.get('dependencies', {})
-        optionalDependencies = pdata.get('optionalDependencies', {})
-        dependencies.update(optionalDependencies)
-        depsfound = {}
-        optdepsfound = {}
-        data[pkg]['deps'] = {}
-        for dep in dependencies:
-            if dep in optionalDependencies:
-                optdepsfound[dep] = dependencies[dep]
-            else:
-                depsfound[dep] = dependencies[dep]
-        for dep, version in optdepsfound.items():
-            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist)
-        for dep, version in depsfound.items():
-            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist)
-
-    def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True):
-        logger.debug(2, "NPM shrinkwrap file is %s" % data)
-        if toplevel:
-            name = data.get('name', None)
-            if name and name != pkg:
-                for obj in data.get('dependencies', []):
-                    if obj == pkg:
-                        self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False)
-                        return
-        outputurl = "invalid"
-        if ('resolved' not in data) or (not data['resolved'].startswith('http')):
-            # will be the case for ${PN}
-            fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry)
-            logger.debug(2, "Found this matching URL: %s" % str(fetchcmd))
-            outputurl = runfetchcmd(fetchcmd, d, True)
-        else:
-            outputurl = data['resolved']
-        self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
-        manifest[pkg] = {}
-        manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip()
-        manifest[pkg]['deps'] = {}
-
-        if pkg in lockdown:
-            sha1_expected = lockdown[pkg][version]
-            sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz']))
-            if sha1_expected != sha1_data:
-                msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected)
-                raise ChecksumError('Checksum mismatch!%s' % msg)
-        else:
-            logger.debug(2, "No lockdown data for %s@%s" % (pkg, version))
-
-        if 'dependencies' in data:
-            for obj in data['dependencies']:
-                logger.debug(2, "Found dep is %s" % str(obj))
-                self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False)
+    def try_mirrors(self, fetch, ud, d, mirrors):
+        """Try to use a mirror"""
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors)
 
     def download(self, ud, d):
         """Fetch url"""
-        jsondepobj = {}
-        shrinkobj = {}
-        lockdown = {}
+        self._setup_proxy(ud, d)
+        ud.proxy.download()
 
-        if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror):
-            dest = d.getVar("DL_DIR")
-            bb.utils.mkdirhier(dest)
-            runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest)
-            return
+    def unpack(self, ud, rootdir, d):
+        """Unpack the downloaded archive"""
+        destsuffix = ud.parm.get("destsuffix", "npm")
+        destdir = os.path.join(rootdir, destsuffix)
+        npm_unpack(ud.localpath, destdir, d)
 
-        if ud.parm.get("noverify", None) != '1':
-            shwrf = d.getVar('NPM_SHRINKWRAP')
-            logger.debug(2, "NPM shrinkwrap file is %s" % shwrf)
-            if shwrf:
-                try:
-                    with open(shwrf) as datafile:
-                        shrinkobj = json.load(datafile)
-                except Exception as e:
-                    raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e)))
-            elif not ud.ignore_checksums:
-                logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname)
-            lckdf = d.getVar('NPM_LOCKDOWN')
-            logger.debug(2, "NPM lockdown file is %s" % lckdf)
-            if lckdf:
-                try:
-                    with open(lckdf) as datafile:
-                        lockdown = json.load(datafile)
-                except Exception as e:
-                    raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e)))
-            elif not ud.ignore_checksums:
-                logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname)
+    def clean(self, ud, d):
+        """Clean any existing full or partial download"""
+        if os.path.exists(ud.resolvefile):
+            self._setup_proxy(ud, d)
+            ud.proxy.clean()
+            bb.utils.remove(ud.resolvefile)
 
-        if ('name' not in shrinkobj):
-            self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud)
-        else:
-            self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj)
-
-        with open(ud.localpath, 'w') as outfile:
-            json.dump(jsondepobj, outfile)
-
-    def build_mirror_data(self, ud, d):
-        # Generate a mirror tarball if needed
-        if ud.write_tarballs and not os.path.exists(ud.fullmirror):
-            # it's possible that this symlink points to read-only filesystem with PREMIRROR
-            if os.path.islink(ud.fullmirror):
-                os.unlink(ud.fullmirror)
-
-            dldir = d.getVar("DL_DIR")
-            logger.info("Creating tarball of npm data")
-            runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d,
-                        workdir=dldir)
-            runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir)
+    def done(self, ud, d):
+        """Is the download done ?"""
+        if not os.path.exists(ud.resolvefile):
+            return False
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        return proxy_m.done(proxy_ud, proxy_d)

bitbake/lib/bb/fetch2/npmsw.py

@@ -0,0 +1,255 @@
+# Copyright (C) 2020 Savoir-Faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+"""
+BitBake 'Fetch' npm shrinkwrap implementation
+
+npm fetcher support the SRC_URI with format of:
+SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..."
+
+Supported SRC_URI options are:
+
+- dev
+   Set to 1 to also install devDependencies.
+
+- destsuffix
+    Specifies the directory to use to unpack the dependencies (default: ${S}).
+"""
+
+import json
+import os
+import re
+import bb
+from bb.fetch2 import Fetch
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import ParameterError
+from bb.fetch2 import URI
+from bb.fetch2.npm import npm_integrity
+from bb.fetch2.npm import npm_localfile
+from bb.fetch2.npm import npm_unpack
+from bb.utils import is_semver
+
+def foreach_dependencies(shrinkwrap, callback=None, dev=False):
+    """
+        Run a callback for each dependencies of a shrinkwrap file.
+        The callback is using the format:
+            callback(name, params, deptree)
+        with:
+            name = the package name (string)
+            params = the package parameters (dictionary)
+            deptree = the package dependency tree (array of strings)
+    """
+    def _walk_deps(deps, deptree):
+        for name in deps:
+            subtree = [*deptree, name]
+            _walk_deps(deps[name].get("dependencies", {}), subtree)
+            if callback is not None:
+                if deps[name].get("dev", False) and not dev:
+                    continue
+                elif deps[name].get("bundled", False):
+                    continue
+                callback(name, deps[name], subtree)
+
+    _walk_deps(shrinkwrap.get("dependencies", {}), [])
+
+class NpmShrinkWrap(FetchMethod):
+    """Class to fetch all package from a shrinkwrap file"""
+
+    def supports(self, ud, d):
+        """Check if a given url can be fetched with npmsw"""
+        return ud.type in ["npmsw"]
+
+    def urldata_init(self, ud, d):
+        """Init npmsw specific variables within url data"""
+
+        # Get the 'shrinkwrap' parameter
+        ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])
+
+        # Get the 'dev' parameter
+        ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False)
+
+        # Resolve the dependencies
+        ud.deps = []
+
+        def _resolve_dependency(name, params, deptree):
+            url = None
+            localpath = None
+            extrapaths = []
+            destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
+            destsuffix = os.path.join(*destsubdirs)
+
+            integrity = params.get("integrity", None)
+            resolved = params.get("resolved", None)
+            version = params.get("version", None)
+
+            # Handle registry sources
+            if is_semver(version) and resolved and integrity:
+                localfile = npm_localfile(name, version)
+
+                uri = URI(resolved)
+                uri.params["downloadfilename"] = localfile
+
+                checksum_name, checksum_expected = npm_integrity(integrity)
+                uri.params[checksum_name] = checksum_expected
+
+                url = str(uri)
+
+                localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+
+                # Create a resolve file to mimic the npm fetcher and allow
+                # re-usability of the downloaded file.
+                resolvefile = localpath + ".resolved"
+
+                bb.utils.mkdirhier(os.path.dirname(resolvefile))
+                with open(resolvefile, "w") as f:
+                    f.write(url)
+
+                extrapaths.append(resolvefile)
+
+            # Handle http tarball sources
+            elif version.startswith("http") and integrity:
+                localfile = os.path.join("npm2", os.path.basename(version))
+
+                uri = URI(version)
+                uri.params["downloadfilename"] = localfile
+
+                checksum_name, checksum_expected = npm_integrity(integrity)
+                uri.params[checksum_name] = checksum_expected
+
+                url = str(uri)
+
+                localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+
+            # Handle git sources
+            elif version.startswith("git"):
+                regex = re.compile(r"""
+                    ^
+                    git\+
+                    (?P<protocol>[a-z]+)
+                    ://
+                    (?P<url>[^#]+)
+                    \#
+                    (?P<rev>[0-9a-f]+)
+                    $
+                    """, re.VERBOSE)
+
+                match = regex.match(version)
+
+                if not match:
+                    raise ParameterError("Invalid git url: %s" % version, ud.url)
+
+                groups = match.groupdict()
+
+                uri = URI("git://" + str(groups["url"]))
+                uri.params["protocol"] = str(groups["protocol"])
+                uri.params["rev"] = str(groups["rev"])
+                uri.params["destsuffix"] = destsuffix
+
+                url = str(uri)
+
+            # local tarball sources and local link sources are unsupported
+            else:
+                raise ParameterError("Unsupported dependency: %s" % name, ud.url)
+
+            ud.deps.append({
+                "url": url,
+                "localpath": localpath,
+                "extrapaths": extrapaths,
+                "destsuffix": destsuffix,
+            })
+
+        try:
+            with open(ud.shrinkwrap_file, "r") as f:
+                shrinkwrap = json.load(f)
+        except Exception as e:
+            raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url)
+
+        foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev)
+
+        # Avoid conflicts between the environment data and:
+        # - the proxy url revision
+        # - the proxy url checksum
+        data = bb.data.createCopy(d)
+        data.delVar("SRCREV")
+        data.delVarFlags("SRC_URI")
+
+        # This fetcher resolves multiple URIs from a shrinkwrap file and then
+        # forwards it to a proxy fetcher. The management of the donestamp file,
+        # the lockfile and the checksums are forwarded to the proxy fetcher.
+        ud.proxy = Fetch([dep["url"] for dep in ud.deps], data)
+        ud.needdonestamp = False
+
+    @staticmethod
+    def _foreach_proxy_method(ud, handle):
+        returns = []
+        for proxy_url in ud.proxy.urls:
+            proxy_ud = ud.proxy.ud[proxy_url]
+            proxy_d = ud.proxy.d
+            proxy_ud.setup_localpath(proxy_d)
+            returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+        return returns
+
+    def verify_donestamp(self, ud, d):
+        """Verify the donestamp file"""
+        def _handle(m, ud, d):
+            return m.verify_donestamp(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def update_donestamp(self, ud, d):
+        """Update the donestamp file"""
+        def _handle(m, ud, d):
+            m.update_donestamp(ud, d)
+        self._foreach_proxy_method(ud, _handle)
+
+    def need_update(self, ud, d):
+        """Force a fetch, even if localpath exists ?"""
+        def _handle(m, ud, d):
+            return m.need_update(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def try_mirrors(self, fetch, ud, d, mirrors):
+        """Try to use a mirror"""
+        def _handle(m, ud, d):
+            return m.try_mirrors(fetch, ud, d, mirrors)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def download(self, ud, d):
+        """Fetch url"""
+        ud.proxy.download()
+
+    def unpack(self, ud, rootdir, d):
+        """Unpack the downloaded dependencies"""
+        destdir = d.getVar("S")
+        destsuffix = ud.parm.get("destsuffix")
+        if destsuffix:
+            destdir = os.path.join(rootdir, destsuffix)
+
+        bb.utils.mkdirhier(destdir)
+        bb.utils.copyfile(ud.shrinkwrap_file,
+                          os.path.join(destdir, "npm-shrinkwrap.json"))
+
+        auto = [dep["url"] for dep in ud.deps if not dep["localpath"]]
+        manual = [dep for dep in ud.deps if dep["localpath"]]
+
+        if auto:
+            ud.proxy.unpack(destdir, auto)
+
+        for dep in manual:
+            depdestdir = os.path.join(destdir, dep["destsuffix"])
+            npm_unpack(dep["localpath"], depdestdir, d)
+
+    def clean(self, ud, d):
+        """Clean any existing full or partial download"""
+        ud.proxy.clean()
+
+        # Clean extra files
+        for dep in ud.deps:
+            for path in dep["extrapaths"]:
+                bb.utils.remove(path)
+
+    def done(self, ud, d):
+        """Is the download done ?"""
+        def _handle(m, ud, d):
+            return m.done(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))

bitbake/lib/bb/fetch2/osc.py

@@ -1,13 +1,12 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
 """
 Bitbake "Fetch" implementation for osc (Opensuse build service client).
 Based on the svn "Fetch" implementation.
 
 """
 
-import  os
-import  sys
 import logging
 import  bb
 from    bb.fetch2 import FetchMethod
@@ -32,8 +31,9 @@ class Osc(FetchMethod):
         ud.module = ud.parm["module"]
 
         # Create paths to osc checkouts
+        oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.getVar('OSCDIR'), ud.host)
+        ud.pkgdir = os.path.join(oscdir, ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
         if 'rev' in ud.parm:
@@ -41,7 +41,7 @@ class Osc(FetchMethod):
         else:
             pv = d.getVar("PV", False)
             rev = bb.fetch2.srcrev_internal_helper(ud, d)
-            if rev and rev != True:
+            if rev:
                 ud.revision = rev
             else:
                 ud.revision = ""
@@ -54,7 +54,7 @@ class Osc(FetchMethod):
         command is "fetch", "update", "info"
         """
 
-        basecmd = d.expand('${FETCHCMD_osc}')
+        basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc"
 
         proto = ud.parm.get('protocol', 'ocs')
 

bitbake/lib/bb/fetch2/perforce.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' implementation for perforce
 
@@ -8,23 +6,11 @@ BitBake 'Fetch' implementation for perforce
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2016 Kodak Alaris, Inc.
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import os
-import logging
 import bb
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
@@ -43,13 +29,9 @@ class Perforce(FetchMethod):
         provided by the env, use it.  If P4PORT is specified by the recipe, use
         its values, which may override the settings in P4CONFIG.
         """
-        ud.basecmd = d.getVar('FETCHCMD_p4')
-        if not ud.basecmd:
-            ud.basecmd = "/usr/bin/env p4"
+        ud.basecmd = d.getVar("FETCHCMD_p4") or "/usr/bin/env p4"
 
-        ud.dldir = d.getVar('P4DIR')
-        if not ud.dldir:
-            ud.dldir = '%s/%s' % (d.getVar('DL_DIR'), 'p4')
+        ud.dldir = d.getVar("P4DIR") or (d.getVar("DL_DIR") + "/p4")
 
         path = ud.url.split('://')[1]
         path = path.split(';')[0]
@@ -122,7 +104,7 @@ class Perforce(FetchMethod):
         if command == 'changes':
             p4cmd = '%s%s changes -m 1 //%s' % (ud.basecmd, p4opt, pathnrev)
         elif command == 'print':
-            if depot_filename != None:
+            if depot_filename is not None:
                 p4cmd = '%s%s print -o "p4/%s" "%s"' % (ud.basecmd, p4opt, filename, depot_filename)
             else:
                 raise FetchError('No depot file name provided to p4 %s' % command, ud.url)

bitbake/lib/bb/fetch2/repo.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake "Fetch" repo (git) implementation
 
@@ -8,20 +6,10 @@ BitBake "Fetch" repo (git) implementation
 # Copyright (C) 2009 Tom Rini <trini@embeddedalley.com>
 #
 # Based on git.py which is:
-#Copyright (C) 2005 Richard Purdie
+# Copyright (C) 2005 Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
 import bb
@@ -45,6 +33,8 @@ class Repo(FetchMethod):
         "master".
         """
 
+        ud.basecmd = d.getVar("FETCHCMD_repo") or "/usr/bin/env repo"
+
         ud.proto = ud.parm.get('protocol', 'git')
         ud.branch = ud.parm.get('branch', 'master')
         ud.manifest = ud.parm.get('manifest', 'default.xml')
@@ -60,8 +50,8 @@ class Repo(FetchMethod):
             logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
             return
 
+        repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo")
         gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
-        repodir = d.getVar("REPODIR") or os.path.join(d.getVar("DL_DIR"), "repo")
         codir = os.path.join(repodir, gitsrcname, ud.manifest)
 
         if ud.user:
@@ -72,11 +62,11 @@ class Repo(FetchMethod):
         repodir = os.path.join(codir, "repo")
         bb.utils.mkdirhier(repodir)
         if not os.path.exists(os.path.join(repodir, ".repo")):
-            bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
-            runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
+            bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
+            runfetchcmd("%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
 
-        bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
-        runfetchcmd("repo sync", d, workdir=repodir)
+        bb.fetch2.check_network_access(d, "%s sync %s" % (ud.basecmd, ud.url), ud.url)
+        runfetchcmd("%s sync" % ud.basecmd, d, workdir=repodir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":

bitbake/lib/bb/fetch2/s3.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' implementation for Amazon AWS S3.
 
@@ -13,18 +11,7 @@ The aws tool must be correctly installed and configured prior to use.
 # Based in part on bb.fetch2.wget:
 #    Copyright (C) 2003, 2004  Chris Larson
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 

bitbake/lib/bb/fetch2/sftp.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake SFTP Fetch implementation
 
@@ -44,18 +42,7 @@ SRC_URI = "sftp://user@host.example.com/dir/path.file.txt"
 # Based in part on bb.fetch2.wget:
 #    Copyright (C) 2003, 2004  Chris Larson
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 

bitbake/lib/bb/fetch2/ssh.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 '''
 BitBake 'Fetch' implementations
 
@@ -29,23 +27,11 @@ IETF secsh internet draft:
 #            Copyright 2003 Holger Schurig
 #
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import re, os
 from   bb.fetch2 import FetchMethod
-from   bb.fetch2 import FetchError
-from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
 
 
@@ -72,7 +58,7 @@ class SSH(FetchMethod):
     '''Class to fetch a module or modules via Secure Shell'''
 
     def supports(self, urldata, d):
-        return __pattern__.match(urldata.url) != None
+        return __pattern__.match(urldata.url) is not None
 
     def supports_checksum(self, urldata):
         return False

bitbake/lib/bb/fetch2/svn.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' implementation for svn.
 
@@ -8,24 +6,11 @@ BitBake 'Fetch' implementation for svn.
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2004        Marcin Juszkiewicz
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import os
-import sys
-import logging
 import bb
 import re
 from   bb.fetch2 import FetchMethod
@@ -49,7 +34,7 @@ class Svn(FetchMethod):
         if not "module" in ud.parm:
             raise MissingParameterError('module', ud.url)
 
-        ud.basecmd = d.getVar('FETCHCMD_svn')
+        ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive --trust-server-cert"
 
         ud.module = ud.parm["module"]
 
@@ -59,9 +44,13 @@ class Svn(FetchMethod):
             ud.path_spec = ud.parm["path_spec"]
 
         # Create paths to svn checkouts
+        svndir = d.getVar("SVNDIR") or (d.getVar("DL_DIR") + "/svn")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.expand('${SVNDIR}'), ud.host, relpath)
-        ud.moddir = os.path.join(ud.pkgdir, ud.module)
+        ud.pkgdir = os.path.join(svndir, ud.host, relpath)
+        ud.moddir = os.path.join(ud.pkgdir, ud.path_spec)
+        # Protects the repository from concurrent updates, e.g. from two
+        # recipes fetching different revisions at the same time
+        ud.svnlock = os.path.join(ud.pkgdir, "svn.lock")
 
         ud.setup_revisions(d)
 
@@ -100,6 +89,13 @@ class Svn(FetchMethod):
             svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         else:
             suffix = ""
+
+            # externals may be either 'allowed' or 'nowarn', but not both.  Allowed
+            # will not issue a warning, but will log to the debug buffer what has likely
+            # been downloaded by SVN.
+            if not ("externals" in ud.parm and ud.parm["externals"] == "allowed"):
+                options.append("--ignore-externals")
+
             if ud.revision:
                 options.append("-r %s" % ud.revision)
                 suffix = "@%s" % (ud.revision)
@@ -122,35 +118,52 @@ class Svn(FetchMethod):
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
-            svnupdatecmd = self._buildsvncommand(ud, d, "update")
-            logger.info("Update " + ud.url)
-            # We need to attempt to run svn upgrade first in case its an older working format
-            try:
-                runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
-            except FetchError:
-                pass
-            logger.debug(1, "Running %s", svnupdatecmd)
-            bb.fetch2.check_network_access(d, svnupdatecmd, ud.url)
-            runfetchcmd(svnupdatecmd, d, workdir=ud.moddir)
-        else:
-            svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
-            logger.info("Fetch " + ud.url)
-            # check out sources there
-            bb.utils.mkdirhier(ud.pkgdir)
-            logger.debug(1, "Running %s", svnfetchcmd)
-            bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
-            runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir)
+        lf = bb.utils.lockfile(ud.svnlock)
 
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            tar_flags = ""
-        else:
-            tar_flags = "--exclude='.svn'"
+        try:
+            if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
+                svncmd = self._buildsvncommand(ud, d, "update")
+                logger.info("Update " + ud.url)
+                # We need to attempt to run svn upgrade first in case its an older working format
+                try:
+                    runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
+                except FetchError:
+                    pass
+                logger.debug(1, "Running %s", svncmd)
+                bb.fetch2.check_network_access(d, svncmd, ud.url)
+                runfetchcmd(svncmd, d, workdir=ud.moddir)
+            else:
+                svncmd = self._buildsvncommand(ud, d, "fetch")
+                logger.info("Fetch " + ud.url)
+                # check out sources there
+                bb.utils.mkdirhier(ud.pkgdir)
+                logger.debug(1, "Running %s", svncmd)
+                bb.fetch2.check_network_access(d, svncmd, ud.url)
+                runfetchcmd(svncmd, d, workdir=ud.pkgdir)
 
-        # tar them up to a defined filename
-        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d,
-                    cleanup=[ud.localpath], workdir=ud.pkgdir)
+            if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"):
+                # Warn the user if this had externals (won't catch them all)
+                output = runfetchcmd("svn propget svn:externals || true", d, workdir=ud.moddir)
+                if output:
+                    if "--ignore-externals" in svncmd.split():
+                        bb.warn("%s contains svn:externals." % ud.url)
+                        bb.warn("These should be added to the recipe SRC_URI as necessary.")
+                        bb.warn("svn fetch has ignored externals:\n%s" % output)
+                        bb.warn("To disable this warning add ';externals=nowarn' to the url.")
+                    else:
+                        bb.debug(1, "svn repository has externals:\n%s" % output)
+
+            scmdata = ud.parm.get("scmdata", "")
+            if scmdata == "keep":
+                tar_flags = ""
+            else:
+                tar_flags = "--exclude='.svn'"
+
+            # tar them up to a defined filename
+            runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d,
+                        cleanup=[ud.localpath], workdir=ud.pkgdir)
+        finally:
+            bb.utils.unlockfile(lf)
 
     def clean(self, ud, d):
         """ Clean SVN specific files and dirs """

bitbake/lib/bb/fetch2/wget.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' implementations
 
@@ -10,29 +8,19 @@ BitBake build tools.
 
 # Copyright (C) 2003, 2004  Chris Larson
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
+import shlex
 import re
 import tempfile
-import subprocess
 import os
-import logging
 import errno
 import bb
 import bb.progress
+import socket
+import http.client
 import urllib.request, urllib.parse, urllib.error
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
@@ -104,9 +92,9 @@ class Wget(FetchMethod):
         fetchcmd = self.basecmd
 
         if 'downloadfilename' in ud.parm:
-            dldir = d.getVar("DL_DIR")
-            bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile))
-            fetchcmd += " -O " + dldir + os.sep + ud.localfile
+            localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
+            bb.utils.mkdirhier(os.path.dirname(localpath))
+            fetchcmd += " -O %s" % shlex.quote(localpath)
 
         if ud.user and ud.pswd:
             fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
@@ -132,10 +120,6 @@ class Wget(FetchMethod):
         return True
 
     def checkstatus(self, fetch, ud, d, try_again=True):
-        import urllib.request, urllib.error, urllib.parse, socket, http.client
-        from urllib.response import addinfourl
-        from bb.fetch2 import FetchConnectionCache
-
         class HTTPConnectionCache(http.client.HTTPConnection):
             if fetch.connection_cache:
                 def connect(self):
@@ -168,7 +152,7 @@ class Wget(FetchMethod):
                 """
                 host = req.host
                 if not host:
-                    raise urlllib2.URLError('no host given')
+                    raise urllib.error.URLError('no host given')
 
                 h = http_class(host, timeout=req.timeout) # will parse host:port
                 h.set_debuglevel(self._debuglevel)
@@ -185,7 +169,7 @@ class Wget(FetchMethod):
                 # request.
 
                 # Don't close connection when connection_cache is enabled,
-                if fetch.connection_cache is None: 
+                if fetch.connection_cache is None:
                     headers["Connection"] = "close"
                 else:
                     headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0
@@ -252,7 +236,7 @@ class Wget(FetchMethod):
                         pass
                     closed = False
 
-                resp = addinfourl(fp_dummy(), r.msg, req.get_full_url())
+                resp = urllib.response.addinfourl(fp_dummy(), r.msg, req.get_full_url())
                 resp.code = r.status
                 resp.msg = r.reason
 
@@ -271,17 +255,18 @@ class Wget(FetchMethod):
                 fp.read()
                 fp.close()
 
-                newheaders = dict((k,v) for k,v in list(req.headers.items())
-                                  if k.lower() not in ("content-length", "content-type"))
-                return self.parent.open(urllib.request.Request(req.get_full_url(),
-                                                        headers=newheaders,
-                                                        origin_req_host=req.origin_req_host,
-                                                        unverifiable=True))
+                if req.get_method() != 'GET':
+                    newheaders = dict((k, v) for k, v in list(req.headers.items())
+                                      if k.lower() not in ("content-length", "content-type"))
+                    return self.parent.open(urllib.request.Request(req.get_full_url(),
+                                                            headers=newheaders,
+                                                            origin_req_host=req.origin_req_host,
+                                                            unverifiable=True))
 
-            """
-            Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403
-            Forbidden when they actually mean 405 Method Not Allowed.
-            """
+                raise urllib.request.HTTPError(req, code, msg, headers, None)
+
+            # Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403
+            # Forbidden when they actually mean 405 Method Not Allowed.
             http_error_403 = http_error_405
 
 
@@ -292,15 +277,15 @@ class Wget(FetchMethod):
             """
             def redirect_request(self, req, fp, code, msg, headers, newurl):
                 newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
-                newreq.get_method = lambda: req.get_method()
+                newreq.get_method = req.get_method
                 return newreq
         exported_proxies = export_proxies(d)
 
         handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
-        if export_proxies:
+        if exported_proxies:
             handlers.append(urllib.request.ProxyHandler())
         handlers.append(CacheHTTPHandler())
-        # XXX: Since Python 2.7.9 ssl cert validation is enabled by default
+        # Since Python 2.7.9 ssl cert validation is enabled by default
         # see PEP-0476, this causes verification errors on some https servers
         # so disable by default.
         import ssl
@@ -315,23 +300,24 @@ class Wget(FetchMethod):
             # Some servers (FusionForge, as used on Alioth) require that the
             # optional Accept header is set.
             r.add_header("Accept", "*/*")
+            r.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12")
             def add_basic_auth(login_str, request):
                 '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64
                 encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
-                authheader =  "Basic %s" % encodeuser
+                authheader = "Basic %s" % encodeuser
                 r.add_header("Authorization", authheader)
 
-            if ud.user:
-                add_basic_auth(ud.user, r)
+            if ud.user and ud.pswd:
+                add_basic_auth(ud.user + ':' + ud.pswd, r)
 
             try:
-                import netrc, urllib.parse
+                import netrc
                 n = netrc.netrc()
                 login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
                 add_basic_auth("%s:%s" % (login, password), r)
             except (TypeError, ImportError, IOError, netrc.NetrcParseError):
-                 pass
+                pass
 
             with opener.open(r) as response:
                 pass
@@ -396,18 +382,14 @@ class Wget(FetchMethod):
         (oldpn, oldpv, oldsuffix) = old
         (newpn, newpv, newsuffix) = new
 
-        """
-        Check for a new suffix type that we have never heard of before
-        """
-        if (newsuffix):
+        # Check for a new suffix type that we have never heard of before
+        if newsuffix:
             m = self.suffix_regex_comp.search(newsuffix)
             if not m:
                 bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix))
                 return False
 
-        """
-        Not our package so ignore it
-        """
+        # Not our package so ignore it
         if oldpn != newpn:
             return False
 
@@ -473,15 +455,14 @@ class Wget(FetchMethod):
 
         return ""
 
-    def _check_latest_version_by_dir(self, dirver, package, package_regex,
-            current_version, ud, d):
+    def _check_latest_version_by_dir(self, dirver, package, package_regex, current_version, ud, d):
         """
-            Scan every directory in order to get upstream version.
+        Scan every directory in order to get upstream version.
         """
         version_dir = ['', '', '']
         version = ['', '', '']
 
-        dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
+        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
         s = dirver_regex.search(dirver)
         if s:
             version_dir[1] = s.group('ver')
@@ -541,26 +522,26 @@ class Wget(FetchMethod):
                 gst-fluendo-mp3
         """
         # match most patterns which uses "-" as separator to version digits
-        pn_prefix1 = "[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]"
+        pn_prefix1 = r"[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]"
         # a loose pattern such as for unzip552.tar.gz
-        pn_prefix2 = "[a-zA-Z]+"
+        pn_prefix2 = r"[a-zA-Z]+"
         # a loose pattern such as for 80325-quicky-0.4.tar.gz
-        pn_prefix3 = "[0-9]+[-]?[a-zA-Z]+"
+        pn_prefix3 = r"[0-9]+[-]?[a-zA-Z]+"
         # Save the Package Name (pn) Regex for use later
-        pn_regex = "(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3)
+        pn_regex = r"(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3)
 
         # match version
-        pver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)"
+        pver_regex = r"(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)"
 
         # match arch
         parch_regex = "-source|_all_"
 
         # src.rpm extension was added only for rpm package. Can be removed if the rpm
         # packaged will always be considered as having to be manually upgraded
-        psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
+        psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
 
         # match name, version and archive type of a package
-        package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
+        package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
                                                     % (pn_regex, pver_regex, parch_regex, psuffix_regex))
         self.suffix_regex_comp = re.compile(psuffix_regex)
 
@@ -572,7 +553,7 @@ class Wget(FetchMethod):
             version = self._parse_path(package_regex_comp, package)
             if version:
                 package_custom_regex_comp = re.compile(
-                    "(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" %
+                    r"(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" %
                     (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex))
             else:
                 package_custom_regex_comp = None
@@ -589,7 +570,7 @@ class Wget(FetchMethod):
         current_version = ['', d.getVar('PV'), '']
 
         """possible to have no version in pkg name, such as spectrum-fw"""
-        if not re.search("\d+", package):
+        if not re.search(r"\d+", package):
             current_version[1] = re.sub('_', '.', current_version[1])
             current_version[1] = re.sub('-', '.', current_version[1])
             return (current_version[1], '')
@@ -607,13 +588,13 @@ class Wget(FetchMethod):
 
             # search for version matches on folders inside the path, like:
             # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
-            dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
+            dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
             m = dirver_regex.search(path)
             if m:
                 pn = d.getVar('PN')
                 dirver = m.group('dirver')
 
-                dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn)))
+                dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
                 if not dirver_pn_regex.search(dirver):
                     return (self._check_latest_version_by_dir(dirver,
                         package, package_regex, current_version, ud, d), '')

bitbake/lib/bb/main.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -9,18 +6,8 @@
 # Copyright (C) 2005        ROAD GmbH
 # Copyright (C) 2006        Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
 import sys
@@ -267,6 +254,11 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="Do not run any setscene tasks. sstate will be ignored and "
                                "everything needed, built.")
 
+        parser.add_option("", "--skip-setscene", action="store_true",
+                          dest="skipsetscene", default=False,
+                          help="Skip setscene tasks if they would be executed. Tasks previously "
+                               "restored from sstate will be kept, unlike --no-setscene")
+
         parser.add_option("", "--setscene-only", action="store_true",
                           dest="setsceneonly", default=False,
                           help="Only run setscene tasks, don't run any real tasks.")
@@ -405,9 +397,6 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
         # In status only mode there are no logs and no UI
         logger.addHandler(handler)
 
-    # Clear away any spurious environment variables while we stoke up the cooker
-    cleanedvars = bb.utils.clean_environment()
-
     if configParams.server_only:
         featureset = []
         ui_module = None
@@ -423,6 +412,10 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
 
     server_connection = None
 
+    # Clear away any spurious environment variables while we stoke up the cooker
+    # (done after import_extension_module() above since for example import gi triggers env var usage)
+    cleanedvars = bb.utils.clean_environment()
+
     if configParams.remote_server:
         # Connect to a remote XMLRPC server
         server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset,
@@ -447,7 +440,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                 else:
                     logger.info("Reconnecting to bitbake server...")
                     if not os.path.exists(sockname):
-                        print("Previous bitbake instance shutting down?, waiting to retry...")
+                        logger.info("Previous bitbake instance shutting down?, waiting to retry...")
                         i = 0
                         lock = None
                         # Wait for 5s or until we can get the lock
@@ -459,12 +452,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                             bb.utils.unlockfile(lock)
                         raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?")
                 if not configParams.server_only:
-                    try:
-                        server_connection = bb.server.process.connectProcessServer(sockname, featureset)
-                    except EOFError:
-                        # The server may have been shutting down but not closed the socket yet. If that happened,
-                        # ignore it.
-                        pass
+                    server_connection = bb.server.process.connectProcessServer(sockname, featureset)
 
                 if server_connection or configParams.server_only:
                     break
@@ -474,12 +462,14 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                 if not retries:
                     raise
                 retries -= 1
-                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)):
-                    logger.info("Retrying server connection...")
+                tryno = 8 - retries
+                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError)):
+                    logger.info("Retrying server connection (#%d)..." % tryno)
                 else:
-                    logger.info("Retrying server connection... (%s)" % traceback.format_exc())
+                    logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
             if not retries:
-                bb.fatal("Unable to connect to bitbake server, or start one")
+                bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
+            bb.event.print_ui_queue()
             if retries < 5:
                 time.sleep(5)
 
@@ -501,7 +491,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
 def lockBitbake():
     topdir = bb.cookerdata.findTopdir()
     if not topdir:
-        bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBAPTH is unset and/or not in a build directory?")
+        bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBPATH is unset and/or not in a build directory?")
         raise BBMainFatal
     lockfile = topdir + "/bitbake.lock"
     return topdir, bb.utils.lockfile(lockfile, False, False)

bitbake/lib/bb/methodpool.py

@@ -1,21 +1,8 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
 #
 # Copyright (C)       2006 Holger Hans Peter Freyther
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from bb.utils import better_compile, better_exec
 

bitbake/lib/bb/monitordisk.py

@@ -1,23 +1,10 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2012 Robert Yang
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import os, logging, re, sys
+import os, logging, re
 import bb
 logger = logging.getLogger("BitBake.Monitor")
 
@@ -28,16 +15,16 @@ def convertGMK(unit):
 
     """ Convert the space unit G, M, K, the unit is case-insensitive """
 
-    unitG = re.match('([1-9][0-9]*)[gG]\s?$', unit)
+    unitG = re.match(r'([1-9][0-9]*)[gG]\s?$', unit)
     if unitG:
         return int(unitG.group(1)) * (1024 ** 3)
-    unitM = re.match('([1-9][0-9]*)[mM]\s?$', unit)
+    unitM = re.match(r'([1-9][0-9]*)[mM]\s?$', unit)
     if unitM:
         return int(unitM.group(1)) * (1024 ** 2)
-    unitK = re.match('([1-9][0-9]*)[kK]\s?$', unit)
+    unitK = re.match(r'([1-9][0-9]*)[kK]\s?$', unit)
     if unitK:
         return int(unitK.group(1)) * 1024
-    unitN = re.match('([1-9][0-9]*)\s?$', unit)
+    unitN = re.match(r'([1-9][0-9]*)\s?$', unit)
     if unitN:
         return int(unitN.group(1))
     else:
@@ -83,7 +70,7 @@ def getDiskData(BBDirs, configuration):
     for pathSpaceInode in BBDirs.split():
         # The input format is: "dir,space,inode", dir is a must, space
         # and inode are optional
-        pathSpaceInodeRe = re.match('([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
+        pathSpaceInodeRe = re.match(r'([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
         if not pathSpaceInodeRe:
             printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
             return None
@@ -147,7 +134,7 @@ def getInterval(configuration):
     else:
         # The disk space or inode interval is optional, but it should
         # have a correct value once it is specified
-        intervalRe = re.match('([^,]*),?\s*(.*)', interval)
+        intervalRe = re.match(r'([^,]*),?\s*(.*)', interval)
         if intervalRe:
             intervalSpace = intervalRe.group(1)
             if intervalSpace:

bitbake/lib/bb/msg.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'msg' implementation
 
@@ -9,25 +7,14 @@ Message handling infrastructure for bitbake
 
 # Copyright (C) 2006        Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import sys
 import copy
 import logging
-import collections
+import logging.config
 from itertools import groupby
-import warnings
 import bb
 import bb.event
 
@@ -40,6 +27,7 @@ class BBLogFormatter(logging.Formatter):
     VERBOSE = logging.INFO - 1
     NOTE = logging.INFO
     PLAIN = logging.INFO + 1
+    VERBNOTE = logging.INFO + 2
     ERROR = logging.ERROR
     WARNING = logging.WARNING
     CRITICAL = logging.CRITICAL
@@ -51,6 +39,7 @@ class BBLogFormatter(logging.Formatter):
         VERBOSE: 'NOTE',
         NOTE    : 'NOTE',
         PLAIN  : '',
+        VERBNOTE: 'NOTE',
         WARNING : 'WARNING',
         ERROR   : 'ERROR',
         CRITICAL: 'ERROR',
@@ -66,6 +55,7 @@ class BBLogFormatter(logging.Formatter):
         VERBOSE : BASECOLOR,
         NOTE    : BASECOLOR,
         PLAIN   : BASECOLOR,
+        VERBNOTE: BASECOLOR,
         WARNING : YELLOW,
         ERROR   : RED,
         CRITICAL: RED,
@@ -109,6 +99,9 @@ class BBLogFormatter(logging.Formatter):
     def enable_color(self):
         self.color_enabled = True
 
+    def __repr__(self):
+        return "%s fmt='%s' color=%s" % (self.__class__.__name__, self._fmt, "True" if self.color_enabled else "False")
+
 class BBLogFilter(object):
     def __init__(self, handler, level, debug_domains):
         self.stdlevel = level
@@ -127,60 +120,59 @@ class BBLogFilter(object):
             return True
         return False
 
-class BBLogFilterStdErr(BBLogFilter):
-    def filter(self, record):
-        if not BBLogFilter.filter(self, record):
-            return False
-        if record.levelno >= logging.ERROR:
-            return True
-        return False
+class LogFilterGEQLevel(logging.Filter):
+    def __init__(self, level):
+        self.strlevel = str(level)
+        self.level = stringToLevel(level)
+
+    def __repr__(self):
+        return "%s level >= %s (%d)" % (self.__class__.__name__, self.strlevel, self.level)
 
-class BBLogFilterStdOut(BBLogFilter):
     def filter(self, record):
-        if not BBLogFilter.filter(self, record):
-            return False
-        if record.levelno < logging.ERROR:
-            return True
-        return False
+        return (record.levelno >= self.level)
+
+class LogFilterLTLevel(logging.Filter):
+    def __init__(self, level):
+        self.strlevel = str(level)
+        self.level = stringToLevel(level)
+
+    def __repr__(self):
+        return "%s level < %s (%d)" % (self.__class__.__name__, self.strlevel, self.level)
+
+    def filter(self, record):
+        return (record.levelno < self.level)
 
 # Message control functions
 #
 
-loggerDefaultDebugLevel = 0
+loggerDefaultLogLevel = BBLogFormatter.NOTE
 loggerDefaultVerbose = False
 loggerVerboseLogs = False
-loggerDefaultDomains = []
+loggerDefaultDomains = {}
 
 def init_msgconfig(verbose, debug, debug_domains=None):
     """
     Set default verbosity and debug levels config the logger
     """
-    bb.msg.loggerDefaultDebugLevel = debug
     bb.msg.loggerDefaultVerbose = verbose
     if verbose:
         bb.msg.loggerVerboseLogs = True
-    if debug_domains:
-        bb.msg.loggerDefaultDomains = debug_domains
-    else:
-        bb.msg.loggerDefaultDomains = []
-
-def constructLogOptions():
-    debug = loggerDefaultDebugLevel
-    verbose = loggerDefaultVerbose
-    domains = loggerDefaultDomains
 
     if debug:
-        level = BBLogFormatter.DEBUG - debug + 1
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.DEBUG - debug + 1
     elif verbose:
-        level = BBLogFormatter.VERBOSE
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.VERBOSE
     else:
-        level = BBLogFormatter.NOTE
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.NOTE
 
-    debug_domains = {}
-    for (domainarg, iterator) in groupby(domains):
-        dlevel = len(tuple(iterator))
-        debug_domains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
-    return level, debug_domains
+    bb.msg.loggerDefaultDomains = {}
+    if debug_domains:
+        for (domainarg, iterator) in groupby(debug_domains):
+            dlevel = len(tuple(iterator))
+            bb.msg.loggerDefaultDomains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
+
+def constructLogOptions():
+    return loggerDefaultLogLevel, loggerDefaultDomains
 
 def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
     level, debug_domains = constructLogOptions()
@@ -190,6 +182,19 @@ def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
 
     cls(handler, level, debug_domains)
 
+def stringToLevel(level):
+    try:
+        return int(level)
+    except ValueError:
+        pass
+
+    try:
+        return getattr(logging, level)
+    except AttributeError:
+        pass
+
+    return getattr(BBLogFormatter, level)
+
 #
 # Message handling functions
 #
@@ -223,3 +228,105 @@ def has_console_handler(logger):
             if handler.stream in [sys.stderr, sys.stdout]:
                 return True
     return False
+
+def mergeLoggingConfig(logconfig, userconfig):
+    logconfig = copy.deepcopy(logconfig)
+    userconfig = copy.deepcopy(userconfig)
+
+    # Merge config with the default config
+    if userconfig.get('version') != logconfig['version']:
+        raise BaseException("Bad user configuration version. Expected %r, got %r" % (logconfig['version'], userconfig.get('version')))
+
+    # Set some defaults to make merging easier
+    userconfig.setdefault("loggers", {})
+
+    # If a handler, formatter, or filter is defined in the user
+    # config, it will replace an existing one in the default config
+    for k in ("handlers", "formatters", "filters"):
+        logconfig.setdefault(k, {}).update(userconfig.get(k, {}))
+
+    seen_loggers = set()
+    for name, l in logconfig["loggers"].items():
+        # If the merge option is set, merge the handlers and
+        # filters. Otherwise, if it is False, this logger won't get
+        # add to the set of seen loggers and will replace the
+        # existing one
+        if l.get('bitbake_merge', True):
+            ulogger = userconfig["loggers"].setdefault(name, {})
+            ulogger.setdefault("handlers", [])
+            ulogger.setdefault("filters", [])
+
+            # Merge lists
+            l.setdefault("handlers", []).extend(ulogger["handlers"])
+            l.setdefault("filters", []).extend(ulogger["filters"])
+
+            # Replace other properties if present
+            if "level" in ulogger:
+                l["level"] = ulogger["level"]
+
+            if "propagate" in ulogger:
+                l["propagate"] = ulogger["propagate"]
+
+            seen_loggers.add(name)
+
+    # Add all loggers present in the user config, but not any that
+    # have already been processed
+    for name in set(userconfig["loggers"].keys()) - seen_loggers:
+        logconfig["loggers"][name] = userconfig["loggers"][name]
+
+    return logconfig
+
+def setLoggingConfig(defaultconfig, userconfigfile=None):
+    logconfig = copy.deepcopy(defaultconfig)
+
+    if userconfigfile:
+        with open(os.path.normpath(userconfigfile), 'r') as f:
+            if userconfigfile.endswith('.yml') or userconfigfile.endswith('.yaml'):
+                import yaml
+                userconfig = yaml.load(f)
+            elif userconfigfile.endswith('.json') or userconfigfile.endswith('.cfg'):
+                import json
+                userconfig = json.load(f)
+            else:
+                raise BaseException("Unrecognized file format: %s" % userconfigfile)
+
+            if userconfig.get('bitbake_merge', True):
+                logconfig = mergeLoggingConfig(logconfig, userconfig)
+            else:
+                # Replace the entire default config
+                logconfig = userconfig
+
+    # Convert all level parameters to integers in case users want to use the
+    # bitbake defined level names
+    for h in logconfig["handlers"].values():
+        if "level" in h:
+            h["level"] = bb.msg.stringToLevel(h["level"])
+
+    for l in logconfig["loggers"].values():
+        if "level" in l:
+            l["level"] = bb.msg.stringToLevel(l["level"])
+
+    conf = logging.config.dictConfigClass(logconfig)
+    conf.configure()
+
+    # The user may have specified logging domains they want at a higher debug
+    # level than the standard.
+    for name, l in logconfig["loggers"].items():
+        if not name.startswith("BitBake."):
+            continue
+
+        if not "level" in l:
+            continue
+
+        curlevel = bb.msg.loggerDefaultDomains.get(name)
+        # Note: level parameter should already be a int because of conversion
+        # above
+        newlevel = int(l["level"])
+        if curlevel is None or newlevel < curlevel:
+            bb.msg.loggerDefaultDomains[name] = newlevel
+
+        # TODO: I don't think that setting the global log level should be necessary
+        #if newlevel < bb.msg.loggerDefaultLogLevel:
+        #    bb.msg.loggerDefaultLogLevel = newlevel
+
+    return conf

bitbake/lib/bb/namedtuple_with_abc.py

@@ -1,6 +1,8 @@
 # http://code.activestate.com/recipes/577629-namedtupleabc-abstract-base-class-mix-in-for-named/
-#!/usr/bin/env python
 # Copyright (c) 2011 Jan Kaliszewski (zuo). Available under the MIT License.
+#
+# SPDX-License-Identifier: MIT
+#
 
 """
 namedtuple_with_abc.py:

bitbake/lib/bb/parse/__init__.py

@@ -9,20 +9,10 @@ File parsers for the BitBake build tools.
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
+#
 
 handlers = []
 

bitbake/lib/bb/parse/ast.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
  AbstractSyntaxTree classes for the Bitbake language
 """
@@ -8,25 +6,10 @@
 # Copyright (C) 2003, 2004 Phil Blundell
 # Copyright (C) 2009 Holger Hans Peter Freyther
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-
-import re
-import string
-import logging
 import bb
-import itertools
 from bb import methodpool
 from bb.parse import logger
 
@@ -106,7 +89,7 @@ class DataNode(AstNode):
         self.groupd = groupd
 
     def getFunc(self, key, data):
-        if 'flag' in self.groupd and self.groupd['flag'] != None:
+        if 'flag' in self.groupd and self.groupd['flag'] is not None:
             return data.getVarFlag(key, self.groupd['flag'], expand=False, noweakdefault=True)
         else:
             return data.getVar(key, False, noweakdefault=True, parsing=True)
@@ -119,36 +102,36 @@ class DataNode(AstNode):
             'file': self.filename,
             'line': self.lineno,
         }
-        if "exp" in groupd and groupd["exp"] != None:
+        if "exp" in groupd and groupd["exp"] is not None:
             data.setVarFlag(key, "export", 1, op = 'exported', **loginfo)
 
         op = "set"
-        if "ques" in groupd and groupd["ques"] != None:
+        if "ques" in groupd and groupd["ques"] is not None:
             val = self.getFunc(key, data)
             op = "set?"
-            if val == None:
+            if val is None:
                 val = groupd["value"]
-        elif "colon" in groupd and groupd["colon"] != None:
+        elif "colon" in groupd and groupd["colon"] is not None:
             e = data.createCopy()
             op = "immediate"
             val = e.expand(groupd["value"], key + "[:=]")
-        elif "append" in groupd and groupd["append"] != None:
+        elif "append" in groupd and groupd["append"] is not None:
             op = "append"
             val = "%s %s" % ((self.getFunc(key, data) or ""), groupd["value"])
-        elif "prepend" in groupd and groupd["prepend"] != None:
+        elif "prepend" in groupd and groupd["prepend"] is not None:
             op = "prepend"
             val = "%s %s" % (groupd["value"], (self.getFunc(key, data) or ""))
-        elif "postdot" in groupd and groupd["postdot"] != None:
+        elif "postdot" in groupd and groupd["postdot"] is not None:
             op = "postdot"
             val = "%s%s" % ((self.getFunc(key, data) or ""), groupd["value"])
-        elif "predot" in groupd and groupd["predot"] != None:
+        elif "predot" in groupd and groupd["predot"] is not None:
             op = "predot"
             val = "%s%s" % (groupd["value"], (self.getFunc(key, data) or ""))
         else:
             val = groupd["value"]
 
         flag = None
-        if 'flag' in groupd and groupd['flag'] != None:
+        if 'flag' in groupd and groupd['flag'] is not None:
             flag = groupd['flag']
         elif groupd["lazyques"]:
             flag = "_defaultval"
@@ -178,7 +161,7 @@ class MethodNode(AstNode):
             funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl)))
             self.python = True
             text = "def %s(d):\n" % (funcname) + text
-            bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body))
+            bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body) - 1)
             anonfuncs = data.getVar('__BBANONFUNCS', False) or []
             anonfuncs.append(funcname)
             data.setVar('__BBANONFUNCS', anonfuncs)
@@ -343,30 +326,31 @@ def runAnonFuncs(d):
 
 def finalize(fn, d, variant = None):
     saved_handlers = bb.event.get_handlers().copy()
+    try:
+        for var in d.getVar('__BBHANDLERS', False) or []:
+            # try to add the handler
+            handlerfn = d.getVarFlag(var, "filename", False)
+            if not handlerfn:
+                bb.fatal("Undefined event handler function '%s'" % var)
+            handlerln = int(d.getVarFlag(var, "lineno", False))
+            bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
 
-    for var in d.getVar('__BBHANDLERS', False) or []:
-        # try to add the handler
-        handlerfn = d.getVarFlag(var, "filename", False)
-        if not handlerfn:
-            bb.fatal("Undefined event handler function '%s'" % var)
-        handlerln = int(d.getVarFlag(var, "lineno", False))
-        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
+        bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
-    bb.event.fire(bb.event.RecipePreFinalise(fn), d)
+        bb.data.expandKeys(d)
+        runAnonFuncs(d)
 
-    bb.data.expandKeys(d)
-    runAnonFuncs(d)
+        tasklist = d.getVar('__BBTASKS', False) or []
+        bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
+        bb.build.add_tasks(tasklist, d)
 
-    tasklist = d.getVar('__BBTASKS', False) or []
-    bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
-    bb.build.add_tasks(tasklist, d)
+        bb.parse.siggen.finalise(fn, d, variant)
 
-    bb.parse.siggen.finalise(fn, d, variant)
+        d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
 
-    d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
-
-    bb.event.fire(bb.event.RecipeParsed(fn), d)
-    bb.event.set_handlers(saved_handlers)
+        bb.event.fire(bb.event.RecipeParsed(fn), d)
+    finally:
+        bb.event.set_handlers(saved_handlers)
 
 def _create_variants(datastores, names, function, onlyfinalise):
     def create_variant(name, orig_d, arg = None):

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
    class for handling .bb files
 
@@ -12,24 +9,11 @@
 #  Copyright (C) 2003, 2004  Chris Larson
 #  Copyright (C) 2003, 2004  Phil Blundell
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
 
 import re, bb, os
-import logging
 import bb.build, bb.utils
-from bb import data
 
 from . import ConfHandler
 from .. import resolve_file, ast, logger, ParseError
@@ -38,14 +22,15 @@ from .ConfHandler import include, init
 # For compatibility
 bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"])
 
-__func_start_regexp__    = re.compile( r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" )
-__inherit_regexp__       = re.compile( r"inherit\s+(.+)" )
-__export_func_regexp__   = re.compile( r"EXPORT_FUNCTIONS\s+(.+)" )
-__addtask_regexp__       = re.compile("addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
-__deltask_regexp__       = re.compile("deltask\s+(?P<func>\w+)")
-__addhandler_regexp__    = re.compile( r"addhandler\s+(.+)" )
-__def_regexp__           = re.compile( r"def\s+(\w+).*:" )
-__python_func_regexp__   = re.compile( r"(\s+.*)|(^$)" )
+__func_start_regexp__    = re.compile(r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" )
+__inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
+__export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
+__addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
+__deltask_regexp__       = re.compile(r"deltask\s+(?P<func>\w+)(?P<ignores>.*)")
+__addhandler_regexp__    = re.compile(r"addhandler\s+(.+)" )
+__def_regexp__           = re.compile(r"def\s+(\w+).*:" )
+__python_func_regexp__   = re.compile(r"(\s+.*)|(^$)|(^#)" )
+__python_tab_regexp__    = re.compile(r" *\t")
 
 __infunc__ = []
 __inpython__ = False
@@ -160,6 +145,16 @@ def handle(fn, d, include):
 
 def feeder(lineno, s, fn, root, statements, eof=False):
     global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, bb, __residue__, __classname__
+
+    # Check tabs in python functions:
+    # - def py_funcname(): covered by __inpython__
+    # - python(): covered by '__anonymous' == __infunc__[0]
+    # - python funcname(): covered by __infunc__[3]
+    if __inpython__ or (__infunc__ and ('__anonymous' == __infunc__[0] or __infunc__[3])):
+        tab = __python_tab_regexp__.match(s)
+        if tab:
+            bb.warn('python should use 4 spaces indentation, but found tabs in %s, line %s' % (root, lineno))
+
     if __infunc__:
         if s == '}':
             __body__.append('')
@@ -225,11 +220,27 @@ def feeder(lineno, s, fn, root, statements, eof=False):
 
     m = __addtask_regexp__.match(s)
     if m:
+        if len(m.group().split()) == 2:
+            # Check and warn for "addtask task1 task2"
+            m2 = re.match(r"addtask\s+(?P<func>\w+)(?P<ignores>.*)", s)
+            if m2 and m2.group('ignores'):
+                logger.warning('addtask ignored: "%s"' % m2.group('ignores'))
+
+        # Check and warn for "addtask task1 before task2 before task3", the
+        # similar to "after"
+        taskexpression = s.split()
+        for word in ('before', 'after'):
+            if taskexpression.count(word) > 1:
+                logger.warning("addtask contained multiple '%s' keywords, only one is supported" % word)
+
         ast.handleAddTask(statements, fn, lineno, m)
         return
 
     m = __deltask_regexp__.match(s)
     if m:
+        # Check and warn "for deltask task1 task2"
+        if m.group('ignores'):
+            logger.warning('deltask ignored: "%s"' % m.group('ignores'))
         ast.handleDelTask(statements, fn, lineno, m)
         return
 

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
    class for handling configuration data files
 
@@ -11,18 +8,8 @@
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import errno
 import re
@@ -132,30 +119,30 @@ def handle(fn, data, include):
         oldfile = data.getVar('FILE', False)
 
     abs_fn = resolve_file(fn, data)
-    f = open(abs_fn, 'r')
+    with open(abs_fn, 'r') as f:
 
-    statements = ast.StatementGroup()
-    lineno = 0
-    while True:
-        lineno = lineno + 1
-        s = f.readline()
-        if not s:
-            break
-        w = s.strip()
-        # skip empty lines
-        if not w:
-            continue
-        s = s.rstrip()
-        while s[-1] == '\\':
-            s2 = f.readline().strip()
+        statements = ast.StatementGroup()
+        lineno = 0
+        while True:
             lineno = lineno + 1
-            if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
-                bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
-            s = s[:-1] + s2
-        # skip comments
-        if s[0] == '#':
-            continue
-        feeder(lineno, s, abs_fn, statements)
+            s = f.readline()
+            if not s:
+                break
+            w = s.strip()
+            # skip empty lines
+            if not w:
+                continue
+            s = s.rstrip()
+            while s[-1] == '\\':
+                s2 = f.readline().rstrip()
+                lineno = lineno + 1
+                if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
+                    bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+                s = s[:-1] + s2
+            # skip comments
+            if s[0] == '#':
+                continue
+            feeder(lineno, s, abs_fn, statements)
 
     # DONE WITH PARSING... time to evaluate
     data.setVar('FILE', abs_fn)

bitbake/lib/bb/parse/parse_py/__init__.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake Parsers
 
@@ -11,20 +8,10 @@ File parsers for the BitBake build tools.
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# SPDX-License-Identifier: GPL-2.0-only
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
+#
 
 from __future__ import absolute_import
 from . import ConfHandler

bitbake/lib/bb/persist_data.py

@@ -8,18 +8,8 @@ currently, providing a key/value store accessed by 'domain'.
 # Copyright (C) 2007        Richard Purdie
 # Copyright (C) 2010        Chris Larson <chris_larson@mentor.com>
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import collections
 import logging
@@ -29,6 +19,7 @@ import warnings
 from bb.compat import total_ordering
 from collections import Mapping
 import sqlite3
+import contextlib
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -36,84 +27,184 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 
 
 logger = logging.getLogger("BitBake.PersistData")
-if hasattr(sqlite3, 'enable_shared_cache'):
-    try:
-        sqlite3.enable_shared_cache(True)
-    except sqlite3.OperationalError:
-        pass
-
 
 @total_ordering
 class SQLTable(collections.MutableMapping):
+    class _Decorators(object):
+        @staticmethod
+        def retry(*, reconnect=True):
+            """
+            Decorator that restarts a function if a database locked sqlite
+            exception occurs. If reconnect is True, the database connection
+            will be closed and reopened each time a failure occurs
+            """
+            def retry_wrapper(f):
+                def wrap_func(self, *args, **kwargs):
+                    # Reconnect if necessary
+                    if self.connection is None and reconnect:
+                        self.reconnect()
+
+                    count = 0
+                    while True:
+                        try:
+                            return f(self, *args, **kwargs)
+                        except sqlite3.OperationalError as exc:
+                            if count < 500 and ('is locked' in str(exc) or 'locking protocol' in str(exc)):
+                                count = count + 1
+                                if reconnect:
+                                    self.reconnect()
+                                continue
+                            raise
+                return wrap_func
+            return retry_wrapper
+
+        @staticmethod
+        def transaction(f):
+            """
+            Decorator that starts a database transaction and creates a database
+            cursor for performing queries. If no exception is thrown, the
+            database results are commited. If an exception occurs, the database
+            is rolled back. In all cases, the cursor is closed after the
+            function ends.
+
+            Note that the cursor is passed as an extra argument to the function
+            after `self` and before any of the normal arguments
+            """
+            def wrap_func(self, *args, **kwargs):
+                # Context manager will COMMIT the database on success,
+                # or ROLLBACK on an exception
+                with self.connection:
+                    # Automatically close the cursor when done
+                    with contextlib.closing(self.connection.cursor()) as cursor:
+                        return f(self, cursor, *args, **kwargs)
+            return wrap_func
+
     """Object representing a table/domain in the database"""
     def __init__(self, cachefile, table):
         self.cachefile = cachefile
         self.table = table
-        self.cursor = connect(self.cachefile)
 
-        self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);"
-                      % table)
+        self.connection = None
+        self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table)
 
-    def _execute(self, *query):
-        """Execute a query, waiting to acquire a lock if necessary"""
-        count = 0
-        while True:
-            try:
-                return self.cursor.execute(*query)
-            except sqlite3.OperationalError as exc:
-                if 'database is locked' in str(exc) and count < 500:
-                    count = count + 1
+    @_Decorators.retry(reconnect=False)
+    @_Decorators.transaction
+    def _setup_database(self, cursor):
+        cursor.execute("pragma synchronous = off;")
+        # Enable WAL and keep the autocheckpoint length small (the default is
+        # usually 1000). Persistent caches are usually read-mostly, so keeping
+        # this short will keep readers running quickly
+        cursor.execute("pragma journal_mode = WAL;")
+        cursor.execute("pragma wal_autocheckpoint = 100;")
+
+    def reconnect(self):
+        if self.connection is not None:
+            self.connection.close()
+        self.connection = sqlite3.connect(self.cachefile, timeout=5)
+        self.connection.text_factory = str
+        self._setup_database()
+
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def _execute_single(self, cursor, *query):
+        """
+        Executes a single query and discards the results. This correctly closes
+        the database cursor when finished
+        """
+        cursor.execute(*query)
+
+    @_Decorators.retry()
+    def _row_iter(self, f, *query):
+        """
+        Helper function that returns a row iterator. Each time __next__ is
+        called on the iterator, the provided function is evaluated to determine
+        the return value
+        """
+        class CursorIter(object):
+            def __init__(self, cursor):
+                self.cursor = cursor
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                row = self.cursor.fetchone()
+                if row is None:
                     self.cursor.close()
-                    self.cursor = connect(self.cachefile)
-                    continue
-                raise
+                    raise StopIteration
+                return f(row)
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, typ, value, traceback):
+                self.cursor.close()
+                return False
+
+        cursor = self.connection.cursor()
+        try:
+            cursor.execute(*query)
+            return CursorIter(cursor)
+        except:
+            cursor.close()
 
     def __enter__(self):
-        self.cursor.__enter__()
+        self.connection.__enter__()
         return self
 
     def __exit__(self, *excinfo):
-        self.cursor.__exit__(*excinfo)
+        self.connection.__exit__(*excinfo)
 
-    def __getitem__(self, key):
-        data = self._execute("SELECT * from %s where key=?;" %
-                             self.table, [key])
-        for row in data:
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __getitem__(self, cursor, key):
+        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
+        row = cursor.fetchone()
+        if row is not None:
             return row[1]
         raise KeyError(key)
 
-    def __delitem__(self, key):
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __delitem__(self, cursor, key):
         if key not in self:
             raise KeyError(key)
-        self._execute("DELETE from %s where key=?;" % self.table, [key])
+        cursor.execute("DELETE from %s where key=?;" % self.table, [key])
 
-    def __setitem__(self, key, value):
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __setitem__(self, cursor, key, value):
         if not isinstance(key, str):
             raise TypeError('Only string keys are supported')
         elif not isinstance(value, str):
             raise TypeError('Only string values are supported')
 
-        data = self._execute("SELECT * from %s where key=?;" %
-                                   self.table, [key])
-        exists = len(list(data))
-        if exists:
-            self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table,
-                          [value, key])
+        # Ensure the entire transaction (including SELECT) executes under write lock
+        cursor.execute("BEGIN EXCLUSIVE")
+
+        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
+        row = cursor.fetchone()
+        if row is not None:
+            cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key])
         else:
-            self._execute("INSERT into %s(key, value) values (?, ?);" %
-                          self.table, [key, value])
+            cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value])
 
-    def __contains__(self, key):
-        return key in set(self)
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __contains__(self, cursor, key):
+        cursor.execute('SELECT * from %s where key=?;' % self.table, [key])
+        return cursor.fetchone() is not None
 
-    def __len__(self):
-        data = self._execute("SELECT COUNT(key) FROM %s;" % self.table)
-        for row in data:
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __len__(self, cursor):
+        cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
+        row = cursor.fetchone()
+        if row is not None:
             return row[0]
 
     def __iter__(self):
-        data = self._execute("SELECT key FROM %s;" % self.table)
-        return (row[0] for row in data)
+        return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table)
 
     def __lt__(self, other):
         if not isinstance(other, Mapping):
@@ -122,25 +213,27 @@ class SQLTable(collections.MutableMapping):
         return len(self) < len(other)
 
     def get_by_pattern(self, pattern):
-        data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" %
-                             self.table, [pattern])
-        return [row[1] for row in data]
+        return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" %
+                              self.table, [pattern])
 
     def values(self):
         return list(self.itervalues())
 
     def itervalues(self):
-        data = self._execute("SELECT value FROM %s;" % self.table)
-        return (row[0] for row in data)
+        return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" %
+                              self.table)
 
     def items(self):
         return list(self.iteritems())
 
     def iteritems(self):
-        return self._execute("SELECT * FROM %s;" % self.table)
+        return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" %
+                              self.table)
 
-    def clear(self):
-        self._execute("DELETE FROM %s;" % self.table)
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def clear(self, cursor):
+        cursor.execute("DELETE FROM %s;" % self.table)
 
     def has_key(self, key):
         return key in self
@@ -194,12 +287,6 @@ class PersistData(object):
         """
         del self.data[domain][key]
 
-def connect(database):
-    connection = sqlite3.connect(database, timeout=5, isolation_level=None)
-    connection.execute("pragma synchronous = off;")
-    connection.text_factory = str
-    return connection
-
 def persist(domain, d):
     """Convenience factory for SQLTable objects based upon metadata"""
     import bb.utils

bitbake/lib/bb/process.py

@@ -1,3 +1,7 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
 import logging
 import signal
 import subprocess

bitbake/lib/bb/progress.py

@@ -4,25 +4,15 @@ BitBake progress handling code
 
 # Copyright (C) 2016 Intel Corporation
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import sys
 import re
 import time
 import inspect
 import bb.event
 import bb.build
+from bb.build import StdoutNoopContextManager
 
 class ProgressHandler(object):
     """
@@ -37,7 +27,14 @@ class ProgressHandler(object):
         if outfile:
             self._outfile = outfile
         else:
-            self._outfile = sys.stdout
+            self._outfile = StdoutNoopContextManager()
+
+    def __enter__(self):
+        self._outfile.__enter__()
+        return self
+
+    def __exit__(self, *excinfo):
+        self._outfile.__exit__(*excinfo)
 
     def _fire_progress(self, taskprogress, rate=None):
         """Internal function to fire the progress event"""
@@ -157,6 +154,12 @@ class MultiStageProgressReporter(object):
             self._stage_total = None
             self._callers = []
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *excinfo):
+        pass
+
     def _fire_progress(self, taskprogress):
         bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
 

bitbake/lib/bb/providers.py

@@ -1,5 +1,3 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -8,18 +6,8 @@
 # Copyright (C) 2005        ROAD GmbH
 # Copyright (C) 2006        Richard Purdie
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import re
 import logging
@@ -104,11 +92,11 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
     Check if the version pe,pv,pr is the preferred one.
     If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%'
     """
-    if (pr == preferred_r or preferred_r == None):
-        if (pe == preferred_e or preferred_e == None):
+    if pr == preferred_r or preferred_r is None:
+        if pe == preferred_e or preferred_e is None:
             if preferred_v == pv:
                 return True
-            if preferred_v != None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]):
+            if preferred_v is not None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]):
                 return True
     return False
 
@@ -129,7 +117,7 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
         preferred_v = cfgData.getVar("PREFERRED_VERSION")
 
     if preferred_v:
-        m = re.match('(\d+:)*(.*)(_.*)*', preferred_v)
+        m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
         if m:
             if m.group(1):
                 preferred_e = m.group(1)[:-1]
@@ -384,7 +372,7 @@ def getRuntimeProviders(dataCache, rdepend):
 
     # Only search dynamic packages if we can't find anything in other variables
     for pattern in dataCache.packages_dynamic:
-        pattern = pattern.replace('+', "\+")
+        pattern = pattern.replace(r'+', r"\+")
         if pattern in regexp_cache:
             regexp = regexp_cache[pattern]
         else:

bitbake/lib/bb/pysh/builtin.py

@@ -1,710 +0,0 @@
-# builtin.py - builtins and utilities definitions for pysh.
-#
-# Copyright 2007 Patrick Mezard
-#
-# This software may be used and distributed according to the terms
-# of the GNU General Public License, incorporated herein by reference.
-
-"""Builtin and internal utilities implementations.
-
-- Beware not to use python interpreter environment as if it were the shell
-environment. For instance, commands working directory must be explicitely handled
-through env['PWD'] instead of relying on python working directory.
-"""
-import errno
-import optparse
-import os
-import re
-import subprocess
-import sys
-import time
-
-def has_subprocess_bug():
-    return getattr(subprocess, 'list2cmdline') and \
-       (    subprocess.list2cmdline(['']) == '' or \
-            subprocess.list2cmdline(['foo|bar']) == 'foo|bar')
-            
-# Detect python bug 1634343: "subprocess swallows empty arguments under win32"
-# <http://sourceforge.net/tracker/index.php?func=detail&aid=1634343&group_id=5470&atid=105470>
-# Also detect: "[ 1710802 ] subprocess must escape redirection characters under win32"
-# <http://sourceforge.net/tracker/index.php?func=detail&aid=1710802&group_id=5470&atid=105470>
-if has_subprocess_bug():
-    import subprocess_fix
-    subprocess.list2cmdline = subprocess_fix.list2cmdline
-
-from sherrors import *
-
-class NonExitingParser(optparse.OptionParser):
-    """OptionParser default behaviour upon error is to print the error message and
-    exit. Raise a utility error instead.
-    """
-    def error(self, msg):
-        raise UtilityError(msg)
-
-#-------------------------------------------------------------------------------  
-# set special builtin
-#-------------------------------------------------------------------------------  
-OPT_SET = NonExitingParser(usage="set - set or unset options and positional parameters")
-OPT_SET.add_option( '-f', action='store_true', dest='has_f', default=False,
-    help='The shell shall disable pathname expansion.')
-OPT_SET.add_option('-e', action='store_true', dest='has_e', default=False,
-    help="""When this option is on, if a simple command fails for any of the \
-    reasons listed in Consequences of Shell Errors or returns an exit status \
-    value >0, and is not part of the compound list following a while, until, \
-    or if keyword, and is not a part of an AND or OR list, and is not a \
-    pipeline preceded by the ! reserved word, then the shell shall immediately \
-    exit.""")
-OPT_SET.add_option('-x', action='store_true', dest='has_x', default=False,
-    help="""The shell shall write to standard error a trace for each command \
-    after it expands the command and before it executes it. It is unspecified \
-    whether the command that turns tracing off is traced.""")
-
-def builtin_set(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    option, args = OPT_SET.parse_args(args)
-    env = interp.get_env()
-    
-    if option.has_f:
-        env.set_opt('-f')
-    if option.has_e:
-        env.set_opt('-e')
-    if option.has_x:
-        env.set_opt('-x')
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# shift special builtin
-#-------------------------------------------------------------------------------  
-def builtin_shift(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    params = interp.get_env().get_positional_args()
-    if args:
-        try:
-            n = int(args[0])
-            if n > len(params):
-                raise ValueError()
-        except ValueError:
-            return 1
-    else:
-        n = 1
-        
-    params[:n] = []
-    interp.get_env().set_positional_args(params)
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# export special builtin
-#-------------------------------------------------------------------------------  
-OPT_EXPORT = NonExitingParser(usage="set - set or unset options and positional parameters")
-OPT_EXPORT.add_option('-p', action='store_true', dest='has_p', default=False)
-
-def builtin_export(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    option, args = OPT_EXPORT.parse_args(args)
-    if option.has_p:
-        raise NotImplementedError()
-    
-    for arg in args:
-        try:
-            name, value = arg.split('=', 1)
-        except ValueError:
-            name, value = arg, None
-        env = interp.get_env().export(name, value)
-    
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# return special builtin
-#-------------------------------------------------------------------------------  
-def builtin_return(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    res = 0
-    if args:
-        try:
-            res = int(args[0])
-        except ValueError:
-            res = 0
-        if not 0<=res<=255:
-            res = 0
-            
-    # BUG: should be last executed command exit code        
-    raise ReturnSignal(res)
-
-#-------------------------------------------------------------------------------  
-# trap special builtin
-#-------------------------------------------------------------------------------  
-def builtin_trap(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    if len(args) < 2:
-        stderr.write('trap: usage: trap [[arg] signal_spec ...]\n')
-        return 2
-
-    action = args[0]
-    for sig in args[1:]:
-        try:
-            env.traps[sig] = action
-        except Exception as e:
-            stderr.write('trap: %s\n' % str(e))
-    return 0
-
-#-------------------------------------------------------------------------------  
-# unset special builtin
-#-------------------------------------------------------------------------------
-OPT_UNSET = NonExitingParser("unset - unset values and attributes of variables and functions")
-OPT_UNSET.add_option( '-f', action='store_true', dest='has_f', default=False)
-OPT_UNSET.add_option( '-v', action='store_true', dest='has_v', default=False)
-
-def builtin_unset(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    option, args = OPT_UNSET.parse_args(args)
-    
-    status = 0
-    env = interp.get_env()
-    for arg in args:    
-        try:
-            if option.has_f:
-                env.remove_function(arg)
-            else:
-                del env[arg]
-        except KeyError:
-            pass
-        except VarAssignmentError:
-            status = 1
-            
-    return status
-
-#-------------------------------------------------------------------------------  
-# wait special builtin
-#-------------------------------------------------------------------------------  
-def builtin_wait(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    return interp.wait([int(arg) for arg in args])
-
-#-------------------------------------------------------------------------------  
-# cat utility
-#-------------------------------------------------------------------------------
-def utility_cat(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    if not args:
-        args = ['-']
-
-    status = 0
-    for arg in args:
-        if arg == '-':
-            data = stdin.read()
-        else:
-            path = os.path.join(env['PWD'], arg)
-            try:
-                f = file(path, 'rb')
-                try:
-                    data = f.read()
-                finally:
-                    f.close()
-            except IOError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-                status = 1
-                continue
-        stdout.write(data)
-        stdout.flush()
-    return status
-    
-#-------------------------------------------------------------------------------  
-# cd utility
-#-------------------------------------------------------------------------------  
-OPT_CD = NonExitingParser("cd - change the working directory")
-
-def utility_cd(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    option, args = OPT_CD.parse_args(args)
-    env = interp.get_env()
-    
-    directory = None
-    printdir = False
-    if not args:
-        home = env.get('HOME')
-        if home:
-            # Unspecified, do nothing
-            return 0
-        else:
-            directory = home
-    elif len(args)==1:
-        directory = args[0]
-        if directory=='-':
-            if 'OLDPWD' not in env:
-                raise UtilityError("OLDPWD not set")
-            printdir = True
-            directory = env['OLDPWD']
-    else:
-        raise UtilityError("too many arguments")
-            
-    curpath = None
-    # Absolute directories will be handled correctly by the os.path.join call.
-    if not directory.startswith('.') and not directory.startswith('..'):
-        cdpaths = env.get('CDPATH', '.').split(';')
-        for cdpath in cdpaths:
-            p = os.path.join(cdpath, directory)
-            if os.path.isdir(p):
-                curpath = p
-                break
-    
-    if curpath is None:
-        curpath = directory
-    curpath = os.path.join(env['PWD'], directory)
-
-    env['OLDPWD'] = env['PWD']
-    env['PWD'] = curpath
-    if printdir:
-        stdout.write('%s\n' % curpath)
-    return 0
-
-#-------------------------------------------------------------------------------  
-# colon utility
-#-------------------------------------------------------------------------------  
-def utility_colon(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# echo utility
-#-------------------------------------------------------------------------------  
-def utility_echo(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    # Echo only takes arguments, no options. Use printf if you need fancy stuff.
-    output = ' '.join(args) + '\n'
-    stdout.write(output)
-    stdout.flush()
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# egrep utility
-#-------------------------------------------------------------------------------
-# egrep is usually a shell script.
-# Unfortunately, pysh does not support shell scripts *with arguments* right now,
-# so the redirection is implemented here, assuming grep is available.
-def utility_egrep(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    return run_command('grep', ['-E'] + args, interp, env, stdin, stdout, 
-        stderr, debugflags)
-    
-#-------------------------------------------------------------------------------  
-# env utility
-#-------------------------------------------------------------------------------  
-def utility_env(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    
-    if args and args[0]=='-i':
-        raise NotImplementedError('env: -i option is not implemented')
-    
-    i = 0
-    for arg in args:
-        if '=' not in arg:
-            break
-        # Update the current environment
-        name, value = arg.split('=', 1)
-        env[name] = value
-        i += 1
-        
-    if args[i:]:
-        # Find then execute the specified interpreter
-        utility = env.find_in_path(args[i])
-        if not utility:
-            return 127
-        args[i:i+1] = utility
-        name = args[i]
-        args = args[i+1:]
-        try:
-            return run_command(name, args, interp, env, stdin, stdout, stderr, 
-                debugflags)
-        except UtilityError:
-            stderr.write('env: failed to execute %s' % ' '.join([name]+args))
-            return 126            
-    else:
-        for pair in env.get_variables().iteritems():
-            stdout.write('%s=%s\n' % pair)
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# exit utility
-#-------------------------------------------------------------------------------
-def utility_exit(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    res = None
-    if args:
-        try:
-            res = int(args[0])
-        except ValueError:
-            res = None
-        if not 0<=res<=255:
-            res = None
-            
-    if res is None:
-        # BUG: should be last executed command exit code
-        res = 0
-        
-    raise ExitSignal(res)
-
-#-------------------------------------------------------------------------------  
-# fgrep utility
-#-------------------------------------------------------------------------------
-# see egrep
-def utility_fgrep(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    return run_command('grep', ['-F'] + args, interp, env, stdin, stdout, 
-        stderr, debugflags)
-
-#-------------------------------------------------------------------------------  
-# gunzip utility
-#-------------------------------------------------------------------------------
-# see egrep
-def utility_gunzip(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    return run_command('gzip', ['-d'] + args, interp, env, stdin, stdout, 
-        stderr, debugflags)
-    
-#-------------------------------------------------------------------------------  
-# kill utility
-#-------------------------------------------------------------------------------
-def utility_kill(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    for arg in args:
-        pid = int(arg)
-        status = subprocess.call(['pskill', '/T', str(pid)],
-                                shell=True,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
-        # pskill is asynchronous, hence the stupid polling loop
-        while 1:
-            p = subprocess.Popen(['pslist', str(pid)],
-                                shell=True,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.STDOUT)
-            output = p.communicate()[0]
-            if ('process %d was not' % pid) in output:
-                break
-            time.sleep(1)
-    return status
-    
-#-------------------------------------------------------------------------------  
-# mkdir utility
-#-------------------------------------------------------------------------------
-OPT_MKDIR = NonExitingParser("mkdir - make directories.")
-OPT_MKDIR.add_option('-p', action='store_true', dest='has_p', default=False)
-
-def utility_mkdir(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    # TODO: implement umask
-    # TODO: implement proper utility error report
-    option, args = OPT_MKDIR.parse_args(args)
-    for arg in args:
-        path = os.path.join(env['PWD'], arg)
-        if option.has_p:
-            try:
-                os.makedirs(path)
-            except IOError as e:
-                if e.errno != errno.EEXIST:
-                    raise
-        else:               
-            os.mkdir(path)
-    return 0
-
-#-------------------------------------------------------------------------------  
-# netstat utility
-#-------------------------------------------------------------------------------
-def utility_netstat(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    # Do you really expect me to implement netstat ?
-    # This empty form is enough for Mercurial tests since it's
-    # supposed to generate nothing upon success. Faking this test
-    # is not a big deal either.
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# pwd utility
-#-------------------------------------------------------------------------------  
-OPT_PWD = NonExitingParser("pwd - return working directory name")
-OPT_PWD.add_option('-L', action='store_true', dest='has_L', default=True,
-    help="""If the PWD environment variable contains an absolute pathname of \
-    the current directory that does not contain the filenames dot or dot-dot, \
-    pwd shall write this pathname to standard output. Otherwise, the -L option \
-    shall behave as the -P option.""")
-OPT_PWD.add_option('-P', action='store_true', dest='has_L', default=False,
-    help="""The absolute pathname written shall not contain filenames that, in \
-    the context of the pathname, refer to files of type symbolic link.""")
-
-def utility_pwd(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    option, args = OPT_PWD.parse_args(args)        
-    stdout.write('%s\n' % env['PWD'])
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# printf utility
-#-------------------------------------------------------------------------------
-RE_UNESCAPE = re.compile(r'(\\x[a-zA-Z0-9]{2}|\\[0-7]{1,3}|\\.)')
-
-def utility_printf(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    def replace(m):
-        assert m.group()
-        g = m.group()[1:]
-        if g.startswith('x'):
-            return chr(int(g[1:], 16))
-        if len(g) <= 3 and len([c for c in g if c in '01234567']) == len(g):
-            # Yay, an octal number
-            return chr(int(g, 8))
-        return {
-            'a': '\a',
-            'b': '\b',
-            'f': '\f',
-            'n': '\n',
-            'r': '\r',
-            't': '\t',
-            'v': '\v',
-            '\\': '\\',
-        }.get(g)
-        
-    # Convert escape sequences
-    format = re.sub(RE_UNESCAPE, replace, args[0])
-    stdout.write(format % tuple(args[1:]))
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# true utility
-#-------------------------------------------------------------------------------
-def utility_true(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    return 0
-
-#-------------------------------------------------------------------------------  
-# sed utility
-#-------------------------------------------------------------------------------
-RE_SED = re.compile(r'^s(.).*\1[a-zA-Z]*$')
-
-# cygwin sed fails with some expressions when they do not end with a single space.
-# see unit tests for details. Interestingly, the same expressions works perfectly
-# in cygwin shell.
-def utility_sed(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    # Scan pattern arguments and append a space if necessary
-    for i in range(len(args)):
-        if not RE_SED.search(args[i]):
-            continue
-        args[i] = args[i] + ' '
-
-    return run_command(name, args, interp, env, stdin, stdout, 
-        stderr, debugflags)
-
-#-------------------------------------------------------------------------------  
-# sleep utility
-#-------------------------------------------------------------------------------
-def utility_sleep(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    time.sleep(int(args[0]))
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# sort utility
-#-------------------------------------------------------------------------------
-OPT_SORT = NonExitingParser("sort - sort, merge, or sequence check text files")
-
-def utility_sort(name, args, interp, env, stdin, stdout, stderr, debugflags):
-
-    def sort(path):
-        if path == '-':
-            lines = stdin.readlines()
-        else:
-            try:
-                f = file(path)
-                try:
-                    lines = f.readlines()
-                finally:
-                    f.close()
-            except IOError as e:
-                stderr.write(str(e) + '\n')
-                return 1
-        
-        if lines and lines[-1][-1]!='\n':
-            lines[-1] = lines[-1] + '\n'
-        return lines
-    
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    option, args = OPT_SORT.parse_args(args)
-    alllines = []
-    
-    if len(args)<=0:
-        args += ['-']
-        
-    # Load all files lines
-    curdir = os.getcwd()
-    try:
-        os.chdir(env['PWD'])
-        for path in args:
-            alllines += sort(path)
-    finally:
-        os.chdir(curdir)
-            
-    alllines.sort()
-    for line in alllines:
-        stdout.write(line)
-    return 0
-    
-#-------------------------------------------------------------------------------
-# hg utility
-#-------------------------------------------------------------------------------
-
-hgcommands = [
-    'add',
-    'addremove',
-    'commit', 'ci',
-    'debugrename',
-    'debugwalk',
-    'falabala', # Dummy command used in a mercurial test
-    'incoming',
-    'locate',
-    'pull',
-    'push',
-    'qinit',
-    'remove', 'rm',
-    'rename', 'mv',
-    'revert',    
-    'showconfig',
-    'status', 'st',
-    'strip',
-    ]
-
-def rewriteslashes(name, args):
-    # Several hg commands output file paths, rewrite the separators
-    if len(args) > 1 and name.lower().endswith('python') \
-       and args[0].endswith('hg'):
-        for cmd in hgcommands:
-            if cmd in args[1:]:
-                return True
-            
-    # svn output contains many paths with OS specific separators.
-    # Normalize these to unix paths.
-    base = os.path.basename(name)
-    if base.startswith('svn'):
-        return True
-    
-    return False
-
-def rewritehg(output):
-    if not output:
-        return output
-    # Rewrite os specific messages
-    output = output.replace(': The system cannot find the file specified',
-                            ': No such file or directory')
-    output = re.sub(': Access is denied.*$', ': Permission denied', output)
-    output = output.replace(': No connection could be made because the target machine actively refused it',
-                            ': Connection refused')
-    return output
-                            
-
-def run_command(name, args, interp, env, stdin, stdout,
-                stderr, debugflags):
-    # Execute the command
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    hgbin = interp.options().hgbinary
-    ishg = hgbin and ('hg' in name or args and 'hg' in args[0])
-    unixoutput = 'cygwin' in name or ishg
-    
-    exec_env = env.get_variables()        
-    try:
-        # BUG: comparing file descriptor is clearly not a reliable way to tell
-        # whether they point on the same underlying object. But in pysh limited
-        # scope this is usually right, we do not expect complicated redirections
-        # besides usual 2>&1.
-        # Still there is one case we have but cannot deal with is when stdout
-        # and stderr are redirected *by pysh caller*. This the reason for the
-        # --redirect pysh() option.
-        # Now, we want to know they are the same because we sometimes need to 
-        # transform the command output, mostly remove CR-LF to ensure that
-        # command output is unix-like. Cygwin utilies are a special case because
-        # they explicitely set their output streams to binary mode, so we have
-        # nothing to do. For all others commands, we have to guess whether they
-        # are sending text data, in which case the transformation must be done.
-        # Again, the NUL character test is unreliable but should be enough for
-        # hg tests.
-        redirected = stdout.fileno()==stderr.fileno()
-        if not redirected:
-            p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, 
-                    stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        else:
-            p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, 
-                    stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        out, err = p.communicate()
-    except WindowsError as e:
-        raise UtilityError(str(e))
-
-    if not unixoutput:
-        def encode(s):
-            if '\0' in s:
-                return s
-            return s.replace('\r\n', '\n')
-    else:
-        encode = lambda s: s
-
-    if rewriteslashes(name, args):
-        encode1_ = encode
-        def encode(s):
-            s = encode1_(s)
-            s = s.replace('\\\\', '\\')
-            s = s.replace('\\', '/')
-            return s
-
-    if ishg:
-        encode2_ = encode
-        def encode(s):
-            return rewritehg(encode2_(s))
-    
-    stdout.write(encode(out))
-    if not redirected:
-        stderr.write(encode(err))
-    return p.returncode
-            

bitbake/lib/bb/pysh/lsprof.py

@@ -1,116 +0,0 @@
-#! /usr/bin/env python
-
-import sys
-from _lsprof import Profiler, profiler_entry
-
-__all__ = ['profile', 'Stats']
-
-def profile(f, *args, **kwds):
-    """XXX docstring"""
-    p = Profiler()
-    p.enable(subcalls=True, builtins=True)
-    try:
-        f(*args, **kwds)
-    finally:
-        p.disable()
-    return Stats(p.getstats())
-
-
-class Stats(object):
-    """XXX docstring"""
-
-    def __init__(self, data):
-        self.data = data
-
-    def sort(self, crit="inlinetime"):
-        """XXX docstring"""
-        if crit not in profiler_entry.__dict__:
-            raise ValueError("Can't sort by %s" % crit)
-        self.data.sort(lambda b, a: cmp(getattr(a, crit),
-                                        getattr(b, crit)))
-        for e in self.data:
-            if e.calls:
-                e.calls.sort(lambda b, a: cmp(getattr(a, crit),
-                                              getattr(b, crit)))
-
-    def pprint(self, top=None, file=None, limit=None, climit=None):
-        """XXX docstring"""
-        if file is None:
-            file = sys.stdout
-        d = self.data
-        if top is not None:
-            d = d[:top]
-        cols = "% 12s %12s %11.4f %11.4f   %s\n"
-        hcols = "% 12s %12s %12s %12s %s\n"
-        cols2 = "+%12s %12s %11.4f %11.4f +  %s\n"
-        file.write(hcols % ("CallCount", "Recursive", "Total(ms)",
-                            "Inline(ms)", "module:lineno(function)"))
-        count = 0
-        for e in d:
-            file.write(cols % (e.callcount, e.reccallcount, e.totaltime,
-                               e.inlinetime, label(e.code)))
-            count += 1
-            if limit is not None and count == limit:
-                return
-            ccount = 0
-            if e.calls:
-                for se in e.calls:
-                    file.write(cols % ("+%s" % se.callcount, se.reccallcount,
-                                       se.totaltime, se.inlinetime,
-                                       "+%s" % label(se.code)))
-                    count += 1
-                    ccount += 1
-                    if limit is not None and count == limit:
-                        return
-                    if climit is not None and ccount == climit:
-                        break
-
-    def freeze(self):
-        """Replace all references to code objects with string
-        descriptions; this makes it possible to pickle the instance."""
-
-        # this code is probably rather ickier than it needs to be!
-        for i in range(len(self.data)):
-            e = self.data[i]
-            if not isinstance(e.code, str):
-                self.data[i] = type(e)((label(e.code),) + e[1:])
-            if e.calls:
-                for j in range(len(e.calls)):
-                    se = e.calls[j]
-                    if not isinstance(se.code, str):
-                        e.calls[j] = type(se)((label(se.code),) + se[1:])
-
-_fn2mod = {}
-
-def label(code):
-    if isinstance(code, str):
-        return code
-    try:
-        mname = _fn2mod[code.co_filename]
-    except KeyError:
-        for k, v in sys.modules.items():
-            if v is None:
-                continue
-            if not hasattr(v, '__file__'):
-                continue
-            if not isinstance(v.__file__, str):
-                continue
-            if v.__file__.startswith(code.co_filename):
-                mname = _fn2mod[code.co_filename] = k
-                break
-        else:
-            mname = _fn2mod[code.co_filename] = '<%s>'%code.co_filename
-
-    return '%s:%d(%s)' % (mname, code.co_firstlineno, code.co_name)
-
-
-if __name__ == '__main__':
-    import os
-    sys.argv = sys.argv[1:]
-    if not sys.argv:
-        print >> sys.stderr, "usage: lsprof.py <script> <arguments...>"
-        sys.exit(2)
-    sys.path.insert(0, os.path.abspath(os.path.dirname(sys.argv[0])))
-    stats = profile(execfile, sys.argv[0], globals(), locals())
-    stats.sort()
-    stats.pprint()

bitbake/lib/bb/pysh/pysh.py

@@ -1,167 +0,0 @@
-# pysh.py - command processing for pysh.
-#
-# Copyright 2007 Patrick Mezard
-#
-# This software may be used and distributed according to the terms
-# of the GNU General Public License, incorporated herein by reference.
-
-import optparse
-import os
-import sys
-
-import interp
-
-SH_OPT = optparse.OptionParser(prog='pysh', usage="%prog [OPTIONS]", version='0.1')
-SH_OPT.add_option('-c', action='store_true', dest='command_string', default=None, 
-    help='A string that shall be interpreted by the shell as one or more commands')
-SH_OPT.add_option('--redirect-to', dest='redirect_to', default=None, 
-    help='Redirect script commands stdout and stderr to the specified file')
-# See utility_command in builtin.py about the reason for this flag.
-SH_OPT.add_option('--redirected', dest='redirected', action='store_true', default=False, 
-    help='Tell the interpreter that stdout and stderr are actually the same objects, which is really stdout')
-SH_OPT.add_option('--debug-parsing', action='store_true', dest='debug_parsing', default=False, 
-    help='Trace PLY execution')
-SH_OPT.add_option('--debug-tree', action='store_true', dest='debug_tree', default=False, 
-    help='Display the generated syntax tree.')
-SH_OPT.add_option('--debug-cmd', action='store_true', dest='debug_cmd', default=False, 
-    help='Trace command execution before parameters expansion and exit status.')
-SH_OPT.add_option('--debug-utility', action='store_true', dest='debug_utility', default=False, 
-    help='Trace utility calls, after parameters expansions')
-SH_OPT.add_option('--ast', action='store_true', dest='ast', default=False,
-    help='Encoded commands to execute in a subprocess')
-SH_OPT.add_option('--profile', action='store_true', default=False,
-    help='Profile pysh run')
-    
-    
-def split_args(args):
-    # Separate shell arguments from command ones
-    # Just stop at the first argument not starting with a dash. I know, this is completely broken,
-    # it ignores files starting with a dash or may take option values for command file. This is not
-    # supposed to happen for now
-    command_index = len(args)
-    for i,arg in enumerate(args):
-        if not arg.startswith('-'):
-            command_index = i
-            break
-            
-    return args[:command_index], args[command_index:]
-
-
-def fixenv(env):
-    path = env.get('PATH')
-    if path is not None:
-        parts = path.split(os.pathsep)
-        # Remove Windows utilities from PATH, they are useless at best and
-        # some of them (find) may be confused with other utilities.
-        parts = [p for p in parts if 'system32' not in p.lower()]
-        env['PATH'] = os.pathsep.join(parts)
-    if env.get('HOME') is None:
-        # Several utilities, including cvsps, cannot work without
-        # a defined HOME directory.
-        env['HOME'] = os.path.expanduser('~')
-    return env
-
-def _sh(cwd, shargs, cmdargs, options, debugflags=None, env=None):
-    if os.environ.get('PYSH_TEXT') != '1':
-        import msvcrt
-        for fp in (sys.stdin, sys.stdout, sys.stderr):
-            msvcrt.setmode(fp.fileno(), os.O_BINARY)
-
-    hgbin = os.environ.get('PYSH_HGTEXT') != '1'
-    
-    if debugflags is None:
-        debugflags = []
-        if options.debug_parsing:    debugflags.append('debug-parsing')
-        if options.debug_utility:    debugflags.append('debug-utility')
-        if options.debug_cmd:        debugflags.append('debug-cmd')
-        if options.debug_tree:       debugflags.append('debug-tree')
-    
-    if env is None:
-        env = fixenv(dict(os.environ))
-    if cwd is None:
-        cwd = os.getcwd()
-
-    if not cmdargs:
-        # Nothing to do
-        return 0
-
-    ast = None
-    command_file = None
-    if options.command_string:
-        input = cmdargs[0]
-        if not options.ast:
-            input += '\n'
-        else:
-            args, input = interp.decodeargs(input), None
-            env, ast = args
-            cwd = env.get('PWD', cwd)
-    else:
-        command_file = cmdargs[0]
-        arguments = cmdargs[1:]
-
-        prefix = interp.resolve_shebang(command_file, ignoreshell=True)
-        if prefix:
-            input = ' '.join(prefix + [command_file] + arguments)
-        else:
-            # Read commands from file
-            f = file(command_file)
-            try:
-                # Trailing newline to help the parser
-                input = f.read() + '\n'
-            finally:
-                f.close()
-    
-    redirect = None
-    try:
-        if options.redirected:
-            stdout = sys.stdout
-            stderr = stdout
-        elif options.redirect_to:
-            redirect = open(options.redirect_to, 'wb')
-            stdout = redirect
-            stderr = redirect
-        else:
-            stdout = sys.stdout
-            stderr = sys.stderr
-            
-        # TODO: set arguments to environment variables
-        opts = interp.Options()
-        opts.hgbinary = hgbin
-        ip = interp.Interpreter(cwd, debugflags, stdout=stdout, stderr=stderr,
-                                opts=opts)
-        try:
-            # Export given environment in shell object
-            for k,v in env.iteritems():
-                ip.get_env().export(k,v)
-            return ip.execute_script(input, ast, scriptpath=command_file)
-        finally:
-            ip.close()
-    finally:
-        if redirect is not None:
-            redirect.close()
-
-def sh(cwd=None, args=None, debugflags=None, env=None):
-    if args is None:
-        args = sys.argv[1:]
-    shargs, cmdargs = split_args(args)
-    options, shargs = SH_OPT.parse_args(shargs)
-
-    if options.profile:
-        import lsprof
-        p = lsprof.Profiler()
-        p.enable(subcalls=True)
-        try:
-            return _sh(cwd, shargs, cmdargs, options, debugflags, env)
-        finally:
-            p.disable()
-            stats = lsprof.Stats(p.getstats())
-            stats.sort()
-            stats.pprint(top=10, file=sys.stderr, climit=5)
-    else:
-        return _sh(cwd, shargs, cmdargs, options, debugflags, env)
-            
-def main():
-    sys.exit(sh())
-
-if __name__=='__main__':
-    main()

bitbake/lib/bb/pysh/pyshlex.py

@@ -13,11 +13,6 @@
 # PLY in pull mode. It was designed to work incrementally and it would not be
 # that hard to enable pull mode.
 import re
-try:
-    s = set()
-    del s
-except NameError:
-    from Set import Set as set
 
 from ply import lex
 from bb.pysh.sherrors import *

bitbake/lib/bb/pysh/pyshyacc.py

@@ -570,6 +570,7 @@ def p_linebreak(p):
 
 def p_separator_op(p):                 
     """separator_op : COMMA
+                    | COMMA COMMA
                     | AMP"""
     p[0] = p[1]
 
@@ -636,13 +637,16 @@ def p_empty(p):
 def p_error(p):
     msg = []
     w = msg.append
-    w('%r\n' % p)
-    w('followed by:\n')
-    for i in range(5):
-        n = yacc.token()
-        if not n:
-            break
-        w('  %r\n' % n)
+    if p:
+        w('%r\n' % p)
+        w('followed by:\n')
+        for i in range(5):
+            n = yacc.token()
+            if not n:
+                break
+            w('  %r\n' % n)
+    else:
+        w('Unexpected EOF')
     raise sherrors.ShellSyntaxError(''.join(msg))
 
 # Build the parser

bitbake/lib/bb/pysh/sherrors.py

@@ -13,29 +13,3 @@ class ShellError(Exception):
 
 class ShellSyntaxError(ShellError):
     pass
-    
-class UtilityError(ShellError):
-    """Raised upon utility syntax error (option or operand error)."""
-    pass
-   
-class ExpansionError(ShellError):
-    pass
-     
-class CommandNotFound(ShellError):
-    """Specified command was not found."""
-    pass
-    
-class RedirectionError(ShellError):
-    pass
-    
-class VarAssignmentError(ShellError):
-    """Variable assignment error."""
-    pass
-    
-class ExitSignal(ShellError):
-    """Exit signal."""
-    pass
-    
-class ReturnSignal(ShellError):
-    """Exit signal."""
-    pass

bitbake/lib/bb/pysh/subprocess_fix.py

@@ -1,77 +0,0 @@
-# subprocess - Subprocesses with accessible I/O streams
-#
-# For more information about this module, see PEP 324.
-#
-# This module should remain compatible with Python 2.2, see PEP 291.
-#
-# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
-#
-# Licensed to PSF under a Contributor Agreement.
-# See http://www.python.org/2.4/license for licensing details.
-
-def list2cmdline(seq):
-    """
-    Translate a sequence of arguments into a command line
-    string, using the same rules as the MS C runtime:
-
-    1) Arguments are delimited by white space, which is either a
-       space or a tab.
-
-    2) A string surrounded by double quotation marks is
-       interpreted as a single argument, regardless of white space
-       contained within.  A quoted string can be embedded in an
-       argument.
-
-    3) A double quotation mark preceded by a backslash is
-       interpreted as a literal double quotation mark.
-
-    4) Backslashes are interpreted literally, unless they
-       immediately precede a double quotation mark.
-
-    5) If backslashes immediately precede a double quotation mark,
-       every pair of backslashes is interpreted as a literal
-       backslash.  If the number of backslashes is odd, the last
-       backslash escapes the next double quotation mark as
-       described in rule 3.
-    """
-
-    # See
-    # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
-    result = []
-    needquote = False
-    for arg in seq:
-        bs_buf = []
-
-        # Add a space to separate this argument from the others
-        if result:
-            result.append(' ')
-
-        needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or arg == ""
-        if needquote:
-            result.append('"')
-
-        for c in arg:
-            if c == '\\':
-                # Don't know if we need to double yet.
-                bs_buf.append(c)
-            elif c == '"':
-                # Double backspaces.
-                result.append('\\' * len(bs_buf)*2)
-                bs_buf = []
-                result.append('\\"')
-            else:
-                # Normal char
-                if bs_buf:
-                    result.extend(bs_buf)
-                    bs_buf = []
-                result.append(c)
-
-        # Add remaining backspaces, if any.
-        if bs_buf:
-            result.extend(bs_buf)
-
-        if needquote:
-            result.extend(bs_buf)
-            result.append('"')
-            
-    return ''.join(result)

bitbake/lib/bb/remotedata.py

@@ -6,18 +6,8 @@ Provides support for using a datastore from the bitbake client
 
 # Copyright (C) 2016  Intel Corporation
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import bb.data
 
@@ -27,16 +17,16 @@ class RemoteDatastores:
         self.cooker = cooker
         self.datastores = {}
         self.locked = []
+        self.datastores[0] = self.cooker.data
         self.nextindex = 1
 
     def __len__(self):
         return len(self.datastores)
 
     def __getitem__(self, key):
-        if key is None:
-            return self.cooker.data
-        else:
-            return self.datastores[key]
+        # Cooker could have changed its datastore from under us
+        self.datastores[0] = self.cooker.data
+        return self.datastores[key]
 
     def items(self):
         return self.datastores.items()
@@ -73,44 +63,3 @@ class RemoteDatastores:
             raise Exception('Tried to release locked datastore %d' % idx)
         del self.datastores[idx]
 
-    def receive_datastore(self, remote_data):
-        """Receive a datastore object sent from the client (as prepared by transmit_datastore())"""
-        dct = dict(remote_data)
-        d = bb.data_smart.DataSmart()
-        d.dict = dct
-        while True:
-            if '_remote_data' in dct:
-                dsindex = dct['_remote_data']['_content']
-                del dct['_remote_data']
-                if dsindex is None:
-                    dct['_data'] = self.cooker.data.dict
-                else:
-                    dct['_data'] = self.datastores[dsindex].dict
-                break
-            elif '_data' in dct:
-                idct = dict(dct['_data'])
-                dct['_data'] = idct
-                dct = idct
-            else:
-                break
-        return d
-
-    @staticmethod
-    def transmit_datastore(d):
-        """Prepare a datastore object for sending over IPC from the client end"""
-        # FIXME content might be a dict, need to turn that into a list as well
-        def copy_dicts(dct):
-            if '_remote_data' in dct:
-                dsindex = dct['_remote_data']['_content'].dsindex
-                newdct = dct.copy()
-                newdct['_remote_data'] = {'_content': dsindex}
-                return list(newdct.items())
-            elif '_data' in dct:
-                newdct = dct.copy()
-                newdata = copy_dicts(dct['_data'])
-                if newdata:
-                    newdct['_data'] = newdata
-                return list(newdct.items())
-            return None
-        main_dict = copy_dicts(d.dict)
-        return main_dict

bitbake/lib/bb/server/__init__.py

@@ -5,17 +5,5 @@
 # Copyright (C) 2006 - 2008  Richard Purdie
 # Copyright (C) 2013         Alexandru Damian
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-

bitbake/lib/bb/server/process.py

@@ -3,18 +3,8 @@
 #
 # Copyright (C) 2010 Bob Foerster <robert@erafx.com>
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 """
     This module implements a multiprocessing.Process based server for bitbake.
@@ -57,8 +47,9 @@ class ProcessServer(multiprocessing.Process):
         self.next_heartbeat = time.time()
 
         self.event_handle = None
+        self.hadanyui = False
         self.haveui = False
-        self.lastui = False
+        self.maxuiwait = 30
         self.xmlrpc = False
 
         self._idlefuns = {}
@@ -130,6 +121,7 @@ class ProcessServer(multiprocessing.Process):
         bb.utils.set_process_name("Cooker")
 
         ready = []
+        newconnections = []
 
         self.controllersock = False
         fds = [self.sock]
@@ -138,37 +130,49 @@ class ProcessServer(multiprocessing.Process):
         print("Entering server connection loop")
 
         def disconnect_client(self, fds):
-            if not self.haveui:
-                return
             print("Disconnecting Client")
-            fds.remove(self.controllersock)
-            fds.remove(self.command_channel)
-            bb.event.unregister_UIHhandler(self.event_handle, True)
-            self.command_channel_reply.writer.close()
-            self.event_writer.writer.close()
-            del self.event_writer
-            self.controllersock.close()
-            self.controllersock = False
-            self.haveui = False
-            self.lastui = time.time()
-            self.cooker.clientComplete()
-            if self.timeout is None:
+            if self.controllersock:
+                fds.remove(self.controllersock)
+                self.controllersock.close()
+                self.controllersock = False
+            if self.haveui:
+                fds.remove(self.command_channel)
+                bb.event.unregister_UIHhandler(self.event_handle, True)
+                self.command_channel_reply.writer.close()
+                self.event_writer.writer.close()
+                self.command_channel.close()
+                self.command_channel = False
+                del self.event_writer
+                self.lastui = time.time()
+                self.cooker.clientComplete()
+                self.haveui = False
+            ready = select.select(fds,[],[],0)[0]
+            if newconnections:
+                print("Starting new client")
+                conn = newconnections.pop(-1)
+                fds.append(conn)
+                self.controllersock = conn
+            elif self.timeout is None and not ready:
                 print("No timeout, exiting.")
                 self.quit = True
 
+        self.lastui = time.time()
         while not self.quit:
             if self.sock in ready:
-                self.controllersock, address = self.sock.accept()
-                if self.haveui:
-                    print("Dropping connection attempt as we have a UI %s" % (str(ready)))
-                    self.controllersock.close()
-                else:
-                    print("Accepting %s" % (str(ready)))
-                    fds.append(self.controllersock)
+                while select.select([self.sock],[],[],0)[0]:
+                    controllersock, address = self.sock.accept()
+                    if self.controllersock:
+                        print("Queuing %s (%s)" % (str(ready), str(newconnections)))
+                        newconnections.append(controllersock)
+                    else:
+                        print("Accepting %s (%s)" % (str(ready), str(newconnections)))
+                        self.controllersock = controllersock
+                        fds.append(controllersock)
             if self.controllersock in ready:
                 try:
-                    print("Connecting Client")
+                    print("Processing Client")
                     ui_fds = recvfds(self.controllersock, 3)
+                    print("Connecting Client")
 
                     # Where to write events to
                     writer = ConnectionWriter(ui_fds[0])
@@ -185,15 +189,23 @@ class ProcessServer(multiprocessing.Process):
                     self.command_channel_reply = writer
 
                     self.haveui = True
+                    self.hadanyui = True
 
                 except (EOFError, OSError):
                     disconnect_client(self, fds)
 
-            if not self.timeout == -1.0 and not self.haveui and self.lastui and self.timeout and \
+            if not self.timeout == -1.0 and not self.haveui and self.timeout and \
                     (self.lastui + self.timeout) < time.time():
                 print("Server timeout, exiting.")
                 self.quit = True
 
+            # If we don't see a UI connection within maxuiwait, its unlikely we're going to see
+            # one. We have had issue with processes hanging indefinitely so timing out UI-less
+            # servers is useful.
+            if not self.hadanyui and not self.xmlrpc and not self.timeout and (self.lastui + self.maxuiwait) < time.time():
+                print("No UI connection within max timeout, exiting to avoid infinite loop.")
+                self.quit = True
+
             if self.command_channel in ready:
                 try:
                     command = self.command_channel.get()
@@ -218,10 +230,13 @@ class ProcessServer(multiprocessing.Process):
 
         print("Exiting")
         # Remove the socket file so we don't get any more connections to avoid races
-        os.unlink(self.sockname)
+        try:
+            os.unlink(self.sockname)
+        except:
+            pass
         self.sock.close()
 
-        try: 
+        try:
             self.cooker.shutdown(True)
             self.cooker.notifier.stop()
             self.cooker.confignotifier.stop()
@@ -239,6 +254,12 @@ class ProcessServer(multiprocessing.Process):
         while not lock:
             with bb.utils.timeout(3):
                 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
+                if lock:
+                    # We hold the lock so we can remove the file (hide stale pid data)
+                    # via unlockfile.
+                    bb.utils.unlockfile(lock)
+                    return
+
                 if not lock:
                     # Some systems may not have lsof available
                     procs = None
@@ -259,10 +280,6 @@ class ProcessServer(multiprocessing.Process):
                     if procs:
                         msg += ":\n%s" % str(procs)
                     print(msg)
-                    return
-        # We hold the lock so we can remove the file (hide stale pid data)
-        bb.utils.remove(lockfile)
-        bb.utils.unlockfile(lock)
 
     def idle_commands(self, delay, fds=None):
         nextsleep = delay
@@ -327,7 +344,9 @@ class ServerCommunicator():
     def runCommand(self, command):
         self.connection.send(command)
         if not self.recv.poll(30):
-            raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server")
+            logger.info("No reply from server in 30s")
+            if not self.recv.poll(30):
+                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
         return self.recv.get()
 
     def updateFeatureSet(self, featureset):
@@ -377,11 +396,12 @@ class BitBakeServer(object):
         if os.path.exists(sockname):
             os.unlink(sockname)
 
+        # Place the log in the builddirectory alongside the lock file
+        logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
+
         self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         # AF_UNIX has path length issues so chdir here to workaround
         cwd = os.getcwd()
-        logfile = os.path.join(cwd, "bitbake-cookerdaemon.log")
-
         try:
             os.chdir(os.path.dirname(sockname))
             self.sock.bind(os.path.basename(sockname))
@@ -394,55 +414,75 @@ class BitBakeServer(object):
         bb.daemonize.createDaemon(self._startServer, logfile)
         self.sock.close()
         self.bitbake_lock.close()
+        os.close(self.readypipein)
 
         ready = ConnectionReader(self.readypipe)
-        r = ready.poll(30)
+        r = ready.poll(5)
+        if not r:
+            bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
+            r = ready.poll(90)
         if r:
-            r = ready.get()
-        if not r or r != "ready":
+            try:
+                r = ready.get()
+            except EOFError:
+                # Trap the child exitting/closing the pipe and error out
+                r = None
+        if not r or r[0] != "r":
             ready.close()
-            bb.error("Unable to start bitbake server")
+            bb.error("Unable to start bitbake server (%s)" % str(r))
             if os.path.exists(logfile):
                 logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
                 started = False
                 lines = []
+                lastlines = []
                 with open(logfile, "r") as f:
                     for line in f:
                         if started:
                             lines.append(line)
                         else:
+                            lastlines.append(line)
                             res = logstart_re.match(line.rstrip())
                             if res:
                                 ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
                                 if ldatetime >= startdatetime:
                                     started = True
                                     lines.append(line)
+                        if len(lastlines) > 60:
+                            lastlines = lastlines[-60:]
                 if lines:
-                    if len(lines) > 10:
-                        bb.error("Last 10 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-10:])))
+                    if len(lines) > 60:
+                        bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:])))
                     else:
                         bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
+                elif lastlines:
+                        bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines)))
+            else:
+                bb.error("%s doesn't exist" % logfile)
+
             raise SystemExit(1)
+
         ready.close()
-        os.close(self.readypipein)
 
     def _startServer(self):
         print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
+        sys.stdout.flush()
+
         server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
         self.configuration.setServerRegIdleCallback(server.register_idle_function)
+        os.close(self.readypipe)
         writer = ConnectionWriter(self.readypipein)
         try:
             self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
-            writer.send("ready")
-        except:
-            writer.send("fail")
-            raise
-        finally:
-            os.close(self.readypipein)
+        except bb.BBHandledException:
+            return None
+        writer.send("r")
+        writer.close()
         server.cooker = self.cooker
         server.server_timeout = self.configuration.server_timeout
         server.xmlrpcinterface = self.configuration.xmlrpcinterface
         print("Started bitbake server pid %d" % os.getpid())
+        sys.stdout.flush()
+
         server.start()
 
 def connectProcessServer(sockname, featureset):
@@ -451,16 +491,25 @@ def connectProcessServer(sockname, featureset):
     # AF_UNIX has path length issues so chdir here to workaround
     cwd = os.getcwd()
 
-    try:
-        os.chdir(os.path.dirname(sockname))
-        sock.connect(os.path.basename(sockname))
-    finally:
-        os.chdir(cwd)
-
     readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
     eq = command_chan_recv = command_chan = None
 
+    sock.settimeout(10)
+
     try:
+        try:
+            os.chdir(os.path.dirname(sockname))
+            finished = False
+            while not finished:
+                try:
+                    sock.connect(os.path.basename(sockname))
+                    finished = True
+                except IOError as e:
+                    if e.errno == errno.EWOULDBLOCK:
+                        pass
+                    raise
+        finally:
+            os.chdir(cwd)
 
         # Send an fd for the remote to write events to
         readfd, writefd = os.pipe()
@@ -489,7 +538,8 @@ def connectProcessServer(sockname, featureset):
             command_chan.close()
         for i in [writefd, readfd1, writefd2]:
             try:
-                os.close(i)
+                if i:
+                    os.close(i)
             except OSError:
                 pass
         sock.close()