mirror of
https://git.yoctoproject.org/poky
synced 2026-03-01 12:59:39 +01:00
5cd00e3e53
The prior fetcher did not know how to work with MIRRORS, and did not
honor BB_NO_NETWORK and similar.
The new fetcher approach recursively calls 'gitsm' download on each
submodule detected. This ensures that it will go throug the
standard download process.
Each downloaded submodule is then 'attached' to the original download in
the 'modules' directory. This mimics the behavior of:
git submodule init
but there is no chance it will contact the network without permission.
It then corrects upstream reference URIs.
The unpack steps simply copies the items from the downloads to the destdir.
Once copied the submodules are connected and we then run:
git submodule update
According to the git documentation, git submodule init can and will modify
the project configuration and may connect to the network. Doing the
work manually prevents this. (This manual process is allowed based
on my reading of the documentation.)
See: https://git-scm.com/book/en/v2/Git-Tools-Submodules
The small change to the existing test is due to this new code always assuming
the code is from a remote system, and not a 'local' repository. If this
assumption proves to be incorrect -- code will need to be added to deal
with local repositories without an upstream URI.
(Bitbake rev: 9c6b39adf9781fa6745f48913a97c859fa37eb5b)
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
236 lines
9.3 KiB
Python
236 lines
9.3 KiB
Python
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
"""
|
|
BitBake 'Fetch' git submodules implementation
|
|
|
|
Inherits from and extends the Git fetcher to retrieve submodules of a git repository
|
|
after cloning.
|
|
|
|
SRC_URI = "gitsm://<see Git fetcher for syntax>"
|
|
|
|
See the Git fetcher, git://, for usage documentation.
|
|
|
|
NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
|
|
|
|
"""
|
|
|
|
# Copyright (C) 2013 Richard Purdie
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
import os
|
|
import bb
|
|
from bb.fetch2.git import Git
|
|
from bb.fetch2 import runfetchcmd
|
|
from bb.fetch2 import logger
|
|
from bb.fetch2 import Fetch
|
|
from bb.fetch2 import BBFetchException
|
|
|
|
class GitSM(Git):
|
|
def supports(self, ud, d):
|
|
"""
|
|
Check to see if a given url can be fetched with git.
|
|
"""
|
|
return ud.type in ['gitsm']
|
|
|
|
def update_submodules(self, ud, d):
|
|
submodules = []
|
|
paths = {}
|
|
uris = {}
|
|
local_paths = {}
|
|
|
|
for name in ud.names:
|
|
try:
|
|
gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir)
|
|
except:
|
|
# No submodules to update
|
|
continue
|
|
|
|
module = ""
|
|
for line in gitmodules.splitlines():
|
|
if line.startswith('[submodule'):
|
|
module = line.split('"')[1]
|
|
submodules.append(module)
|
|
elif module and line.strip().startswith('path'):
|
|
path = line.split('=')[1].strip()
|
|
paths[module] = path
|
|
elif module and line.strip().startswith('url'):
|
|
url = line.split('=')[1].strip()
|
|
uris[module] = url
|
|
|
|
for module in submodules:
|
|
module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
|
|
module_hash = module_hash.split()[2]
|
|
|
|
# Build new SRC_URI
|
|
proto = uris[module].split(':', 1)[0]
|
|
url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
|
|
url += ';protocol=%s' % proto
|
|
url += ";name=%s" % module
|
|
url += ";qbareclone=1;nocheckout=1"
|
|
|
|
ld = d.createCopy()
|
|
# Not necessary to set SRC_URI, since we're passing the URI to
|
|
# Fetch.
|
|
#ld.setVar('SRC_URI', url)
|
|
ld.setVar('SRCREV_%s' % module, module_hash)
|
|
|
|
# Workaround for issues with SRCPV/SRCREV_FORMAT errors
|
|
# error refer to 'multiple' repositories. Only the repository
|
|
# in the original SRC_URI actually matters...
|
|
ld.setVar('SRCPV', d.getVar('SRCPV'))
|
|
ld.setVar('SRCREV_FORMAT', module)
|
|
|
|
newfetch = Fetch([url], ld, cache=False)
|
|
newfetch.download()
|
|
local_paths[module] = newfetch.localpath(url)
|
|
|
|
# Correct the submodule references to the local download version...
|
|
runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir)
|
|
try:
|
|
os.mkdir(os.path.join(ud.clonedir, 'modules'))
|
|
except OSError:
|
|
pass
|
|
if not os.path.exists(os.path.join(ud.clonedir, 'modules', paths[module])):
|
|
os.symlink(local_paths[module], os.path.join(ud.clonedir, 'modules', paths[module]))
|
|
|
|
return True
|
|
|
|
def need_update(self, ud, d):
|
|
main_repo_needs_update = Git.need_update(self, ud, d)
|
|
|
|
# First check that the main repository has enough history fetched. If it doesn't, then we don't
|
|
# even have the .gitmodules and gitlinks for the submodules to attempt asking whether the
|
|
# submodules' histories are recent enough.
|
|
if main_repo_needs_update:
|
|
return True
|
|
|
|
# Now check that the submodule histories are new enough. The git-submodule command doesn't have
|
|
# any clean interface for doing this aside from just attempting the checkout (with network
|
|
# fetched disabled).
|
|
return not self.update_submodules(ud, d)
|
|
|
|
def download(self, ud, d):
|
|
Git.download(self, ud, d)
|
|
|
|
if not ud.shallow or ud.localpath != ud.fullshallow:
|
|
self.update_submodules(ud, d)
|
|
|
|
def copy_submodules(self, submodules, ud, destdir, d):
|
|
if ud.bareclone:
|
|
repo_conf = destdir
|
|
else:
|
|
repo_conf = os.path.join(destdir, '.git')
|
|
|
|
if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')):
|
|
os.mkdir(os.path.join(repo_conf, 'modules'))
|
|
|
|
for module in submodules:
|
|
srcpath = os.path.join(ud.clonedir, 'modules', module)
|
|
modpath = os.path.join(repo_conf, 'modules', module)
|
|
|
|
if os.path.exists(srcpath):
|
|
if os.path.exists(os.path.join(srcpath, '.git')):
|
|
srcpath = os.path.join(srcpath, '.git')
|
|
|
|
target = modpath
|
|
if os.path.exists(modpath):
|
|
target = os.path.dirname(modpath)
|
|
|
|
runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d)
|
|
elif os.path.exists(modpath):
|
|
# Module already exists, likely unpacked from a shallow mirror clone
|
|
pass
|
|
else:
|
|
# This is fatal, as we do NOT want git-submodule to hit the network
|
|
raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath))
|
|
|
|
def clone_shallow_local(self, ud, dest, d):
|
|
super(GitSM, self).clone_shallow_local(ud, dest, d)
|
|
|
|
# Copy over the submodules' fetched histories too.
|
|
repo_conf = os.path.join(dest, '.git')
|
|
|
|
submodules = []
|
|
for name in ud.names:
|
|
try:
|
|
gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest)
|
|
except:
|
|
# No submodules to update
|
|
continue
|
|
|
|
for line in gitmodules.splitlines():
|
|
if line.startswith('[submodule'):
|
|
module = line.split('"')[1]
|
|
submodules.append(module)
|
|
|
|
self.copy_submodules(submodules, ud, dest, d)
|
|
|
|
def unpack(self, ud, destdir, d):
|
|
Git.unpack(self, ud, destdir, d)
|
|
|
|
# Copy over the submodules' fetched histories too.
|
|
if ud.bareclone:
|
|
repo_conf = ud.destdir
|
|
else:
|
|
repo_conf = os.path.join(ud.destdir, '.git')
|
|
|
|
submodules = []
|
|
paths = {}
|
|
uris = {}
|
|
local_paths = {}
|
|
for name in ud.names:
|
|
try:
|
|
gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
|
|
except:
|
|
# No submodules to update
|
|
continue
|
|
|
|
module = ""
|
|
for line in gitmodules.splitlines():
|
|
if line.startswith('[submodule'):
|
|
module = line.split('"')[1]
|
|
submodules.append(module)
|
|
elif module and line.strip().startswith('path'):
|
|
path = line.split('=')[1].strip()
|
|
paths[module] = path
|
|
elif module and line.strip().startswith('url'):
|
|
url = line.split('=')[1].strip()
|
|
uris[module] = url
|
|
|
|
self.copy_submodules(submodules, ud, ud.destdir, d)
|
|
|
|
for module in submodules:
|
|
srcpath = os.path.join(ud.clonedir, 'modules', module)
|
|
modpath = os.path.join(repo_conf, 'modules', module)
|
|
|
|
# Determine (from the submodule) the correct url to reference
|
|
try:
|
|
output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath)
|
|
except bb.fetch2.FetchError as e:
|
|
# No remote url defined in this submodule
|
|
continue
|
|
|
|
local_paths[module] = output
|
|
|
|
# Setup the local URL properly (like git submodule init or sync would do...)
|
|
runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
|
|
|
|
# Ensure the submodule repository is NOT set to bare, since we're checking it out...
|
|
runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath)
|
|
|
|
if submodules:
|
|
# Run submodule update, this sets up the directories -- without touching the config
|
|
runfetchcmd("%s submodule update --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
|