mirror of
https://git.yoctoproject.org/poky
synced 2026-04-21 21:32:12 +02:00
bitbake: fetch2: Add gomodgit fetcher
Add a go module fetcher for downloading module dependencies to the module cache directly from a git repository. The fetcher can be used with the go-mod class in OE-Core. A module dependency can be specified with: SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;srcrev=..." (Bitbake rev: 29ff38ccf0d5389a5bee81e252a78548361a9d7c) Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Christian Lindeberg
committed by
Richard Purdie
Richard Purdie
parent
dd7631426c
commit
87c29b5a94
@@ -2112,3 +2112,4 @@ methods.append(az.Az())
|
||||
methods.append(crate.Crate())
|
||||
methods.append(gcp.GCP())
|
||||
methods.append(gomod.GoMod())
|
||||
methods.append(gomod.GoModGit())
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
"""
|
||||
BitBake 'Fetch' implementation for Go modules
|
||||
|
||||
The gomod fetcher is used to download Go modules to the module cache from a
|
||||
module proxy.
|
||||
The gomod/gomodgit fetchers are used to download Go modules to the module cache
|
||||
from a module proxy or directly from a version control repository.
|
||||
|
||||
Example SRC_URI:
|
||||
|
||||
SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
|
||||
SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..."
|
||||
|
||||
Required SRC_URI parameters:
|
||||
|
||||
@@ -27,6 +28,23 @@ Optional SRC_URI parameters:
|
||||
only the go.mod file. Alternatively, set the SRC_URI varible flag for
|
||||
"module@version.sha256sum".
|
||||
|
||||
- protocol
|
||||
The method used when fetching directly from a version control repository.
|
||||
The default is "https" for git.
|
||||
|
||||
- repo
|
||||
The URL when fetching directly from a version control repository. Required
|
||||
when the URL is different from the module path.
|
||||
|
||||
- srcrev
|
||||
The revision identifier used when fetching directly from a version control
|
||||
repository. Alternatively, set the SRCREV varible for "module@version".
|
||||
|
||||
- subdir
|
||||
The module subdirectory when fetching directly from a version control
|
||||
repository. Required when the module is not located in the root of the
|
||||
repository.
|
||||
|
||||
Related variables:
|
||||
|
||||
- GO_MOD_PROXY
|
||||
@@ -41,14 +59,19 @@ See the Go modules reference, https://go.dev/ref/mod, for more information
|
||||
about the module cache, module proxies and version control systems.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
|
||||
import bb
|
||||
from bb.fetch2 import FetchError
|
||||
from bb.fetch2 import MissingParameterError
|
||||
from bb.fetch2 import runfetchcmd
|
||||
from bb.fetch2 import subprocess_setup
|
||||
from bb.fetch2.git import Git
|
||||
from bb.fetch2.wget import Wget
|
||||
|
||||
|
||||
@@ -126,3 +149,116 @@ class GoMod(Wget):
|
||||
# If the module does not have a go.mod file, synthesize
|
||||
# one containing only a module statement.
|
||||
mf.write(f'module {module}\n'.encode())
|
||||
|
||||
|
||||
class GoModGit(Git):
|
||||
"""Class to fetch Go modules directly from a git repository"""
|
||||
|
||||
def supports(self, ud, d):
|
||||
"""Check to see if a given URL is for this fetcher."""
|
||||
return ud.type == 'gomodgit'
|
||||
|
||||
def urldata_init(self, ud, d):
|
||||
"""Set up to download the module from the git repository.
|
||||
|
||||
Set up to download the git repository to the module cache directory and
|
||||
unpack the module zip file and the go.mod file:
|
||||
|
||||
cache/vcs/<hash>: The bare git repository.
|
||||
cache/download/<module>/@v/<version>.zip: The module zip file.
|
||||
cache/download/<module>/@v/<version>.mod: The go.mod file.
|
||||
"""
|
||||
|
||||
moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
|
||||
|
||||
if 'version' not in ud.parm:
|
||||
raise MissingParameterError('version', ud.url)
|
||||
|
||||
module = ud.host + ud.path
|
||||
ud.parm['module'] = module
|
||||
|
||||
# Set host, path and srcrev for git download
|
||||
if 'repo' in ud.parm:
|
||||
repo = ud.parm['repo']
|
||||
idx = repo.find('/')
|
||||
if idx != -1:
|
||||
ud.host = repo[:idx]
|
||||
ud.path = repo[idx:]
|
||||
else:
|
||||
ud.host = repo
|
||||
ud.path = ''
|
||||
if 'protocol' not in ud.parm:
|
||||
ud.parm['protocol'] = 'https'
|
||||
name = f"{module}@{ud.parm['version']}"
|
||||
ud.names = [name]
|
||||
srcrev = d.getVar('SRCREV_' + name)
|
||||
if srcrev:
|
||||
if 'srcrev' not in ud.parm:
|
||||
ud.parm['srcrev'] = srcrev
|
||||
else:
|
||||
if 'srcrev' in ud.parm:
|
||||
d.setVar('SRCREV_' + name, ud.parm['srcrev'])
|
||||
if 'branch' not in ud.parm:
|
||||
ud.parm['nobranch'] = '1'
|
||||
|
||||
# Set subpath, subdir and bareclone for git unpack
|
||||
if 'subdir' in ud.parm:
|
||||
ud.parm['subpath'] = ud.parm['subdir']
|
||||
key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode()
|
||||
ud.parm['key'] = key
|
||||
ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs',
|
||||
hashlib.sha256(key).hexdigest())
|
||||
ud.parm['bareclone'] = '1'
|
||||
|
||||
super().urldata_init(ud, d)
|
||||
|
||||
def unpack(self, ud, rootdir, d):
|
||||
"""Unpack the module in the module cache."""
|
||||
|
||||
# Unpack the bare git repository
|
||||
super().unpack(ud, rootdir, d)
|
||||
|
||||
moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
|
||||
|
||||
# Create the info file
|
||||
module = ud.parm['module']
|
||||
repodir = os.path.join(rootdir, ud.parm['subdir'])
|
||||
with open(repodir + '.info', 'wb') as f:
|
||||
f.write(ud.parm['key'])
|
||||
|
||||
# Unpack the go.mod file from the repository
|
||||
unpackdir = os.path.join(rootdir, moddir, 'cache/download',
|
||||
escape(module), '@v')
|
||||
bb.utils.mkdirhier(unpackdir)
|
||||
srcrev = ud.parm['srcrev']
|
||||
version = ud.parm['version']
|
||||
escaped_version = escape(version)
|
||||
cmd = f"git ls-tree -r --name-only '{srcrev}'"
|
||||
if 'subpath' in ud.parm:
|
||||
cmd += f" '{ud.parm['subpath']}'"
|
||||
files = runfetchcmd(cmd, d, workdir=repodir).split()
|
||||
name = escaped_version + '.mod'
|
||||
bb.note(f"Unpacking {name} to {unpackdir}/")
|
||||
with open(os.path.join(unpackdir, name), mode='wb') as mf:
|
||||
f = 'go.mod'
|
||||
if 'subpath' in ud.parm:
|
||||
f = os.path.join(ud.parm['subpath'], f)
|
||||
if f in files:
|
||||
cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
|
||||
subprocess.check_call(cmd, stdout=mf, cwd=repodir,
|
||||
preexec_fn=subprocess_setup)
|
||||
else:
|
||||
# If the module does not have a go.mod file, synthesize one
|
||||
# containing only a module statement.
|
||||
mf.write(f'module {module}\n'.encode())
|
||||
|
||||
# Synthesize the module zip file from the repository
|
||||
name = escaped_version + '.zip'
|
||||
bb.note(f"Unpacking {name} to {unpackdir}/")
|
||||
with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf:
|
||||
prefix = module + '@' + version + '/'
|
||||
for f in files:
|
||||
cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
|
||||
data = subprocess.check_output(cmd, cwd=repodir,
|
||||
preexec_fn=subprocess_setup)
|
||||
zf.writestr(prefix + f, data)
|
||||
|
||||
@@ -3455,3 +3455,92 @@ class GoModTest(FetcherTest):
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
|
||||
|
||||
class GoModGitTest(FetcherTest):
|
||||
|
||||
@skipIfNoNetwork()
|
||||
def test_gomodgit_url_repo(self):
|
||||
urls = ['gomodgit://golang.org/x/net;version=v0.9.0;'
|
||||
'repo=go.googlesource.com/net;'
|
||||
'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1']
|
||||
|
||||
fetcher = bb.fetch2.Fetch(urls, self.d)
|
||||
ud = fetcher.ud[urls[0]]
|
||||
self.assertEqual(ud.host, 'go.googlesource.com')
|
||||
self.assertEqual(ud.path, '/net')
|
||||
self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0'])
|
||||
self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1')
|
||||
|
||||
fetcher.download()
|
||||
self.assertTrue(os.path.exists(ud.localpath))
|
||||
|
||||
fetcher.unpack(self.unpackdir)
|
||||
vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
|
||||
self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca')))
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip')))
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')))
|
||||
|
||||
@skipIfNoNetwork()
|
||||
def test_gomodgit_url_subdir(self):
|
||||
urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
|
||||
'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;'
|
||||
'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3']
|
||||
|
||||
fetcher = bb.fetch2.Fetch(urls, self.d)
|
||||
ud = fetcher.ud[urls[0]]
|
||||
self.assertEqual(ud.host, 'github.com')
|
||||
self.assertEqual(ud.path, '/Azure/azure-sdk-for-go')
|
||||
self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob')
|
||||
self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'])
|
||||
self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3')
|
||||
|
||||
fetcher.download()
|
||||
self.assertTrue(os.path.exists(ud.localpath))
|
||||
|
||||
fetcher.unpack(self.unpackdir)
|
||||
vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
|
||||
self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3')))
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
|
||||
|
||||
@skipIfNoNetwork()
|
||||
def test_gomodgit_url_srcrev_var(self):
|
||||
urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0']
|
||||
self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951')
|
||||
|
||||
fetcher = bb.fetch2.Fetch(urls, self.d)
|
||||
ud = fetcher.ud[urls[0]]
|
||||
self.assertEqual(ud.host, 'gopkg.in')
|
||||
self.assertEqual(ud.path, '/ini.v1')
|
||||
self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0'])
|
||||
self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951')
|
||||
|
||||
fetcher.download()
|
||||
fetcher.unpack(self.unpackdir)
|
||||
vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
|
||||
self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
|
||||
|
||||
@skipIfNoNetwork()
|
||||
def test_gomodgit_url_no_go_mod_in_module(self):
|
||||
urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;'
|
||||
'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951']
|
||||
|
||||
fetcher = bb.fetch2.Fetch(urls, self.d)
|
||||
ud = fetcher.ud[urls[0]]
|
||||
self.assertEqual(ud.host, 'gopkg.in')
|
||||
self.assertEqual(ud.path, '/ini.v1')
|
||||
self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0'])
|
||||
self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951')
|
||||
|
||||
fetcher.download()
|
||||
fetcher.unpack(self.unpackdir)
|
||||
vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
|
||||
self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
|
||||
|
||||
Reference in New Issue
Block a user