mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-09-19 18:49:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: siggen: Capture SSL environment for hashserver

Browse Source 
Now that the bitbake hash server supports SSL connections, we need to
capture a few environment variables which can affect the ability to
connect via SSL. Note that the variables are only put in place to affect
the environment while actually invoking the server

[RP: Tweak to use BB_ORIGENV as well]
[RP: Tweak to handle os.environ restore correctly]
(Bitbake rev: 0bacf6551821beb8915513b120ae672ae8eb1612)

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Joshua Watt
2024-04-12 09:57:09 -06:00
committed by Richard Purdie
parent 6bd8367aa9
commit 2ecd97fa59
1 changed files with 71 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
bitbake/lib/bb/siggen.py
View File

@@ -15,6 +15,7 @@ import difflib
import simplediff import simplediff
import json import json
import types import types
from contextlib import contextmanager
import bb.compress.zstd import bb.compress.zstd
from bb.checksum import FileChecksumCache from bb.checksum import FileChecksumCache
from bb import runqueue from bb import runqueue
@@ -28,6 +29,14 @@ hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
# The minimum version of the find_siginfo function we need # The minimum version of the find_siginfo function we need
find_siginfo_minversion = 2 find_siginfo_minversion = 2
HASHSERV_ENVVARS = [
"SSL_CERT_DIR",
"SSL_CERT_FILE",
"NO_PROXY",
"HTTPS_PROXY",
"HTTP_PROXY"
]
def check_siggen_version(siggen): def check_siggen_version(siggen):
if not hasattr(siggen, "find_siginfo_version"): if not hasattr(siggen, "find_siginfo_version"):
bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (no version found)") bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (no version found)")
@@ -537,14 +546,23 @@ class SignatureGeneratorUniHashMixIn(object):
self.unihash_exists_cache = set() self.unihash_exists_cache = set()
self.username = None self.username = None
self.password = None self.password = None
self.env = {}
origenv = data.getVar("BB_ORIGENV")
for e in HASHSERV_ENVVARS:
value = data.getVar(e)
if not value and origenv:
value = origenv.getVar(e)
if value:
self.env[e] = value
super().__init__(data) super().__init__(data)
def get_taskdata(self): def get_taskdata(self):
return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password) + super().get_taskdata() return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env) + super().get_taskdata()
def set_taskdata(self, data): def set_taskdata(self, data):
self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password = data[:6] self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env = data[:7]
super().set_taskdata(data[6:]) super().set_taskdata(data[7:])
def get_hashserv_creds(self): def get_hashserv_creds(self):
if self.username and self.password: if self.username and self.password:
@@ -555,15 +573,34 @@ class SignatureGeneratorUniHashMixIn(object):
return {} return {}
def client(self): @contextmanager
if getattr(self, '_client', None) is None: def _client_env(self):
self._client = hashserv.create_client(self.server, **self.get_hashserv_creds()) orig_env = os.environ.copy()
return self._client try:
for k, v in self.env.items():
os.environ[k] = v
yield
finally:
for k, v in self.env.items():
if k in orig_env:
os.environ[k] = orig_env[k]
else:
del os.environ[k]
@contextmanager
def client(self):
with self._client_env():
if getattr(self, '_client', None) is None:
self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
yield self._client
@contextmanager
def client_pool(self): def client_pool(self):
if getattr(self, '_client_pool', None) is None: with self._client_env():
self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds()) if getattr(self, '_client_pool', None) is None:
return self._client_pool self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds())
yield self._client_pool
def reset(self, data): def reset(self, data):
self.__close_clients() self.__close_clients()
@@ -574,12 +611,13 @@ class SignatureGeneratorUniHashMixIn(object):
return super().exit() return super().exit()
def __close_clients(self): def __close_clients(self):
if getattr(self, '_client', None) is not None: with self._client_env():
self._client.close() if getattr(self, '_client', None) is not None:
self._client = None self._client.close()
if getattr(self, '_client_pool', None) is not None: self._client = None
self._client_pool.close() if getattr(self, '_client_pool', None) is not None:
self._client_pool = None self._client_pool.close()
self._client_pool = None
def get_stampfile_hash(self, tid): def get_stampfile_hash(self, tid):
if tid in self.taskhash: if tid in self.taskhash:
@@ -650,11 +688,13 @@ class SignatureGeneratorUniHashMixIn(object):
if self.max_parallel <= 1 or len(uncached_query) <= 1: if self.max_parallel <= 1 or len(uncached_query) <= 1:
# No parallelism required. Make the query serially with the single client # No parallelism required. Make the query serially with the single client
uncached_result = { with self.client() as client:
key: self.client().unihash_exists(value) for key, value in uncached_query.items() uncached_result = {
} key: client.unihash_exists(value) for key, value in uncached_query.items()
}
else: else:
uncached_result = self.client_pool().unihashes_exist(uncached_query) with self.client_pool() as client_pool:
uncached_result = client_pool.unihashes_exist(uncached_query)
for key, exists in uncached_result.items(): for key, exists in uncached_result.items():
if exists: if exists:
@@ -687,10 +727,12 @@ class SignatureGeneratorUniHashMixIn(object):
if self.max_parallel <= 1 or len(queries) <= 1: if self.max_parallel <= 1 or len(queries) <= 1:
# No parallelism required. Make the query serially with the single client # No parallelism required. Make the query serially with the single client
for tid, args in queries.items(): with self.client() as client:
query_result[tid] = self.client().get_unihash(*args) for tid, args in queries.items():
query_result[tid] = client.get_unihash(*args)
else: else:
query_result = self.client_pool().get_unihashes(queries) with self.client_pool() as client_pool:
query_result = client_pool.get_unihashes(queries)
for tid, unihash in query_result.items(): for tid, unihash in query_result.items():
# In the absence of being able to discover a unique hash from the # In the absence of being able to discover a unique hash from the
@@ -785,7 +827,9 @@ class SignatureGeneratorUniHashMixIn(object):
if tid in self.extramethod: if tid in self.extramethod:
method = method + self.extramethod[tid] method = method + self.extramethod[tid]
data = self.client().report_unihash(taskhash, method, outhash, unihash, extra_data) with self.client() as client:
data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
new_unihash = data['unihash'] new_unihash = data['unihash']
if new_unihash != unihash: if new_unihash != unihash:
@@ -816,7 +860,9 @@ class SignatureGeneratorUniHashMixIn(object):
if tid in self.extramethod: if tid in self.extramethod:
method = method + self.extramethod[tid] method = method + self.extramethod[tid]
data = self.client().report_unihash_equiv(taskhash, method, wanted_unihash, extra_data) with self.client() as client:
data = client.report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data))) hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
if data is None: if data is None: