mirror of
https://git.yoctoproject.org/poky
synced 2026-04-29 09:32:11 +02:00
sstatesig: Move unihash siggen code to bitbake
This code is closely tied with the hash server in bitbake and also means we can't relibably test the hashserv runqueue functionality without OE metadata. Moving this to bitbake as a MixIn class makes most sense and encourages code collaboration and reuse as well as enabling easier and more accurate testing of the APIs. (From OE-Core rev: a2a9c6092d4dde706ed071b08a972d1d87184295) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
@@ -266,7 +266,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
|
||||
if error_msgs:
|
||||
bb.fatal("\n".join(error_msgs))
|
||||
|
||||
class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
|
||||
class SignatureGeneratorOEEquivHash(bb.siggen.SignatureGeneratorUniHashMixIn, SignatureGeneratorOEBasicHash):
|
||||
name = "OEEquivHash"
|
||||
|
||||
def init_rundepcheck(self, data):
|
||||
@@ -275,167 +275,6 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
|
||||
self.method = data.getVar('SSTATE_HASHEQUIV_METHOD')
|
||||
self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data)
|
||||
|
||||
def get_taskdata(self):
|
||||
return (self.server, self.method) + super().get_taskdata()
|
||||
|
||||
def set_taskdata(self, data):
|
||||
self.server, self.method = data[:2]
|
||||
super().set_taskdata(data[2:])
|
||||
|
||||
def __get_task_unihash_key(self, task):
|
||||
# TODO: The key only *needs* to be the taskhash, the task is just
|
||||
# convenient
|
||||
return '%s:%s' % (task, self.taskhash[task])
|
||||
|
||||
def get_stampfile_hash(self, task):
|
||||
if task in self.taskhash:
|
||||
# If a unique hash is reported, use it as the stampfile hash. This
|
||||
# ensures that if a task won't be re-run if the taskhash changes,
|
||||
# but it would result in the same output hash
|
||||
unihash = self.unihashes.get(self.__get_task_unihash_key(task))
|
||||
if unihash is not None:
|
||||
return unihash
|
||||
|
||||
return super().get_stampfile_hash(task)
|
||||
|
||||
def get_unihash(self, task):
|
||||
import urllib
|
||||
import json
|
||||
|
||||
taskhash = self.taskhash[task]
|
||||
|
||||
key = self.__get_task_unihash_key(task)
|
||||
|
||||
# TODO: This cache can grow unbounded. It probably only needs to keep
|
||||
# for each task
|
||||
unihash = self.unihashes.get(key)
|
||||
if unihash is not None:
|
||||
return unihash
|
||||
|
||||
# In the absence of being able to discover a unique hash from the
|
||||
# server, make it be equivalent to the taskhash. The unique "hash" only
|
||||
# really needs to be a unique string (not even necessarily a hash), but
|
||||
# making it match the taskhash has a few advantages:
|
||||
#
|
||||
# 1) All of the sstate code that assumes hashes can be the same
|
||||
# 2) It provides maximal compatibility with builders that don't use
|
||||
# an equivalency server
|
||||
# 3) The value is easy for multiple independent builders to derive the
|
||||
# same unique hash from the same input. This means that if the
|
||||
# independent builders find the same taskhash, but it isn't reported
|
||||
# to the server, there is a better chance that they will agree on
|
||||
# the unique hash.
|
||||
unihash = taskhash
|
||||
|
||||
try:
|
||||
url = '%s/v1/equivalent?%s' % (self.server,
|
||||
urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]}))
|
||||
|
||||
request = urllib.request.Request(url)
|
||||
response = urllib.request.urlopen(request)
|
||||
data = response.read().decode('utf-8')
|
||||
|
||||
json_data = json.loads(data)
|
||||
|
||||
if json_data:
|
||||
unihash = json_data['unihash']
|
||||
# A unique hash equal to the taskhash is not very interesting,
|
||||
# so it is reported it at debug level 2. If they differ, that
|
||||
# is much more interesting, so it is reported at debug level 1
|
||||
bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server))
|
||||
else:
|
||||
bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server))
|
||||
except urllib.error.URLError as e:
|
||||
bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
|
||||
except (KeyError, json.JSONDecodeError) as e:
|
||||
bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
|
||||
|
||||
self.unihashes[key] = unihash
|
||||
return unihash
|
||||
|
||||
def report_unihash(self, path, task, d):
|
||||
import urllib
|
||||
import json
|
||||
import tempfile
|
||||
import base64
|
||||
import importlib
|
||||
|
||||
taskhash = d.getVar('BB_TASKHASH')
|
||||
unihash = d.getVar('BB_UNIHASH')
|
||||
report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
|
||||
tempdir = d.getVar('T')
|
||||
fn = d.getVar('BB_FILENAME')
|
||||
key = fn + '.do_' + task + ':' + taskhash
|
||||
|
||||
# Sanity checks
|
||||
cache_unihash = self.unihashes.get(key)
|
||||
if cache_unihash is None:
|
||||
bb.fatal('%s not in unihash cache. Please report this error' % key)
|
||||
|
||||
if cache_unihash != unihash:
|
||||
bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash))
|
||||
|
||||
sigfile = None
|
||||
sigfile_name = "depsig.do_%s.%d" % (task, os.getpid())
|
||||
sigfile_link = "depsig.do_%s" % task
|
||||
|
||||
try:
|
||||
sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
|
||||
|
||||
locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
|
||||
|
||||
(module, method) = self.method.rsplit('.', 1)
|
||||
locs['method'] = getattr(importlib.import_module(module), method)
|
||||
|
||||
outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
|
||||
|
||||
try:
|
||||
url = '%s/v1/equivalent' % self.server
|
||||
task_data = {
|
||||
'taskhash': taskhash,
|
||||
'method': self.method,
|
||||
'outhash': outhash,
|
||||
'unihash': unihash,
|
||||
'owner': d.getVar('SSTATE_HASHEQUIV_OWNER')
|
||||
}
|
||||
|
||||
if report_taskdata:
|
||||
sigfile.seek(0)
|
||||
|
||||
task_data['PN'] = d.getVar('PN')
|
||||
task_data['PV'] = d.getVar('PV')
|
||||
task_data['PR'] = d.getVar('PR')
|
||||
task_data['task'] = task
|
||||
task_data['outhash_siginfo'] = sigfile.read().decode('utf-8')
|
||||
|
||||
headers = {'content-type': 'application/json'}
|
||||
|
||||
request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers)
|
||||
response = urllib.request.urlopen(request)
|
||||
data = response.read().decode('utf-8')
|
||||
|
||||
json_data = json.loads(data)
|
||||
new_unihash = json_data['unihash']
|
||||
|
||||
if new_unihash != unihash:
|
||||
bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
|
||||
else:
|
||||
bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
|
||||
except urllib.error.URLError as e:
|
||||
bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
|
||||
except (KeyError, json.JSONDecodeError) as e:
|
||||
bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
|
||||
finally:
|
||||
if sigfile:
|
||||
sigfile.close()
|
||||
|
||||
sigfile_link_path = os.path.join(tempdir, sigfile_link)
|
||||
bb.utils.remove(sigfile_link_path)
|
||||
|
||||
try:
|
||||
os.symlink(sigfile_name, sigfile_link_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Insert these classes into siggen's namespace so it can see and select them
|
||||
bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic
|
||||
|
||||
Reference in New Issue
Block a user