mirror of
https://git.yoctoproject.org/poky
synced 2026-04-04 23:02:22 +02:00
bitbake: bitbake: hashserv: Add hash equivalence reference server
Implements a reference implementation of the hash equivalence server. This server has minimal dependencies (and no dependencies outside of the standard Python library), and implements the minimum required to be a conforming hash equivalence server. [YOCTO #13030] (Bitbake rev: 1bb2ad0b44b94ee04870bf3f7dac4e663bed6e4d) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Joshua Watt
committed by
Richard Purdie
Richard Purdie
parent
cea00c1283
commit
df85dd5485
67
bitbake/bin/bitbake-hashserv
Executable file
67
bitbake/bin/bitbake-hashserv
Executable file
@@ -0,0 +1,67 @@
|
||||
#! /usr/bin/env python3
|
||||
#
|
||||
# Copyright (C) 2018 Garmin Ltd.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import argparse
|
||||
import sqlite3
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib'))
|
||||
|
||||
import hashserv
|
||||
|
||||
VERSION = "1.0.0"
|
||||
|
||||
DEFAULT_HOST = ''
|
||||
DEFAULT_PORT = 8686
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='HTTP Equivalence Reference Server. Version=%s' % VERSION)
|
||||
parser.add_argument('--address', default=DEFAULT_HOST, help='Bind address (default "%(default)s")')
|
||||
parser.add_argument('--port', type=int, default=DEFAULT_PORT, help='Bind port (default %(default)d)')
|
||||
parser.add_argument('--prefix', default='', help='HTTP path prefix (default "%(default)s")')
|
||||
parser.add_argument('--database', default='./hashserv.db', help='Database file (default "%(default)s")')
|
||||
parser.add_argument('--log', default='WARNING', help='Set logging level')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
logger = logging.getLogger('hashserv')
|
||||
|
||||
level = getattr(logging, args.log.upper(), None)
|
||||
if not isinstance(level, int):
|
||||
raise ValueError('Invalid log level: %s' % args.log)
|
||||
|
||||
logger.setLevel(level)
|
||||
console = logging.StreamHandler()
|
||||
console.setLevel(level)
|
||||
logger.addHandler(console)
|
||||
|
||||
db = sqlite3.connect(args.database)
|
||||
|
||||
server = hashserv.create_server((args.address, args.port), db, args.prefix)
|
||||
server.serve_forever()
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
ret = main()
|
||||
except Exception:
|
||||
ret = 1
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(ret)
|
||||
|
||||
@@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib
|
||||
import unittest
|
||||
try:
|
||||
import bb
|
||||
import hashserv
|
||||
import layerindexlib
|
||||
except RuntimeError as exc:
|
||||
sys.exit(str(exc))
|
||||
@@ -35,6 +36,7 @@ tests = ["bb.tests.codeparser",
|
||||
"bb.tests.parse",
|
||||
"bb.tests.persist_data",
|
||||
"bb.tests.utils",
|
||||
"hashserv.tests",
|
||||
"layerindexlib.tests.layerindexobj",
|
||||
"layerindexlib.tests.restapi",
|
||||
"layerindexlib.tests.cooker"]
|
||||
|
||||
152
bitbake/lib/hashserv/__init__.py
Normal file
152
bitbake/lib/hashserv/__init__.py
Normal file
@@ -0,0 +1,152 @@
|
||||
# Copyright (C) 2018 Garmin Ltd.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
import contextlib
|
||||
import urllib.parse
|
||||
import sqlite3
|
||||
import json
|
||||
import traceback
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger('hashserv')
|
||||
|
||||
class HashEquivalenceServer(BaseHTTPRequestHandler):
|
||||
def log_message(self, f, *args):
|
||||
logger.debug(f, *args)
|
||||
|
||||
def do_GET(self):
|
||||
try:
|
||||
p = urllib.parse.urlparse(self.path)
|
||||
|
||||
if p.path != self.prefix + '/v1/equivalent':
|
||||
self.send_error(404)
|
||||
return
|
||||
|
||||
query = urllib.parse.parse_qs(p.query, strict_parsing=True)
|
||||
method = query['method'][0]
|
||||
taskhash = query['taskhash'][0]
|
||||
|
||||
d = None
|
||||
with contextlib.closing(self.db.cursor()) as cursor:
|
||||
cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1',
|
||||
{'method': method, 'taskhash': taskhash})
|
||||
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row is not None:
|
||||
logger.debug('Found equivalent task %s', row['taskhash'])
|
||||
d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json; charset=utf-8')
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(d).encode('utf-8'))
|
||||
except:
|
||||
logger.exception('Error in GET')
|
||||
self.send_error(400, explain=traceback.format_exc())
|
||||
return
|
||||
|
||||
def do_POST(self):
|
||||
try:
|
||||
p = urllib.parse.urlparse(self.path)
|
||||
|
||||
if p.path != self.prefix + '/v1/equivalent':
|
||||
self.send_error(404)
|
||||
return
|
||||
|
||||
length = int(self.headers['content-length'])
|
||||
data = json.loads(self.rfile.read(length).decode('utf-8'))
|
||||
|
||||
with contextlib.closing(self.db.cursor()) as cursor:
|
||||
cursor.execute('''
|
||||
SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND outhash=:outhash
|
||||
ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
|
||||
created ASC
|
||||
LIMIT 1
|
||||
''', {k: data[k] for k in ('method', 'outhash', 'taskhash')})
|
||||
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row is None or row['taskhash'] != data['taskhash']:
|
||||
unihash = data['unihash']
|
||||
if row is not None:
|
||||
unihash = row['unihash']
|
||||
|
||||
insert_data = {
|
||||
'method': data['method'],
|
||||
'outhash': data['outhash'],
|
||||
'taskhash': data['taskhash'],
|
||||
'unihash': unihash,
|
||||
'created': datetime.now()
|
||||
}
|
||||
|
||||
for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
|
||||
if k in data:
|
||||
insert_data[k] = data[k]
|
||||
|
||||
cursor.execute('''INSERT INTO tasks_v1 (%s) VALUES (%s)''' % (
|
||||
', '.join(sorted(insert_data.keys())),
|
||||
', '.join(':' + k for k in sorted(insert_data.keys()))),
|
||||
insert_data)
|
||||
|
||||
logger.info('Adding taskhash %s with unihash %s', data['taskhash'], unihash)
|
||||
cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE id=:id', {'id': cursor.lastrowid})
|
||||
row = cursor.fetchone()
|
||||
|
||||
self.db.commit()
|
||||
|
||||
d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json; charset=utf-8')
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(d).encode('utf-8'))
|
||||
except:
|
||||
logger.exception('Error in POST')
|
||||
self.send_error(400, explain=traceback.format_exc())
|
||||
return
|
||||
|
||||
def create_server(addr, db, prefix=''):
|
||||
class Handler(HashEquivalenceServer):
|
||||
pass
|
||||
|
||||
Handler.prefix = prefix
|
||||
Handler.db = db
|
||||
db.row_factory = sqlite3.Row
|
||||
|
||||
with contextlib.closing(db.cursor()) as cursor:
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS tasks_v1 (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
method TEXT NOT NULL,
|
||||
outhash TEXT NOT NULL,
|
||||
taskhash TEXT NOT NULL,
|
||||
unihash TEXT NOT NULL,
|
||||
created DATETIME,
|
||||
|
||||
-- Optional fields
|
||||
owner TEXT,
|
||||
PN TEXT,
|
||||
PV TEXT,
|
||||
PR TEXT,
|
||||
task TEXT,
|
||||
outhash_siginfo TEXT
|
||||
)
|
||||
''')
|
||||
|
||||
logger.info('Starting server on %s', addr)
|
||||
return HTTPServer(addr, Handler)
|
||||
141
bitbake/lib/hashserv/tests.py
Normal file
141
bitbake/lib/hashserv/tests.py
Normal file
@@ -0,0 +1,141 @@
|
||||
#! /usr/bin/env python3
|
||||
#
|
||||
# Copyright (C) 2018 Garmin Ltd.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import unittest
|
||||
import threading
|
||||
import sqlite3
|
||||
import hashlib
|
||||
import urllib.request
|
||||
import json
|
||||
from . import create_server
|
||||
|
||||
class TestHashEquivalenceServer(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Start an in memory hash equivalence server in the background bound to
|
||||
# an ephemeral port
|
||||
db = sqlite3.connect(':memory:', check_same_thread=False)
|
||||
self.server = create_server(('localhost', 0), db)
|
||||
self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1]
|
||||
self.server_thread = threading.Thread(target=self.server.serve_forever)
|
||||
self.server_thread.start()
|
||||
|
||||
def tearDown(self):
|
||||
# Shutdown server
|
||||
s = getattr(self, 'server', None)
|
||||
if s is not None:
|
||||
self.server.shutdown()
|
||||
self.server_thread.join()
|
||||
self.server.server_close()
|
||||
|
||||
def send_get(self, path):
|
||||
url = '%s/%s' % (self.server_addr, path)
|
||||
request = urllib.request.Request(url)
|
||||
response = urllib.request.urlopen(request)
|
||||
return json.loads(response.read().decode('utf-8'))
|
||||
|
||||
def send_post(self, path, data):
|
||||
headers = {'content-type': 'application/json'}
|
||||
url = '%s/%s' % (self.server_addr, path)
|
||||
request = urllib.request.Request(url, json.dumps(data).encode('utf-8'), headers)
|
||||
response = urllib.request.urlopen(request)
|
||||
return json.loads(response.read().decode('utf-8'))
|
||||
|
||||
def test_create_hash(self):
|
||||
# Simple test that hashes can be created
|
||||
taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
|
||||
outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
|
||||
unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
|
||||
|
||||
d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash)
|
||||
self.assertIsNone(d, msg='Found unexpected task, %r' % d)
|
||||
|
||||
d = self.send_post('v1/equivalent', {
|
||||
'taskhash': taskhash,
|
||||
'method': 'TestMethod',
|
||||
'outhash': outhash,
|
||||
'unihash': unihash,
|
||||
})
|
||||
self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash')
|
||||
|
||||
def test_create_equivalent(self):
|
||||
# Tests that a second reported task with the same outhash will be
|
||||
# assigned the same unihash
|
||||
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
|
||||
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
|
||||
unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
|
||||
d = self.send_post('v1/equivalent', {
|
||||
'taskhash': taskhash,
|
||||
'method': 'TestMethod',
|
||||
'outhash': outhash,
|
||||
'unihash': unihash,
|
||||
})
|
||||
self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash')
|
||||
|
||||
# Report a different task with the same outhash. The returned unihash
|
||||
# should match the first task
|
||||
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
|
||||
unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
|
||||
d = self.send_post('v1/equivalent', {
|
||||
'taskhash': taskhash2,
|
||||
'method': 'TestMethod',
|
||||
'outhash': outhash,
|
||||
'unihash': unihash2,
|
||||
})
|
||||
self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash')
|
||||
|
||||
def test_duplicate_taskhash(self):
|
||||
# Tests that duplicate reports of the same taskhash with different
|
||||
# outhash & unihash always return the unihash from the first reported
|
||||
# taskhash
|
||||
taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
|
||||
outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
|
||||
unihash = '218e57509998197d570e2c98512d0105985dffc9'
|
||||
d = self.send_post('v1/equivalent', {
|
||||
'taskhash': taskhash,
|
||||
'method': 'TestMethod',
|
||||
'outhash': outhash,
|
||||
'unihash': unihash,
|
||||
})
|
||||
|
||||
d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash)
|
||||
self.assertEqual(d['unihash'], unihash)
|
||||
|
||||
outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
|
||||
unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
|
||||
d = self.send_post('v1/equivalent', {
|
||||
'taskhash': taskhash,
|
||||
'method': 'TestMethod',
|
||||
'outhash': outhash2,
|
||||
'unihash': unihash2
|
||||
})
|
||||
|
||||
d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash)
|
||||
self.assertEqual(d['unihash'], unihash)
|
||||
|
||||
outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
|
||||
unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603'
|
||||
d = self.send_post('v1/equivalent', {
|
||||
'taskhash': taskhash,
|
||||
'method': 'TestMethod',
|
||||
'outhash': outhash3,
|
||||
'unihash': unihash3
|
||||
})
|
||||
|
||||
d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash)
|
||||
self.assertEqual(d['unihash'], unihash)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user