bitbake: toaster tests: enable url check test

Integrate the HTML5 validation as a test instead of
calling a separate script. This enables us to get the
HTML5 validation report as part of patch-level
testing.

gitignore the cache directory created by the http client

(Bitbake rev: 931caab56301876cb8632b289835c2545a096ef6)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/toaster/contrib/tts/config.py

@@ -22,6 +22,7 @@
 # everything that would be a global variable goes here
 
 import os, sys, logging
+import socket
 
 LOGDIR = "log"
 SETTINGS_FILE = os.path.join(os.path.dirname(__file__), "settings.json")
@@ -29,6 +30,26 @@ TEST_DIR_NAME = "tts_testdir"
 
 OWN_PID = os.getpid()
 
+W3C_VALIDATOR = "http://icarus.local/w3c-validator/check?doctype=HTML5&uri="
+
+#TODO assign port dynamically
+TOASTER_PORT=56789
+
+#we parse the w3c URL to know where to connect
+
+import urlparse
+
+def get_public_ip():
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    p = urlparse.urlparse("http://icarus.local/w3c-validator/check?doctype=HTML5&uri=")
+    s.connect(( p.netloc, 80 if p.port is None else p.port))
+    hn = s.getsockname()[0]
+    s.close()
+    return hn
+
+TOASTER_BASEURL="http://%s:%d/" % (get_public_ip(), TOASTER_PORT)
+
+
 OWN_EMAIL_ADDRESS = "Toaster Testing Framework <alexandru.damian@intel.com>"
 REPORT_EMAIL_ADDRESS = "alexandru.damian@intel.com"
 

bitbake/lib/toaster/contrib/tts/runner.py

@@ -181,7 +181,7 @@ if __name__ == "__main__":
     no_failures = 1
     try:
         if options.testdir is not None and os.path.exists(options.testdir):
-            testdir = options.testdir
+            testdir = os.path.abspath(options.testdir)
             config.logger.info("No checkout, using %s" % testdir)
         else:
             need_cleanup = True

bitbake/lib/toaster/contrib/tts/tests.py

@@ -26,6 +26,9 @@
 import unittest
 from shellutils import *
 
+import pexpect
+import sys, os, signal, time
+
 class TestPyCompilable(unittest.TestCase):
     ''' Verifies that all Python files are syntactically correct '''
     def test_compile_file(self):
@@ -44,14 +47,62 @@ class TestPySystemStart(unittest.TestCase):
 
     def test_start_interactive_mode(self):
         try:
-            run_shell_cmd("bash -c 'source %s/oe-init-build-env && source toaster start webport=56789 && source toaster stop'" % config.testdir, config.testdir)
+            run_shell_cmd("bash -c 'source %s/oe-init-build-env && source toaster start webport=%d && source toaster stop'" % (config.testdir, config.TOASTER_PORT), config.testdir)
         except ShellCmdException as e:
             self.fail("Failed starting interactive mode: %s" % (e))
 
     def test_start_managed_mode(self):
         try:
-            run_shell_cmd("./poky/bitbake/bin/toaster webport=56789 & sleep 10 && curl http://localhost:56789/ && kill -2 %1")
+            run_shell_cmd("%s/bitbake/bin/toaster webport=%d nobrowser & sleep 10 && curl http://localhost:%d/ && kill -2 %1" % (config.testdir, config.TOASTER_PORT, config.TOASTER_PORT), config.testdir)
             pass
         except ShellCmdException as e:
             self.fail("Failed starting managed mode: %s" % (e))
 
+class TestHTML5Compliance(unittest.TestCase):
+    def setUp(self):
+        self.origdir = os.getcwd()
+        self.crtdir = os.path.dirname(config.testdir)
+        os.chdir(self.crtdir)
+        if not os.path.exists(os.path.join(self.crtdir, "toaster.sqlite")):
+            run_shell_cmd("%s/bitbake/lib/toaster/manage.py syncdb --noinput" % config.testdir)
+            run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate orm" % config.testdir)
+            run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate bldcontrol" % config.testdir)
+            run_shell_cmd("%s/bitbake/lib/toaster/manage.py loadconf %s/meta-yocto/conf/toasterconf.json" % (config.testdir, config.testdir))
+
+            setup = pexpect.spawn("%s/bitbake/lib/toaster/manage.py checksettings" % config.testdir)
+            setup.logfile = sys.stdout
+            setup.expect(r".*or type the full path to a different directory: ")
+            setup.sendline('')
+            setup.sendline('')
+            setup.expect(r".*or type the full path to a different directory: ")
+            setup.sendline('')
+            setup.expect(r"Enter your option: ")
+            setup.sendline('0')
+
+        self.child = pexpect.spawn("%s/bitbake/bin/toaster webport=%d nobrowser" % (config.testdir, config.TOASTER_PORT))
+        self.child.logfile=sys.stdout
+        self.child.expect("Toaster is now running. You can stop it with Ctrl-C")
+
+    def test_html5_compliance(self):
+        import urllist, urlcheck
+        results = {}
+        for url in urllist.URLS:
+            results[url] = urlcheck.validate_html5(config.TOASTER_BASEURL + url)
+
+        failed = []
+        for url in results:
+            if results[url][1] != 0:
+                failed.append((url, results[url]))
+
+
+        self.assertTrue(len(failed)== 0, "Not all URLs validate: \n%s " % "\n".join(map(lambda x: "".join(str(x)),failed)))
+
+        #(config.TOASTER_BASEURL + url, status, errors, warnings))
+
+    def tearDown(self):
+        while self.child.isalive():
+            self.child.kill(signal.SIGINT)
+            time.sleep(1)
+        os.chdir(self.origdir)
+#        if os.path.exists(os.path.join(self.crtdir, "toaster.sqlite")):
+#            os.remove(os.path.join(self.crtdir, "toaster.sqlite"))

bitbake/lib/toaster/contrib/tts/urlcheck.py

@@ -7,38 +7,35 @@ import time
 import config
 import urllist
 
-# TODO: spawn server here
-BASEURL="http://localhost:8000/"
-
-#def print_browserlog(url):
-#    driver = webdriver.Firefox()
-#    driver.get(url)
-#    body = driver.find_element_by_tag_name("body")
-#    body.send_keys(Keys.CONTROL + 't')
-#    for i in driver.get_log('browser'):
-#        print(i)
-#    driver.close()
-
 
 # TODO: turn to a test
-def validate_html(url):
+def validate_html5(url):
     h = httplib2.Http(".cache")
+    status = "Failed"
+    errors = -1
+    warnings = -1
+
     # TODO: the w3c-validator must be a configurable setting
-    urlrequest = "http://icarus.local/w3c-validator/check?doctype=HTML5&uri="+url
+    urlrequest = config.W3C_VALIDATOR+url
     try:
         resp, content = h.request(urlrequest, "HEAD")
-        if resp['x-w3c-validator-status'] == "Abort":
-            config.logger.error("FAILed call %s" % url)
-        else:
-            config.logger.error("url %s is %s\terrors %s warnings %s (check at %s)" % (url, resp['x-w3c-validator-status'], resp['x-w3c-validator-errors'], resp['x-w3c-validator-warnings'], urlrequest))
+        if resp['x-w3c-validator-status'] != "Abort":
+            status = resp['x-w3c-validator-status']
+            errors = int(resp['x-w3c-validator-errors'])
+            warnings = int(resp['x-w3c-validator-warnings'])
     except Exception as e:
         config.logger.warn("Failed validation call: %s" % e.__str__())
-
-    print("done %s" % url)
+    return (status, errors, warnings)
 
 if __name__ == "__main__":
+    print("Testing %s with %s" % (config.TOASTER_BASEURL, config.W3C_VALIDATOR))
+
+    def print_validation(url):
+        status, errors, warnings = validate_html5(url)
+        config.logger.error("url %s is %s\terrors %s warnings %s (check at %s)" % (url, status, errors, warnings, config.W3C_VALIDATOR+url))
+
     if len(sys.argv) > 1:
-        validate_html(sys.argv[1])
+        print_validation(sys.argv[1])
     else:
         for url in urllist.URLS:
-            validate_html(BASEURL+url)
+            print_validation(config.TOASTER_BASEURL+url)