ya2 · news · projects · code · about

external testing process
[pmachines.git] / tests / test_functional.py
index a53961f0828c66fec97bd18146700e1e72af7cc9..3db67e030b30615c7946ffcaffa33a7169a22ad4 100644 (file)
@@ -1,11 +1,12 @@
 from pathlib import Path
+from datetime import datetime
 from itertools import product
 from logging import info
 import sys
 if '' in sys.path: sys.path.remove('')
 sys.path.append(str(Path(__file__).parent.parent.parent))
 from unittest import TestCase
-from shutil import rmtree
+from shutil import rmtree, copy
 from time import sleep
 from os import system, remove, environ
 from os.path import exists, basename, join
@@ -47,7 +48,7 @@ class FunctionalTests(TestCase):
         self.__clean()
 
     def tearDown(self):
-        self.__clean()
+        pass  # self.__clean()
 
     def test_ref(self):
         info('test_ref')
@@ -60,7 +61,7 @@ class FunctionalTests(TestCase):
         res = exec_cmd(cmd)
         if exists('diff.png'): remove('diff.png')
         print('compare %s %s: %s' % (ref_img, tst_img, res))
-        return float(res) > .64
+        return float(res) > .8
 
     def __test_template(self, cmd, path):
         if environ.get('FUNCTIONAL') != '1':
@@ -74,16 +75,24 @@ class FunctionalTests(TestCase):
         self.assertGreater(len(files), 1)
         for fname in files:
             self.assertTrue(exists(path + basename(fname)), '%s does not exist' % (path + basename(fname)))
-            self.assertTrue(
-                self.__similar_images(
-                    str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/' % _branch() + basename(fname),
-                    path + basename(fname)))
+            similar = self.__similar_images(
+                str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/' % _branch() + basename(fname),
+                path + basename(fname)),
+            'error while comparing %s and %s' % (
+                str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/' % _branch() + basename(fname),
+                path + basename(fname))
+            if not similar:
+                timestamp = datetime.now().strftime('%y%m%d%H%M%S%f')
+                copy(path + basename(fname), '~/Desktop/' + basename(fname)[:-4] + timestamp + '.png')
+            self.assertTrue(similar)
 
     def test_code(self):
         info('test_code')
         self.__test_template(
-            '~/venv/bin/python main.py --functional-test 1 ; '
-            '~/venv/bin/python main.py --functional-test 2',
+            'timeout 720s ~/venv/bin/python main.py --functional-test & '
+            'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 1; sleep 5; '
+            'timeout 720s ~/venv/bin/python main.py --functional-test & '
+            'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 2',
             str(Path.home()) + '/.local/share/pmachines/tests/functional/')
 
     def test_appimage(self):
@@ -91,28 +100,32 @@ class FunctionalTests(TestCase):
         bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
         bld_branch = '' if bld_branch == 'stable' else ('-' + bld_branch)
         self.__test_template(
-            './dist/Pmachines%s-x86_64.AppImage --functional-test 1 ;'
-            './dist/Pmachines%s-x86_64.AppImage --functional-test 2' % (bld_branch, bld_branch),
+            'timeout 720s ./dist/Pmachines%s-x86_64.AppImage --functional-test & '
+            'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 1; sleep 5; '
+            'timeout 720s ./dist/Pmachines%s-x86_64.AppImage --functional-test & ' % (bld_branch, bld_branch) +
+            'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 2',
             str(Path.home()) + '/.local/share/pmachines/tests/functional/')
 
-    # def test_flatpak(self):
-    #     info('test_flatpak')
-    #     if environ.get('FUNCTIONALPOST') != '1':
-    #         self.skipTest('skipped functional-post tests')
-    #     bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
-    #     cmd = 'flatpak update -y it.ya2.Pmachines//%s' % bld_branch
-    #     info('executing: %s' % cmd)
-    #     system(cmd)
-    #     info('executed: %s' % cmd)
-    #     self.__test_template(
-    #         'flatpak run it.ya2.Pmachines//%s --functional-test 1 ;'
-    #         'flatpak run it.ya2.Pmachines//%s --functional-test 2' % (bld_branch, bld_branch),
-    #         str(Path.home()) + '/.var/app/it.ya2.Pmachines/data/pmachines/tests/functional/')
+    def test_flatpak(self):
+        info('test_flatpak')
+        if environ.get('FUNCTIONALPOST') != '1':
+            self.skipTest('skipped functional-post tests')
+        bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
+        cmd = 'flatpak update -y it.ya2.Pmachines//%s' % bld_branch
+        info('executing: %s' % cmd)
+        system(cmd)
+        info('executed: %s' % cmd)
+        self.__test_template(
+            'timeout 720s flatpak run it.ya2.Pmachines//%s --functional-test & '
+            'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 1; sleep 5; '
+            'timeout 720s flatpak run it.ya2.Pmachines//%s --functional-test & ' % (bld_branch, bld_branch) +
+            'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 2',
+            str(Path.home()) + '/.var/app/it.ya2.Pmachines/data/pmachines/tests/functional/')
 
     # def __update_itchio(self):
     #     system('/home/flavio/.itch/itch')
     #     sleep(5)
-    #     system('xdotool mousemove 1280 620')
+    #     system('xdotool mousemove 860 620')
     #     sleep(1)
     #     system('xdotool click 1')
     #     sleep(300)
@@ -126,50 +139,54 @@ class FunctionalTests(TestCase):
     #         return
     #     self.__update_itchio()
     #     self.__test_template(
-    #         '/home/flavio/.config/itch/apps/pmachines/pmachines --functional-test 1 ;'
-    #         '/home/flavio/.config/itch/apps/pmachines/pmachines --functional-test 2',
+    #         'timeout 720s /home/flavio/.config/itch/apps/pmachines/pmachines --functional-test & '
+    #         'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 1; sleep 5; '
+    #         'timeout 720s /home/flavio/.config/itch/apps/pmachines/pmachines --functional-test & '
+    #         'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 2',
     #         str(Path.home()) + '/.local/share/pmachines/tests/functional/')
 
-    # def test_windows(self):
-    #     info('test_windows')
-    #     system('pkill -f "pmachines.exe"')
-    #     abspath = str(Path(__file__).parent.parent) + '/build/win_amd64/pmachines.exe'
-    #     self.__test_template(
-    #         'timeout 720s wine %s --functional-test 1 ; '
-    #         'timeout 720s wine %s --functional-test 2' % (abspath, abspath),
-    #         str(Path.home()) + '/.wine/drive_c/users/flavio/AppData/Local/pmachines/tests/functional/')
-
-    # def test_versions(self):
-    #     info('test_versions')
-    #     if environ.get('FUNCTIONAL') != '1':
-    #         self.skipTest('skipped functional tests')
-    #     bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
-    #     with open('/home/flavio/pmachines_builder/last_bld.txt') as f:
-    #         lines = f.readlines()
-    #     for line in lines:
-    #         if line.strip().split()[0] == _branch():
-    #             commit = line.strip().split()[1][:7]
-    #     _ver = ver
-    #     if _branch() == 'stable':
-    #         with open('/home/flavio/pmachines_builder/pmachines/assets/version.txt') as fver:
-    #             _ver = fver.read().strip() + '-'
-    #     exp = '%s-%s' % (_ver, commit)
-    #     cmds = [
-    #         ('./build/manylinux1_x86_64/pmachines --version', str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt'),
-    #         ('./dist/Pmachines-%s-x86_64.AppImage --version' % bld_branch, str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt'),
-    #         ('timeout 720s wine ./build/win_amd64/pmachines.exe --version', '/home/flavio/.wine/drive_c/users/flavio/AppData/Local/pmachines/obs_version.txt')
-    #         ]
-    #     if environ.get('FUNCTIONALPOST') == '1':
-    #         if _branch() == 'master':
-    #             self.__update_itchio()
-    #             cmds += [('/home/flavio/.config/itch/apps/pmachines/pmachines --version', str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt')]
-    #         cmds += [('flatpak run it.ya2.Pmachines//%s --version' % bld_branch, '/home/flavio/.var/app/it.ya2.Pmachines/data/pmachines/obs_version.txt')]
-    #     system('flatpak update -y it.ya2.Pmachines//%s' % bld_branch)
-    #     for cmd in cmds:
-    #         if exists(cmd[1]):
-    #             remove(cmd[1])
-    #         info('launching %s' % cmd[0])
-    #         exec_cmd(cmd[0])
-    #         with open(cmd[1]) as f:
-    #             obs = f.read().strip()
-    #         self.assertEqual(obs, exp)
+    #def test_windows(self):
+    #    info('test_windows')
+    #    system('pkill -f "pmachines.exe"')
+    #    abspath = str(Path(__file__).parent.parent) + '/build/win_amd64/pmachines.exe'
+    #    self.__test_template(
+    #        'timeout 720s wine %s --functional-test & '
+    #        'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 1; sleep 5; '
+    #        'timeout 720s wine %s --functional-test & ' % (abspath, abspath) +
+    #        'timeout 720s ~/venv/bin/python -m lib.tools.functional_test.py 2',
+    #        str(Path.home()) + '/.wine/drive_c/users/flavio/AppData/Local/pmachines/tests/functional/')
+
+    def test_versions(self):
+        info('test_versions')
+        if environ.get('FUNCTIONAL') != '1':
+            self.skipTest('skipped functional tests')
+        bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
+        with open('/home/flavio/builders/pmachines_builder/last_bld.txt') as f:
+            lines = f.readlines()
+        for line in lines:
+            if line.strip().split()[0] == _branch():
+                commit = line.strip().split()[1][:7]
+        _ver = ver
+        if _branch() == 'stable':
+            with open('/home/flavio/builders/pmachines_builder/pmachines/assets/version.txt') as fver:
+                _ver = fver.read().strip() + '-'
+        exp = '%s-%s' % (_ver, commit)
+        cmds = [
+            ('timeout 720s ./build/manylinux2010_x86_64/pmachines --version', str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt'),
+            ('timeout 720s ./dist/Pmachines-%s-x86_64.AppImage --version' % bld_branch, str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt'),
+            #('timeout 720s wine ./build/win_amd64/pmachines.exe --version', '/home/flavio/.wine/drive_c/users/flavio/AppData/Local/pmachines/obs_version.txt')
+            ]
+        #if environ.get('FUNCTIONALPOST') == '1':
+        #    if _branch() == 'master':
+        #        self.__update_itchio()
+        #        cmds += [('timeout 720s /home/flavio/.config/itch/apps/pmachines/pmachines --version', str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt')]
+        #    cmds += [('timeout 720s flatpak run it.ya2.Pmachines//%s --version' % bld_branch, '/home/flavio/.var/app/it.ya2.Pmachines/data/pmachines/obs_version.txt')]
+        system('flatpak update -y it.ya2.Pmachines//%s' % bld_branch)
+        for cmd in cmds:
+            if exists(cmd[1]):
+                remove(cmd[1])
+            info('launching %s' % cmd[0])
+            exec_cmd(cmd[0])
+            with open(cmd[1]) as f:
+                obs = f.read().strip()
+            self.assertEqual(obs, exp, 'during ' + cmd[0])