[pmachines.git] / tests / test_functional.py

from pathlib import Path
from datetime import datetime
from itertools import product
from logging import info
import sys
if '' in sys.path: sys.path.remove('')
sys.path.append(str(Path(__file__).parent.parent.parent))
from unittest import TestCase
from shutil import rmtree, copy
from time import sleep
from os import system, remove, environ
from os.path import exists, basename, join
from glob import glob
from subprocess import PIPE, run, Popen
from panda3d.core import Filename
from ya2.build.build import exec_cmd, _branch, _version


class FunctionalTests(TestCase):

    def __clean(self):
        paths = [
            'tests/functional',
            str(Path.home()) + '/.local/share/pmachines/tests/functional/',
            str(Path.home()) + '/.var/app/it.ya2.Pmachines/data/pmachines/tests/functional/',
            str(Path.home()) + '/.wine/drive_c/users/flavio/AppData/Local/pmachines/tests/functional/']
        for path in paths:
            rmtree(path, ignore_errors=True)
        dirs = [
            Filename().get_user_appdata_directory(),
            '/home/flavio/.var/app/it.ya2.Pmachines/data',
            '/home/flavio/.wine/drive_c/users/flavio/AppData/Local']
        files = [
            'pmachines/options.ini',
            'pmachines/obs_version.txt']
        for dir_file in product(dirs, files):
            _file = join(*dir_file)
            if exists(_file):
                remove(_file)
                info('%s removed' % _file)
            else:
                info('%s does not exist' % _file)
        opt_ini = str(Path.home()) + '/builders/pmachines_builder/pmachines/options.ini'
        if exists(opt_ini):
            remove(opt_ini)
        system('pkill -f "pmachines.exe"')

    def __awake(self):
        system('xdotool key shift')

    def setUp(self):
        self.__clean()

    def tearDown(self):
        pass  # self.__clean()

    def test_ref(self):
        info('test_ref')
        path = 'pmachines/tests/functional_ref_%s/*.png' % _branch()
        files = glob(join(Filename().get_user_appdata_directory(), path))
        self.assertGreater(len(files), 1)

    def __similar_images(self, ref_img, tst_img):
        cmd = 'magick compare -metric NCC %s %s diff.png' % (ref_img, tst_img)
        res = exec_cmd(cmd, False)
        if exists('diff.png'): remove('diff.png')
        print('compare %s %s: %s' % (ref_img, tst_img, res))
        return float(res) > .8

    def __test_template(self, app_cmd, test_cmds, path):
        if environ.get('FUNCTIONAL') != '1':
            self.skipTest('skipped functional tests')
        self.__clean()
        self.__awake()
        system('amixer sset Master 0%')
        #ret = system(cmd)
        #proc = run(cmd, shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True, check=check)
        #return proc.stdout.strip() + proc.stderr.strip()
        for test_cmd in test_cmds:
            sleep(8)
            info('launching ' + app_cmd)
            print('launching ' + app_cmd)
            p_app = Popen(app_cmd, shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True)
            sleep(30)
            info('launching ' + test_cmd)
            print('launching ' + test_cmd)
            p_test = Popen(test_cmd, shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True)
            p_app_out, p_app_err = p_app.communicate()
            t_out, t_err = p_test.communicate()
            info('output (%s): %s' % (app_cmd, p_app_out))
            info('error (%s): %s' % (app_cmd, p_app_err))
            info('output (%s): %s' % (test_cmd, t_out))
            info('error (%s): %s' % (test_cmd, t_err))
            print('output (%s): %s' % (app_cmd, p_app_out))
            print('error (%s): %s' % (app_cmd, p_app_err))
            print('output (%s): %s' % (test_cmd, t_out))
            print('error (%s): %s' % (test_cmd, t_err))
            self.assertEqual(p_app.returncode, 0, 'error while executing ' + app_cmd)
            self.assertEqual(p_test.returncode, 0, 'error while executing ' + test_cmd)
        files = glob(str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/*.png' % _branch())
        self.assertGreater(len(files), 1)
        for fname in files:
            self.assertTrue(exists(path + basename(fname)), '%s does not exist' % (path + basename(fname)))
            similar = self.__similar_images(
                str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/' % _branch() + basename(fname),
                path + basename(fname)),
            'error while comparing %s and %s' % (
                str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/' % _branch() + basename(fname),
                path + basename(fname))
            if not similar:
                timestamp = datetime.now().strftime('%y%m%d%H%M%S%f')
                copy(path + basename(fname), '~/Desktop/' + basename(fname)[:-4] + timestamp + '.png')
            self.assertTrue(similar)

    def test_code(self):
        info('test_code')
        self.__test_template(
            '/home/flavio/venv/bin/python main.py --functional-test',
            ['/home/flavio/venv/bin/python -m tests.functional_test.py 1',
             'timeout 7200s ~/venv/bin/python -m tests.functional_test.py 2'],
            str(Path.home()) + '/.local/share/pmachines/tests/functional/')

    def test_appimage(self):
        info('test_appimage')
        bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
        bld_branch = '' if bld_branch == 'stable' else ('-' + bld_branch)
        self.__test_template(
            'timeout 7200s ./dist/Pmachines%s-x86_64.AppImage --functional-test' % bld_branch,
            ['timeout 7200s ~/venv/bin/python -m tests.functional_test.py 1',
             'timeout 7200s ~/venv/bin/python -m tests.functional_test.py 2'],
            str(Path.home()) + '/.local/share/pmachines/tests/functional/')

    # def test_flatpak(self):
    #     info('test_flatpak')
    #     if environ.get('FUNCTIONALPOST') != '1':
    #         self.skipTest('skipped functional-post tests')
    #     bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
    #     cmd = 'flatpak update --user -y it.ya2.Pmachines//%s' % bld_branch
    #     info('executing: %s' % cmd)
    #     #system(cmd)
    #     fout = exec_cmd(cmd)
    #     info('executed: %s' % cmd)
    #     info(fout)
    #     self.__test_template(
    #         'timeout 720s flatpak run it.ya2.Pmachines//%s --functional-test & '
    #         'timeout 720s ~/venv/bin/python -m tests.functional_test.py 1; sleep 5; '
    #         'timeout 720s flatpak run it.ya2.Pmachines//%s --functional-test & ' % (bld_branch, bld_branch) +
    #         'timeout 720s ~/venv/bin/python -m tests.functional_test.py 2',
    #         str(Path.home()) + '/.var/app/it.ya2.Pmachines/data/pmachines/tests/functional/')

    def __update_itchio(self):
        system('/home/flavio/.itch/itch')
        sleep(5)
        system('xdotool mousemove 860 620')
        sleep(1)
        system('xdotool click 1')
        sleep(300)
        system('killall itch')

    def test_itchio(self):
        info('test_itchio')
        if environ.get('FUNCTIONALPOST') != '1':
            self.skipTest('skipped functional-post tests')
        if _branch() != 'master':
            return
        self.__update_itchio()
        self.__test_template(
            'timeout 7200s /home/flavio/.config/itch/apps/pmachines/pmachines --functional-test',
            ['timeout 7200s ~/venv/bin/python -m tests.functional_test.py 1',
             'timeout 7200s ~/venv/bin/python -m tests.functional_test.py 2'],
            str(Path.home()) + '/.local/share/pmachines/tests/functional/')

    def test_windows(self):
        info('test_windows')
        system('pkill -f "pmachines.exe"')
        abspath = str(Path(__file__).parent.parent) + '/build/win_amd64/pmachines.exe'
        self.__test_template(
            'timeout 7200s wine %s --functional-test' % abspath,
            ['timeout 7200s ~/venv/bin/python -m tests.functional_test.py 1',
             'timeout 7200s ~/venv/bin/python -m tests.functional_test.py 2'],
            str(Path.home()) + '/.wine/drive_c/users/flavio/AppData/Local/pmachines/tests/functional/')

    def test_versions(self):
        info('test_versions')
        if environ.get('FUNCTIONAL') != '1':
            self.skipTest('skipped functional tests')
        bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
        with open('/home/flavio/builders/pmachines_builder/last_bld.txt') as f:
            lines = f.readlines()
        for line in lines:
            if line.strip().split()[0] == _branch():
                commit = line.strip().split()[1][:7]
        __ver = _version()
        if _branch() == 'stable':
            with open('/home/flavio/builders/pmachines_builder/pmachines/assets/version.txt') as fver:
                __ver = fver.read().strip() + '-'
        exp = '%s-%s' % (__ver, commit)
        cmds = [
            ('timeout 7200s ./build/manylinux2010_x86_64/pmachines --version', str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt'),
            ('timeout 7200s ./dist/Pmachines-%s-x86_64.AppImage --version' % bld_branch, str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt'),
            ('timeout 7200s wine ./build/win_amd64/pmachines.exe --version', '/home/flavio/.wine/drive_c/users/flavio/AppData/Local/pmachines/obs_version.txt')
        ]
        if environ.get('FUNCTIONALPOST') == '1':
            if _branch() == 'master':
                self.__update_itchio()
                cmds += [('timeout 7200s /home/flavio/.config/itch/apps/pmachines/pmachines --version', str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt')]
            # cmds += [('timeout 7200s flatpak run it.ya2.Pmachines//%s --version' % bld_branch, '/home/flavio/.var/app/it.ya2.Pmachines/data/pmachines/obs_version.txt')]
        # info('executing flatpak update --user -y it.ya2.Pmachines//%s' % bld_branch)
        # #system('flatpak update -y --user it.ya2.Pmachines//%s' % bld_branch)
        # fout = exec_cmd('flatpak update -y --user it.ya2.Pmachines//%s' % bld_branch)
        # info('executed flatpak update --user -y it.ya2.Pmachines//%s' % bld_branch)
        # info(fout)
        for cmd in cmds:
            if exists(cmd[1]):
                remove(cmd[1])
            info('launching %s' % cmd[0])
            exec_cmd(cmd[0])
            with open(cmd[1]) as f:
                obs = f.read().strip()
            self.assertEqual(obs, exp, 'during ' + cmd[0])
Commit	Line	Data
0f9675ba	1	from pathlib import Path
8ce16d6c	2	from datetime import datetime
0f9675ba FC	3	from itertools import product
	4	from logging import info
	5	import sys
	6	if '' in sys.path: sys.path.remove('')
	7	sys.path.append(str(Path(__file__).parent.parent.parent))
	8	from unittest import TestCase
8ce16d6c	9	from shutil import rmtree, copy
0f9675ba FC	10	from time import sleep
	11	from os import system, remove, environ
	12	from os.path import exists, basename, join
	13	from glob import glob
a33967c7	14	from subprocess import PIPE, run, Popen
0f9675ba	15	from panda3d.core import Filename
ea71df6e	16	from ya2.build.build import exec_cmd, _branch, _version
0f9675ba FC	17
	18
	19	class FunctionalTests(TestCase):
	20
	21	def __clean(self):
	22	paths = [
	23	'tests/functional',
edeef6f9 FC	24	str(Path.home()) + '/.local/share/pmachines/tests/functional/',
	25	str(Path.home()) + '/.var/app/it.ya2.Pmachines/data/pmachines/tests/functional/',
	26	str(Path.home()) + '/.wine/drive_c/users/flavio/AppData/Local/pmachines/tests/functional/']
0f9675ba FC	27	for path in paths:
	28	rmtree(path, ignore_errors=True)
	29	dirs = [
	30	Filename().get_user_appdata_directory(),
edeef6f9	31	'/home/flavio/.var/app/it.ya2.Pmachines/data',
68f6c184	32	'/home/flavio/.wine/drive_c/users/flavio/AppData/Local']
0f9675ba	33	files = [
edeef6f9 FC	34	'pmachines/options.ini',
edeef6f9 FC	35	'pmachines/obs_version.txt']
0f9675ba FC	36	for dir_file in product(dirs, files):
	37	_file = join(*dir_file)
	38	if exists(_file):
	39	remove(_file)
	40	info('%s removed' % _file)
	41	else:
	42	info('%s does not exist' % _file)
2fbba335 FC	43	opt_ini = str(Path.home()) + '/builders/pmachines_builder/pmachines/options.ini'
	44	if exists(opt_ini):
	45	remove(opt_ini)
edeef6f9	46	system('pkill -f "pmachines.exe"')
0f9675ba FC	47
	48	def __awake(self):
	49	system('xdotool key shift')
	50
	51	def setUp(self):
	52	self.__clean()
	53
	54	def tearDown(self):
8ce16d6c	55	pass # self.__clean()
0f9675ba FC	56
0f9675ba FC	57	def test_ref(self):
edeef6f9 FC	58	info('test_ref')
edeef6f9 FC	59	path = 'pmachines/tests/functional_ref_%s/*.png' % _branch()
0f9675ba FC	60	files = glob(join(Filename().get_user_appdata_directory(), path))
	61	self.assertGreater(len(files), 1)
	62
	63	def __similar_images(self, ref_img, tst_img):
	64	cmd = 'magick compare -metric NCC %s %s diff.png' % (ref_img, tst_img)
df4f85fc	65	res = exec_cmd(cmd, False)
0f9675ba FC	66	if exists('diff.png'): remove('diff.png')
0f9675ba FC	67	print('compare %s %s: %s' % (ref_img, tst_img, res))
8ce16d6c	68	return float(res) > .8
0f9675ba	69
e5b1e7e0	70	def __test_template(self, app_cmd, test_cmds, path):
0f9675ba FC	71	if environ.get('FUNCTIONAL') != '1':
	72	self.skipTest('skipped functional tests')
	73	self.__clean()
	74	self.__awake()
	75	system('amixer sset Master 0%')
a33967c7 FC	76	#ret = system(cmd)
	77	#proc = run(cmd, shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True, check=check)
	78	#return proc.stdout.strip() + proc.stderr.strip()
e5b1e7e0 FC	79	for test_cmd in test_cmds:
	80	sleep(8)
	81	info('launching ' + app_cmd)
	82	print('launching ' + app_cmd)
	83	p_app = Popen(app_cmd, shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True)
	84	sleep(30)
	85	info('launching ' + test_cmd)
	86	print('launching ' + test_cmd)
	87	p_test = Popen(test_cmd, shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True)
	88	p_app_out, p_app_err = p_app.communicate()
	89	t_out, t_err = p_test.communicate()
	90	info('output (%s): %s' % (app_cmd, p_app_out))
	91	info('error (%s): %s' % (app_cmd, p_app_err))
	92	info('output (%s): %s' % (test_cmd, t_out))
	93	info('error (%s): %s' % (test_cmd, t_err))
	94	print('output (%s): %s' % (app_cmd, p_app_out))
	95	print('error (%s): %s' % (app_cmd, p_app_err))
	96	print('output (%s): %s' % (test_cmd, t_out))
	97	print('error (%s): %s' % (test_cmd, t_err))
	98	self.assertEqual(p_app.returncode, 0, 'error while executing ' + app_cmd)
	99	self.assertEqual(p_test.returncode, 0, 'error while executing ' + test_cmd)
ad5ef398 FC	100	files = glob(str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/*.png' % _branch())
ad5ef398 FC	101	self.assertGreater(len(files), 1)
0f9675ba	102	for fname in files:
ad5ef398	103	self.assertTrue(exists(path + basename(fname)), '%s does not exist' % (path + basename(fname)))
8ce16d6c FC	104	similar = self.__similar_images(
	105	str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/' % _branch() + basename(fname),
	106	path + basename(fname)),
	107	'error while comparing %s and %s' % (
	108	str(Path.home()) + '/.local/share/pmachines/tests/functional_ref_%s/' % _branch() + basename(fname),
	109	path + basename(fname))
	110	if not similar:
	111	timestamp = datetime.now().strftime('%y%m%d%H%M%S%f')
	112	copy(path + basename(fname), '~/Desktop/' + basename(fname)[:-4] + timestamp + '.png')
	113	self.assertTrue(similar)
0f9675ba	114
ad5ef398 FC	115	def test_code(self):
	116	info('test_code')
	117	self.__test_template(
e5b1e7e0 FC	118	'/home/flavio/venv/bin/python main.py --functional-test',
	119	['/home/flavio/venv/bin/python -m tests.functional_test.py 1',
	120	'timeout 7200s ~/venv/bin/python -m tests.functional_test.py 2'],
ad5ef398	121	str(Path.home()) + '/.local/share/pmachines/tests/functional/')
edeef6f9	122
1e689176 FC	123	def test_appimage(self):
	124	info('test_appimage')
	125	bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
	126	bld_branch = '' if bld_branch == 'stable' else ('-' + bld_branch)
	127	self.__test_template(
a33967c7	128	'timeout 7200s ./dist/Pmachines%s-x86_64.AppImage --functional-test' % bld_branch,
e5b1e7e0 FC	129	['timeout 7200s ~/venv/bin/python -m tests.functional_test.py 1',
e5b1e7e0 FC	130	'timeout 7200s ~/venv/bin/python -m tests.functional_test.py 2'],
1e689176	131	str(Path.home()) + '/.local/share/pmachines/tests/functional/')
edeef6f9	132
e9be18cf FC	133	# def test_flatpak(self):
	134	# info('test_flatpak')
	135	# if environ.get('FUNCTIONALPOST') != '1':
	136	# self.skipTest('skipped functional-post tests')
	137	# bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
	138	# cmd = 'flatpak update --user -y it.ya2.Pmachines//%s' % bld_branch
	139	# info('executing: %s' % cmd)
	140	# #system(cmd)
	141	# fout = exec_cmd(cmd)
	142	# info('executed: %s' % cmd)
	143	# info(fout)
	144	# self.__test_template(
	145	# 'timeout 720s flatpak run it.ya2.Pmachines//%s --functional-test & '
	146	# 'timeout 720s ~/venv/bin/python -m tests.functional_test.py 1; sleep 5; '
	147	# 'timeout 720s flatpak run it.ya2.Pmachines//%s --functional-test & ' % (bld_branch, bld_branch) +
	148	# 'timeout 720s ~/venv/bin/python -m tests.functional_test.py 2',
	149	# str(Path.home()) + '/.var/app/it.ya2.Pmachines/data/pmachines/tests/functional/')
addec9c9	150
c8e3e941 FC	151	def __update_itchio(self):
	152	system('/home/flavio/.itch/itch')
	153	sleep(5)
	154	system('xdotool mousemove 860 620')
	155	sleep(1)
	156	system('xdotool click 1')
	157	sleep(300)
	158	system('killall itch')
	159
	160	def test_itchio(self):
	161	info('test_itchio')
	162	if environ.get('FUNCTIONALPOST') != '1':
	163	self.skipTest('skipped functional-post tests')
	164	if _branch() != 'master':
	165	return
	166	self.__update_itchio()
	167	self.__test_template(
a33967c7	168	'timeout 7200s /home/flavio/.config/itch/apps/pmachines/pmachines --functional-test',
e5b1e7e0 FC	169	['timeout 7200s ~/venv/bin/python -m tests.functional_test.py 1',
e5b1e7e0 FC	170	'timeout 7200s ~/venv/bin/python -m tests.functional_test.py 2'],
c8e3e941	171	str(Path.home()) + '/.local/share/pmachines/tests/functional/')
addec9c9	172
2e5f0efc FC	173	def test_windows(self):
	174	info('test_windows')
	175	system('pkill -f "pmachines.exe"')
	176	abspath = str(Path(__file__).parent.parent) + '/build/win_amd64/pmachines.exe'
	177	self.__test_template(
a33967c7	178	'timeout 7200s wine %s --functional-test' % abspath,
e5b1e7e0 FC	179	['timeout 7200s ~/venv/bin/python -m tests.functional_test.py 1',
e5b1e7e0 FC	180	'timeout 7200s ~/venv/bin/python -m tests.functional_test.py 2'],
2e5f0efc	181	str(Path.home()) + '/.wine/drive_c/users/flavio/AppData/Local/pmachines/tests/functional/')
edeef6f9	182
8ce16d6c FC	183	def test_versions(self):
	184	info('test_versions')
	185	if environ.get('FUNCTIONAL') != '1':
	186	self.skipTest('skipped functional tests')
	187	bld_branch = {'master': 'alpha', 'rc': 'rc', 'stable': 'stable'}[_branch()]
	188	with open('/home/flavio/builders/pmachines_builder/last_bld.txt') as f:
	189	lines = f.readlines()
	190	for line in lines:
	191	if line.strip().split()[0] == _branch():
	192	commit = line.strip().split()[1][:7]
ea71df6e	193	__ver = _version()
8ce16d6c FC	194	if _branch() == 'stable':
8ce16d6c FC	195	with open('/home/flavio/builders/pmachines_builder/pmachines/assets/version.txt') as fver:
a7f2fc93 FC	196	__ver = fver.read().strip() + '-'
a7f2fc93 FC	197	exp = '%s-%s' % (__ver, commit)
8ce16d6c	198	cmds = [
a33967c7 FC	199	('timeout 7200s ./build/manylinux2010_x86_64/pmachines --version', str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt'),
	200	('timeout 7200s ./dist/Pmachines-%s-x86_64.AppImage --version' % bld_branch, str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt'),
	201	('timeout 7200s wine ./build/win_amd64/pmachines.exe --version', '/home/flavio/.wine/drive_c/users/flavio/AppData/Local/pmachines/obs_version.txt')
c8e3e941 FC	202	]
	203	if environ.get('FUNCTIONALPOST') == '1':
	204	if _branch() == 'master':
	205	self.__update_itchio()
a33967c7 FC	206	cmds += [('timeout 7200s /home/flavio/.config/itch/apps/pmachines/pmachines --version', str(Filename.get_user_appdata_directory()) + '/pmachines/obs_version.txt')]
a33967c7 FC	207	# cmds += [('timeout 7200s flatpak run it.ya2.Pmachines//%s --version' % bld_branch, '/home/flavio/.var/app/it.ya2.Pmachines/data/pmachines/obs_version.txt')]
e9be18cf FC	208	# info('executing flatpak update --user -y it.ya2.Pmachines//%s' % bld_branch)
	209	# #system('flatpak update -y --user it.ya2.Pmachines//%s' % bld_branch)
	210	# fout = exec_cmd('flatpak update -y --user it.ya2.Pmachines//%s' % bld_branch)
	211	# info('executed flatpak update --user -y it.ya2.Pmachines//%s' % bld_branch)
	212	# info(fout)
8ce16d6c FC	213	for cmd in cmds:
	214	if exists(cmd[1]):
	215	remove(cmd[1])
	216	info('launching %s' % cmd[0])
	217	exec_cmd(cmd[0])
	218	with open(cmd[1]) as f:
	219	obs = f.read().strip()
	220	self.assertEqual(obs, exp, 'during ' + cmd[0])