From 876c432f7a581ea3839deb77f756166572ac1db7 Mon Sep 17 00:00:00 2001 From: Szczepan Zalega Date: Fri, 18 Jan 2019 11:38:38 +0100 Subject: Stress test for firmware export feature Export the firmware multiple times, and test its size. Hardcoded device set to /dev/sde1 Related: https://github.com/Nitrokey/nitrokey-app/issues/399 Signed-off-by: Szczepan Zalega --- unittest/test_storage.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/unittest/test_storage.py b/unittest/test_storage.py index 04b0581..380d42e 100644 --- a/unittest/test_storage.py +++ b/unittest/test_storage.py @@ -395,3 +395,70 @@ def test_struct_multiline_prodtest(C): fwb=info_st.FirmwareVersionInternal_u8 ) print(info) + +@pytest.mark.other +@pytest.mark.firmware +def test_export_firmware_extended_fedora29(C): + """ + Check, whether the firmware file is exported correctly, and in correct size. + Apparently, the auto-remounting side effect of the v0.46 change, is disturbing the export process. + Unmounting the UV just before the export gives the device 20/20 success rate. + Test case for issue https://github.com/Nitrokey/nitrokey-app/issues/399 + """ + + skip_if_device_version_lower_than({'S': 43}) + skip_if_not_fedora() + + import pexpect + from time import sleep + import os + exist = os.path.exists + + device = '/dev/sde1' + firmware_abs_path = '/run/media/sz/Nitrokey/firmware.bin' + pexpect.run(f'udisksctl mount -b {device}') + checks = 0 + checks_add = 0 + + if exist(firmware_abs_path): + os.remove(firmware_abs_path) + + assert not exist(firmware_abs_path) + + ATTEMPTS = 20 + for i in range(ATTEMPTS): + # if umount is disabled, success rate is 3/10, enabled: 10/10 + # pexpect.run(f'udisksctl unmount -b {device}') + assert C.NK_export_firmware(DefaultPasswords.ADMIN) == DeviceErrorCode.STATUS_OK + pexpect.run(f'udisksctl mount -b {device}') + sleep(1) + firmware_file_exist = exist(firmware_abs_path) + if firmware_file_exist: + checks += 1 + getsize = os.path.getsize(firmware_abs_path) + print('Firmware file exist, size: {}'.format(getsize)) + checks_add += 1 if getsize >= 100 * 1024 else 0 + # checks_add += 1 if os.path.getsize(firmware_abs_path) == 256*1024 else 0 + os.remove(firmware_abs_path) + assert not exist(firmware_abs_path) + + print('CHECK {} ; CHECK ADDITIONAL {}'.format(checks, checks_add)) + + assert checks == ATTEMPTS + assert checks_add == checks + + +def skip_if_not_fedora(): + import os + exist = os.path.exists + + def skip(): + pytest.skip('Fedora specific test, due to the mount path. Could be suited for Debian.') + + os_release_fp = '/etc/os-release' + if not exist(os_release_fp): + skip() + with open(os_release_fp) as f: + os_release_lines = f.readlines() + if 'Fedora' not in os_release_lines[0]: + skip() -- cgit v1.2.1 From 908ae6af21e472e19110e0a17371a0c867d56cec Mon Sep 17 00:00:00 2001 From: Szczepan Zalega Date: Fri, 18 Jan 2019 11:42:30 +0100 Subject: Add FIXME comments. Make the skip() method descriptive. Signed-off-by: Szczepan Zalega --- unittest/test_storage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/unittest/test_storage.py b/unittest/test_storage.py index 380d42e..472e2cf 100644 --- a/unittest/test_storage.py +++ b/unittest/test_storage.py @@ -407,15 +407,15 @@ def test_export_firmware_extended_fedora29(C): """ skip_if_device_version_lower_than({'S': 43}) - skip_if_not_fedora() + skip_if_not_fedora('Fedora specific test, due to the mount path. Could be suited for Debian.') import pexpect from time import sleep import os exist = os.path.exists - device = '/dev/sde1' - firmware_abs_path = '/run/media/sz/Nitrokey/firmware.bin' + device = '/dev/sde1' # FIXME autodetect the block device with udev + firmware_abs_path = '/run/media/sz/Nitrokey/firmware.bin' # FIXME use the actual user name in mount path pexpect.run(f'udisksctl mount -b {device}') checks = 0 checks_add = 0 @@ -448,12 +448,12 @@ def test_export_firmware_extended_fedora29(C): assert checks_add == checks -def skip_if_not_fedora(): +def skip_if_not_fedora(message:str) -> None: import os exist = os.path.exists def skip(): - pytest.skip('Fedora specific test, due to the mount path. Could be suited for Debian.') + pytest.skip(message) os_release_fp = '/etc/os-release' if not exist(os_release_fp): -- cgit v1.2.1 From 1e45aacdd9df8ca9322a8c68be488d92fa628cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20M=C3=BCller?= Date: Fri, 18 Jan 2019 17:36:34 +0100 Subject: Stress test for firmware export feature on macOS --- unittest/test_storage.py | 94 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/unittest/test_storage.py b/unittest/test_storage.py index 472e2cf..b4b21ea 100644 --- a/unittest/test_storage.py +++ b/unittest/test_storage.py @@ -412,7 +412,7 @@ def test_export_firmware_extended_fedora29(C): import pexpect from time import sleep import os - exist = os.path.exists + from os.path import exists as exist device = '/dev/sde1' # FIXME autodetect the block device with udev firmware_abs_path = '/run/media/sz/Nitrokey/firmware.bin' # FIXME use the actual user name in mount path @@ -450,7 +450,7 @@ def test_export_firmware_extended_fedora29(C): def skip_if_not_fedora(message:str) -> None: import os - exist = os.path.exists + from os.path import exists as exist def skip(): pytest.skip(message) @@ -462,3 +462,93 @@ def skip_if_not_fedora(message:str) -> None: os_release_lines = f.readlines() if 'Fedora' not in os_release_lines[0]: skip() + + +@pytest.mark.other +@pytest.mark.firmware +def test_export_firmware_extended_macos(C): + """ + Check, whether the firmware file is exported correctly, and in correct size. + Apparently, the auto-remounting side effect of the v0.46 change, is disturbing the export process. + Unmounting the UV just before the export gives the device 20/20 success rate. + Test case for issue https://github.com/Nitrokey/nitrokey-app/issues/399 + """ + + skip_if_device_version_lower_than({'S': 43}) + skip_if_not_macos('macOS specific test, due to the mount path and command.') + + import pexpect + from time import sleep + import os + from os.path import exists as exist + import plistlib + + usb_devices = pexpect.run('system_profiler -xml SPUSBDataType') + usb_devices_parsed = plistlib.loads(usb_devices) + + assert isinstance(usb_devices_parsed, list), 'usb_devices_parsed has unexpected type' + + # Try to get all USB devices + try: + devices = usb_devices_parsed[0]['_items'][0]['_items'] + except KeyError: + devices = None + + assert devices is not None, 'could not list USB devices' + + device_item = None + + for item in devices: + if 'manufacturer' in item and item['manufacturer'] == 'Nitrokey': + device_item = item + + # Try to get first volume of USB device + try: + volume = device_item['Media'][0]['volumes'][0] + except KeyError: + volume = None + + assert volume is not None, 'could not determine volume' + assert 'bsd_name' in volume, 'could not get BSD style device name' + + device = '/dev/' + volume['bsd_name'] + pexpect.run(f'diskutil mount {device}') + sleep(3) + assert 'mount_point' in volume, 'could not get mount point' + firmware_abs_path = volume['mount_point'] + '/firmware.bin' + checks = 0 + checks_add = 0 + + if exist(firmware_abs_path): + os.remove(firmware_abs_path) + + assert not exist(firmware_abs_path) + + ATTEMPTS = 20 + for i in range(ATTEMPTS): + # if umount is disabled, success rate is 3/10, enabled: 10/10 + pexpect.run(f'diskutil unmount {device}') + assert C.NK_export_firmware(DefaultPasswords.ADMIN) == DeviceErrorCode.STATUS_OK + pexpect.run(f'diskutil mount {device}') + sleep(1) + firmware_file_exist = exist(firmware_abs_path) + if firmware_file_exist: + checks += 1 + getsize = os.path.getsize(firmware_abs_path) + print('Firmware file exist, size: {}'.format(getsize)) + checks_add += 1 if getsize >= 100 * 1024 else 0 + # checks_add += 1 if os.path.getsize(firmware_abs_path) == 256*1024 else 0 + os.remove(firmware_abs_path) + assert not exist(firmware_abs_path) + + print('CHECK {} ; CHECK ADDITIONAL {}'.format(checks, checks_add)) + + assert checks == ATTEMPTS + assert checks_add == checks + + +def skip_if_not_macos(message:str) -> None: + import platform + + if platform.system() != 'Darwin': + pytest.skip(message) -- cgit v1.2.1 From 502b534e50737372d9e48b9c3e063b3281988b82 Mon Sep 17 00:00:00 2001 From: Szczepan Zalega Date: Sat, 26 Jan 2019 17:53:53 +0100 Subject: Small correction for macOS 10.13.6 Signed-off-by: Szczepan Zalega --- unittest/test_storage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unittest/test_storage.py b/unittest/test_storage.py index b4b21ea..53ad006 100644 --- a/unittest/test_storage.py +++ b/unittest/test_storage.py @@ -499,13 +499,16 @@ def test_export_firmware_extended_macos(C): device_item = None for item in devices: + if '_items' in item: + # Fix for macOS 10.13.6, Python 3.6.2 + item = item['_items'][0] if 'manufacturer' in item and item['manufacturer'] == 'Nitrokey': device_item = item # Try to get first volume of USB device try: volume = device_item['Media'][0]['volumes'][0] - except KeyError: + except (KeyError, TypeError): volume = None assert volume is not None, 'could not determine volume' -- cgit v1.2.1 From c9b07f6adea66d1edc20acfee759880743e6a5db Mon Sep 17 00:00:00 2001 From: Szczepan Zalega Date: Sat, 26 Jan 2019 17:55:10 +0100 Subject: Additional check and device/mount print Signed-off-by: Szczepan Zalega --- unittest/test_storage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unittest/test_storage.py b/unittest/test_storage.py index 53ad006..73dae9d 100644 --- a/unittest/test_storage.py +++ b/unittest/test_storage.py @@ -484,6 +484,7 @@ def test_export_firmware_extended_macos(C): import plistlib usb_devices = pexpect.run('system_profiler -xml SPUSBDataType') + assert b'Nitrokey' in usb_devices, 'No Nitrokey devices connected' usb_devices_parsed = plistlib.loads(usb_devices) assert isinstance(usb_devices_parsed, list), 'usb_devices_parsed has unexpected type' @@ -520,6 +521,7 @@ def test_export_firmware_extended_macos(C): assert 'mount_point' in volume, 'could not get mount point' firmware_abs_path = volume['mount_point'] + '/firmware.bin' checks = 0 + print('path: {}, device: {}'.format(firmware_abs_path, device)) checks_add = 0 if exist(firmware_abs_path): -- cgit v1.2.1 From 909a2c3c30fc56e42c0193ab528ae86cde31ea7c Mon Sep 17 00:00:00 2001 From: Szczepan Zalega Date: Sat, 26 Jan 2019 20:59:36 +0100 Subject: Detect UV's block device on Fedora automatically Signed-off-by: Szczepan Zalega --- unittest/test_storage.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/unittest/test_storage.py b/unittest/test_storage.py index 73dae9d..55671ae 100644 --- a/unittest/test_storage.py +++ b/unittest/test_storage.py @@ -409,14 +409,36 @@ def test_export_firmware_extended_fedora29(C): skip_if_device_version_lower_than({'S': 43}) skip_if_not_fedora('Fedora specific test, due to the mount path. Could be suited for Debian.') - import pexpect from time import sleep import os from os.path import exists as exist + import re + try: + import pyudev as pu + import pexpect + except: + pytest.skip('Skipping due to missing required packages: pyudev and pexpect.') + + ctx = pu.Context() + devices = ctx.list_devices(subsystem='block', ID_VENDOR='Nitrokey') + device = None + for d in devices: + if d.device_type == 'partition': + device = '/dev/{}'.format(d.sys_name) + break + assert device, 'Device could not be found' + + pexpect.run(f'udisksctl unmount -b {device}').decode() + sleep(1) + _res = pexpect.run(f'udisksctl mount -b {device}').decode() + firmware_abs_path = re.findall('at (/.*)\.', _res) + assert firmware_abs_path, 'Cannot get mount point' + firmware_abs_path = firmware_abs_path[0] + + print('path: {}, device: {}'.format(firmware_abs_path, device)) + assert firmware_abs_path, 'Cannot get mount point' + firmware_abs_path = firmware_abs_path + '/firmware.bin' - device = '/dev/sde1' # FIXME autodetect the block device with udev - firmware_abs_path = '/run/media/sz/Nitrokey/firmware.bin' # FIXME use the actual user name in mount path - pexpect.run(f'udisksctl mount -b {device}') checks = 0 checks_add = 0 @@ -428,7 +450,7 @@ def test_export_firmware_extended_fedora29(C): ATTEMPTS = 20 for i in range(ATTEMPTS): # if umount is disabled, success rate is 3/10, enabled: 10/10 - # pexpect.run(f'udisksctl unmount -b {device}') + pexpect.run(f'udisksctl unmount -b {device}') assert C.NK_export_firmware(DefaultPasswords.ADMIN) == DeviceErrorCode.STATUS_OK pexpect.run(f'udisksctl mount -b {device}') sleep(1) -- cgit v1.2.1 From 63296703be36511ce671a10e9b9193628604f1ca Mon Sep 17 00:00:00 2001 From: Szczepan Zalega Date: Sat, 26 Jan 2019 21:04:31 +0100 Subject: Correct the skip message for other OS than Fedora Signed-off-by: Szczepan Zalega --- unittest/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittest/test_storage.py b/unittest/test_storage.py index 55671ae..b125aa9 100644 --- a/unittest/test_storage.py +++ b/unittest/test_storage.py @@ -407,7 +407,7 @@ def test_export_firmware_extended_fedora29(C): """ skip_if_device_version_lower_than({'S': 43}) - skip_if_not_fedora('Fedora specific test, due to the mount path. Could be suited for Debian.') + skip_if_not_fedora('Tested on Fedora only. To check on other distros.') from time import sleep import os -- cgit v1.2.1