diff options
| -rw-r--r-- | doc/manuals/chapters/config.adoc | 55 | ||||
| l--------- | selftest/schema_test/_prep.py | 1 | ||||
| -rw-r--r-- | selftest/schema_test/schema_case_01.conf | 43 | ||||
| -rw-r--r-- | selftest/schema_test/schema_case_02.conf | 12 | ||||
| -rw-r--r-- | selftest/schema_test/schema_case_03.conf | 12 | ||||
| -rw-r--r-- | selftest/schema_test/schema_case_04.conf | 12 | ||||
| -rw-r--r-- | selftest/schema_test/schema_case_05.conf | 44 | ||||
| -rw-r--r-- | selftest/schema_test/schema_test.err | 0 | ||||
| -rw-r--r-- | selftest/schema_test/schema_test.ok | 63 | ||||
| -rwxr-xr-x | selftest/schema_test/schema_test.py | 53 | ||||
| -rw-r--r-- | selftest/suite_test/suite_test.ok | 168 | ||||
| -rwxr-xr-x | selftest/suite_test/suite_test.py | 15 | ||||
| -rw-r--r-- | selftest/suite_test/test_suite/suite.conf | 6 | ||||
| -rw-r--r-- | selftest/suite_test/test_suite/test_suite_params.py | 25 | ||||
| -rw-r--r-- | src/osmo_gsm_tester/core/schema.py | 24 | ||||
| -rw-r--r-- | src/osmo_gsm_tester/core/suite.py | 21 | ||||
| -rw-r--r-- | src/osmo_gsm_tester/core/test.py | 5 | ||||
| -rw-r--r-- | src/osmo_gsm_tester/testenv.py | 6 |
18 files changed, 543 insertions, 22 deletions
diff --git a/doc/manuals/chapters/config.adoc b/doc/manuals/chapters/config.adoc index 4dd90ff..483fd8c 100644 --- a/doc/manuals/chapters/config.adoc +++ b/doc/manuals/chapters/config.adoc @@ -305,16 +305,26 @@ suites_dir/ This file content is parsed using the <<schema_want,Want>> schema. -It provides -{app-name} with the base restrictions (later to be further filtered by -<<scenario_conf,scenario>> files) to apply when allocating resources. +On the <<schema_want,resources>> section, it provides {app-name} with the base restrictions +(later to be further filtered by <<scenario_conf,scenario>> files) to apply when +allocating resources. It can also override attributes for the allocated resources through the <<schema_want,modifiers>> section (to be further modified by -<<scenario_conf,scenario>> files later on). Similary it can do the same for +<<scenario_conf,scenario>> files later on). Similarly it can do the same for general configuration options (no per-resource) through the <<schema_want,config>> section. +The _schema_ section allows defining a suite's own schema used to validate +parameters passed to it later on through <<scenario_conf,scenario>> files (See +<<scenario_suite_params>>), and which can be retrieved by tests using the +_tenv.config_suite_specific()_ and _tenv.config_test_specific()_ APIs. The first +one will provide the whole dictionary under schema, while the later will return +the dictionary immediatelly inside the former and matching the test name being +run. For instance, if _tenv.config_test_specific()_ is called from test +_a_suite_test_foo.py_, the method will return the contents under dictionary with +key _a_suite_test_foo_. + .Sample 'suite.conf' file: ---- resources: @@ -337,6 +347,12 @@ config: codec_list: - fr1 +schema: + some_suite_parameter: 'uint' + a_suite_test_foo: + one_test_parameter_for_test_foo: 'str' + another_test_parameter_for_test_foo: ['bool_str'] + defaults: timeout: 50s ---- @@ -431,6 +447,37 @@ This way {app-name} when parsing the scenarios and combining them with the suite . Generate the final scenario content from the template available in the matched '.conf' file. +[[scenario_suite_params]] +*_Scenario to set suite/test parameters_*: + +First, the suite needs to define its schema in its <<suite_conf,suite.conf>> +file. Check <<suite_conf>> on how to do so. + +For instance, for a suite named 'mysuite' containing a test 'a_suite_test_foo.py', and containing this schema in its <<suite_conf,suite.conf>> file: +---- +schema: + some_suite_parameter: 'uint' + a_suite_test_foo: + one_test_parameter_for_test_foo: 'str' + another_test_parameter_for_test_foo: ['bool_str'] +---- + +One could define a parametrized scenario 'myparamscenario@.conf' like this: +---- +config: + suite: + mysuite: + some_suite_parameter: ${param1} + a_suite_test_foo: + one_test_parameter_for_test_foo: ${param2} + another_test_parameter_for_test_foo: ['true', 'false', 'false', 'true'] +---- + +And use it in {app-name} this way: +---- +mysuite:myparamscenario@4,hello.conf +---- + [[resources_conf]] ==== 'resources.conf' diff --git a/selftest/schema_test/_prep.py b/selftest/schema_test/_prep.py new file mode 120000 index 0000000..9cea3fe --- /dev/null +++ b/selftest/schema_test/_prep.py @@ -0,0 +1 @@ +../_prep.py
\ No newline at end of file diff --git a/selftest/schema_test/schema_case_01.conf b/selftest/schema_test/schema_case_01.conf new file mode 100644 index 0000000..dacf18a --- /dev/null +++ b/selftest/schema_test/schema_case_01.conf @@ -0,0 +1,43 @@ +schema: + handover: + duration: 'duration' + threshold: 'uint' + +tests: + - foobar: + prefix: + handover: + duration: 3 + threshold: 2 + - foobar: + prefix: + handover: + duration: 22kkk + - foobar: + prefix: + handover: + duration: 22h + - foobar: + wrongprefix: + handover: + duration: 22h + - foobar: + wrongprefix: + handover: + - foobar: + prefix: + handover: + threshold: 1 + - foobar: + prefix: + handover: + threshold: -2 + - foobar: + prefix: + handover: + - threshold: 1 + - foobar: + prefix: + handover: + threshold: + - 1 diff --git a/selftest/schema_test/schema_case_02.conf b/selftest/schema_test/schema_case_02.conf new file mode 100644 index 0000000..ddc02df --- /dev/null +++ b/selftest/schema_test/schema_case_02.conf @@ -0,0 +1,12 @@ +schema: + hey: + ho: + letsgo: ['wrongtype'] + +tests: + - foobar: + prefix: + hey: + ho: + letsgo: + - nanana diff --git a/selftest/schema_test/schema_case_03.conf b/selftest/schema_test/schema_case_03.conf new file mode 100644 index 0000000..e06fa24 --- /dev/null +++ b/selftest/schema_test/schema_case_03.conf @@ -0,0 +1,12 @@ +schema: + hey: + ho: + letsgo: ['str', 'str'] + +tests: + - foobar: + prefix: + hey: + ho: + letsgo: + - nanana diff --git a/selftest/schema_test/schema_case_04.conf b/selftest/schema_test/schema_case_04.conf new file mode 100644 index 0000000..4148310 --- /dev/null +++ b/selftest/schema_test/schema_case_04.conf @@ -0,0 +1,12 @@ +schema: + hey: + ho: + letsgo: [] + +tests: + - foobar: + prefix: + hey: + ho: + letsgo: + - nanana diff --git a/selftest/schema_test/schema_case_05.conf b/selftest/schema_test/schema_case_05.conf new file mode 100644 index 0000000..ee3d5db --- /dev/null +++ b/selftest/schema_test/schema_case_05.conf @@ -0,0 +1,44 @@ +schema: + hey: + ho: + letsgo: ['str'] + +tests: + - foobar: + prefix: + hey: + ho: + letsgo: + - nanana + - foobar: + prefix: + hey: + ho: + letsgo: [] + - foobar: + prefix: + hey: + ho: + letsgo: + - nanana + - nunu + - foobar: + prefix: + hey: + ho: + letsgo: nanana + - foobar: + prefix: + hey: + ho: + letsgo: ['nana', 'nana', 'nana'] + - foobar: + prefix: + hey: + ho: + letsgo: ['nana', {}, 'nana'] + - foobar: + prefix: + hey: + ho: + letsgo: ['nana', [], 'nana'] diff --git a/selftest/schema_test/schema_test.err b/selftest/schema_test/schema_test.err new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/selftest/schema_test/schema_test.err diff --git a/selftest/schema_test/schema_test.ok b/selftest/schema_test/schema_test.ok new file mode 100644 index 0000000..2c4cd6a --- /dev/null +++ b/selftest/schema_test/schema_test.ok @@ -0,0 +1,63 @@ +==== Testing dynamically generated schemas ==== +schema_case_01.conf: +{'foobar.prefix.handover.duration': 'duration', + 'foobar.prefix.handover.threshold': 'uint'} +validating tests[0] +Validation: OK +validating tests[1] +--- foobar.prefix.handover.duration: ERR: ValueError: Invalid duration value: '22kkk' +Validation: Error +validating tests[2] +Validation: OK +validating tests[3] +--- -: ERR: ValueError: config item not known: 'foobar.wrongprefix.handover.duration' +Validation: Error +validating tests[4] +--- -: ERR: ValueError: config item not known: 'foobar.wrongprefix.handover' +Validation: Error +validating tests[5] +Validation: OK +validating tests[6] +--- foobar.prefix.handover.threshold: ERR: ValueError: Positive value expected instead of -2 +Validation: Error +validating tests[7] +--- -: ERR: ValueError: config item not known: 'foobar.prefix.handover[].threshold' +Validation: Error +validating tests[8] +--- -: ERR: ValueError: config item is a list, should be 'uint': 'foobar.prefix.handover.threshold' +Validation: Error +---------------------- +schema_case_02.conf: +{'foobar.prefix.hey.ho.letsgo[]': 'wrongtype'} +validating tests[0] +--- -: ERR: ValueError: unknown type 'wrongtype' at 'foobar.prefix.hey.ho.letsgo[]' +Validation: Error +---------------------- +schema_case_03.conf: +--- -: ERR: AssertionError: +config2schema: Error +---------------------- +schema_case_04.conf: +--- -: ERR: AssertionError: +config2schema: Error +---------------------- +schema_case_05.conf: +{'foobar.prefix.hey.ho.letsgo[]': 'str'} +validating tests[0] +Validation: OK +validating tests[1] +Validation: OK +validating tests[2] +Validation: OK +validating tests[3] +--- -: ERR: ValueError: config item not known: 'foobar.prefix.hey.ho.letsgo' +Validation: Error +validating tests[4] +Validation: OK +validating tests[5] +--- -: ERR: ValueError: config item is dict but should be a leaf node of type 'str': 'foobar.prefix.hey.ho.letsgo[]' +Validation: Error +validating tests[6] +--- -: ERR: ValueError: config item is a list, should be 'str': 'foobar.prefix.hey.ho.letsgo[]' +Validation: Error +---------------------- diff --git a/selftest/schema_test/schema_test.py b/selftest/schema_test/schema_test.py new file mode 100755 index 0000000..3cf2799 --- /dev/null +++ b/selftest/schema_test/schema_test.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import _prep + +import sys +import os +import io +import pprint +import copy + +from osmo_gsm_tester.core import config, log, schema + +def val(which, test_schema): + try: + schema.validate(which, test_schema) + print('Validation: OK') + except ValueError: + log.log_exn() + print('Validation: Error') + +def get_case_list(dir): + li = [] + for f in os.listdir(dir): + if f.startswith('schema_case'): + li.append(f) + return sorted(li) + +print('==== Testing dynamically generated schemas ====') +for f in get_case_list(_prep.script_dir): + print('%s:' % f) + example_config = os.path.join(_prep.script_dir, f) + cfg = config.read(example_config) + try: + schema_def = schema.config_to_schema_def(cfg['schema'], 'foobar.prefix.') + except AssertionError: + schema_def = None + log.log_exn() + print('config2schema: Error') + + if schema_def is not None: + pprint.pprint(schema_def) + i = 0 + for t in cfg['tests']: + print('validating tests[%d]' % i) + val(t, schema_def) + i += 1 + print('----------------------') + + + + + +# vim: expandtab tabstop=4 shiftwidth=4 diff --git a/selftest/suite_test/suite_test.ok b/selftest/suite_test/suite_test.ok index 908f24f..fa38605 100644 --- a/selftest/suite_test/suite_test.ok +++ b/selftest/suite_test/suite_test.ok @@ -1,5 +1,5 @@ - non-existing suite dir -cnf -: DBG: Found config file paths.conf as [PATH]/selftest/suite_test/paths.conf in ./suite_test which is [PATH]/selftest/suite_test +cnf -: DBG: Found config file paths.conf as [PATH]/selftest/suite_test/paths.conf in [PATH]/selftest/suite_test which is [PATH]/selftest/suite_test cnf -: DBG: [PATH]/selftest/suite_test/paths.conf: relative path ./test_work/state_dir is [PATH]/selftest/suite_test/test_work/state_dir cnf -: DBG: [PATH]/selftest/suite_test/paths.conf: relative path . is [PATH]/selftest/suite_test cnf -: DBG: Found path suites_dir as [PATH]/selftest/suite_test @@ -25,7 +25,7 @@ resources: - times: '2' - run hello world test -cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in ./suite_test which is [PATH]/selftest/suite_test +cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in [PATH]/selftest/suite_test which is [PATH]/selftest/suite_test cnf ResourcesPool: DBG: Found path state_dir as [PATH]/selftest/suite_test/test_work/state_dir --------------------------------------------------------------------- @@ -99,13 +99,14 @@ tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- -PASS: test_suite (pass: 1, skip: 5) +PASS: test_suite (pass: 1, skip: 6) pass: hello_world.py (N.N sec) skip: mo_mt_sms.py skip: mo_sms.py skip: test_error.py skip: test_fail.py skip: test_fail_raise.py + skip: test_suite_params.py - a test with an error @@ -122,13 +123,14 @@ tst test_error.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_error.py:[ --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- -FAIL: test_suite (fail: 1, skip: 5) +FAIL: test_suite (fail: 1, skip: 6) skip: hello_world.py (N.N sec) skip: mo_mt_sms.py skip: mo_sms.py FAIL: test_error.py (N.N sec) AssertionError: test_error.py:[LINENR]: assert False skip: test_fail.py skip: test_fail_raise.py + skip: test_suite_params.py - a test with a failure @@ -145,13 +147,14 @@ tst test_fail.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail.py:[LI --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- -FAIL: test_suite (fail: 1, skip: 5) +FAIL: test_suite (fail: 1, skip: 6) skip: hello_world.py (N.N sec) skip: mo_mt_sms.py skip: mo_sms.py skip: test_error.py (N.N sec) FAIL: test_fail.py (N.N sec) EpicFail: This failure is expected skip: test_fail_raise.py + skip: test_suite_params.py - a test with a raised failure @@ -167,15 +170,16 @@ tst test_fail_raise.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail_ --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- -FAIL: test_suite (fail: 1, skip: 5) +FAIL: test_suite (fail: 1, skip: 6) skip: hello_world.py (N.N sec) skip: mo_mt_sms.py skip: mo_sms.py skip: test_error.py (N.N sec) skip: test_fail.py (N.N sec) FAIL: test_fail_raise.py (N.N sec) ExpectedFail: This failure is expected + skip: test_suite_params.py - test with half empty scenario -cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in ./suite_test which is [PATH]/selftest/suite_test [config.py:[LINENR]] +cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in [PATH]/selftest/suite_test which is [PATH]/selftest/suite_test [config.py:[LINENR]] cnf ResourcesPool: DBG: Found path state_dir as [PATH]/selftest/suite_test/test_work/state_dir [config.py:[LINENR]] --------------------------------------------------------------------- @@ -254,15 +258,16 @@ tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- -PASS: test_suite (pass: 1, skip: 5) +PASS: test_suite (pass: 1, skip: 6) pass: hello_world.py (N.N sec) skip: mo_mt_sms.py skip: mo_sms.py skip: test_error.py skip: test_fail.py skip: test_fail_raise.py + skip: test_suite_params.py - test with scenario -cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in ./suite_test which is [PATH]/selftest/suite_test [config.py:[LINENR]] +cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in [PATH]/selftest/suite_test which is [PATH]/selftest/suite_test [config.py:[LINENR]] cnf ResourcesPool: DBG: Found path state_dir as [PATH]/selftest/suite_test/test_work/state_dir [config.py:[LINENR]] --------------------------------------------------------------------- @@ -341,15 +346,16 @@ tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- -PASS: test_suite (pass: 1, skip: 5) +PASS: test_suite (pass: 1, skip: 6) pass: hello_world.py (N.N sec) skip: mo_mt_sms.py skip: mo_sms.py skip: test_error.py skip: test_fail.py skip: test_fail_raise.py + skip: test_suite_params.py - test with scenario and modifiers -cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in ./suite_test which is [PATH]/selftest/suite_test [config.py:[LINENR]] +cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in [PATH]/selftest/suite_test which is [PATH]/selftest/suite_test [config.py:[LINENR]] cnf ResourcesPool: DBG: Found path state_dir as [PATH]/selftest/suite_test/test_work/state_dir [config.py:[LINENR]] tst test_suite: reserving resources in [PATH]/selftest/suite_test/test_work/state_dir ... [suite.py:[LINENR]] tst test_suite: DBG: {combining='resources'} [suite.py:[LINENR]] @@ -474,12 +480,150 @@ tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- -PASS: test_suite (pass: 1, skip: 5) +PASS: test_suite (pass: 1, skip: 6) pass: hello_world.py (N.N sec) skip: mo_mt_sms.py skip: mo_sms.py skip: test_error.py skip: test_fail.py skip: test_fail_raise.py + skip: test_suite_params.py +- test with suite-specific config +cnf ResourcesPool: DBG: Found config file resources.conf as [PATH]/selftest/suite_test/resources.conf in [PATH]/selftest/suite_test which is [PATH]/selftest/suite_test [config.py:[LINENR]] +cnf ResourcesPool: DBG: Found path state_dir as [PATH]/selftest/suite_test/test_work/state_dir [config.py:[LINENR]] +tst test_suite: reserving resources in [PATH]/selftest/suite_test/test_work/state_dir ... [suite.py:[LINENR]] +tst test_suite: DBG: {combining='resources'} [suite.py:[LINENR]] +tst {combining_scenarios='resources'}: DBG: {definition_conf={bts=[{'label': 'sysmoCell 5000'}, {'label': 'sysmoCell 5000'}, {'type': 'sysmo'}], ip_address=[{}], modem=[{}, {}]}} [test_suite↪{combining_scenarios='resources'}] [suite.py:[LINENR]] +tst {combining_scenarios='resources', scenario='foo'}: [RESOURCE_DICT] +tst test_suite: DBG: {combining='modifiers'} [suite.py:[LINENR]] +tst {combining_scenarios='modifiers'}: DBG: {definition_conf={}} [test_suite↪{combining_scenarios='modifiers'}] [suite.py:[LINENR]] +tst {combining_scenarios='modifiers', scenario='foo'}: DBG: {conf={}, scenario='foo'} [test_suite↪{combining_scenarios='modifiers', scenario='foo'}] [suite.py:[LINENR]] +tst test_suite: Reserving 3 x bts (candidates: 6) [resource.py:[LINENR]] +tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace + addr: 10.42.42.53 + band: GSM-1800 + ipa_unit_id: '7' + label: sysmoCell 5000 + osmo_trx: + clock_reference: external + launch_trx: 'False' + trx_ip: 10.42.42.112 + trx_list: + - max_power_red: '3' + nominal_power: '10' + - max_power_red: '0' + nominal_power: '12' + type: osmo-bts-trx +- _hash: c2feabd082c36a1cdeccb9a5237dfff7dbadb009 + addr: 10.42.42.53 + band: GSM-1800 + ipa_unit_id: '7' + label: sysmoCell 5000 + osmo_trx: + clock_reference: external + launch_trx: 'False' + trx_ip: 10.42.42.112 + trx_list: + - nominal_power: '10' + - max_power_red: '1' + nominal_power: '12' + type: osmo-bts-trx +- _hash: 07d9c8aaa940b674efcbbabdd69f58a6ce4e94f9 + addr: 10.42.42.114 + band: GSM-1800 + ipa_unit_id: '1' + label: sysmoBTS 1002 + type: sysmo + [resource.py:[LINENR]] +tst test_suite: Reserving 1 x ip_address (candidates: 3) [resource.py:[LINENR]] +tst test_suite: DBG: Picked - _hash: cde1debf28f07f94f92c761b4b7c6bf35785ced4 + addr: 10.42.42.1 + [resource.py:[LINENR]] +tst test_suite: Reserving 2 x modem (candidates: 16) [resource.py:[LINENR]] +tst test_suite: DBG: Picked - _hash: 19c69e45aa090fb511446bd00797690aa82ff52f + imsi: '901700000007801' + ki: D620F48487B1B782DA55DF6717F08FF9 + label: m7801 + path: /wavecom_0 +- _hash: e1a46516a1fb493b2617ab14fc1693a9a45ec254 + imsi: '901700000007802' + ki: 47FDB2D55CE6A10A85ABDAD034A5B7B3 + label: m7802 + path: /wavecom_1 + [resource.py:[LINENR]] +resources(test_suite)={'bts': [{'_hash': 'a59640b8ba6a373552b24a6f9f65cadd2347bace', + '_reserved_by': 'test_suite-[ID_NUM]-[ID_NUM]', + 'addr': '10.42.42.53', + 'band': 'GSM-1800', + 'ipa_unit_id': '7', + 'label': 'sysmoCell 5000', + 'osmo_trx': {'clock_reference': 'external', + 'launch_trx': 'False', + 'trx_ip': '10.42.42.112'}, + 'trx_list': [{'max_power_red': '3', 'nominal_power': '10'}, + {'max_power_red': '0', 'nominal_power': '12'}], + 'type': 'osmo-bts-trx'}, + {'_hash': 'c2feabd082c36a1cdeccb9a5237dfff7dbadb009', + '_reserved_by': 'test_suite-[ID_NUM]-[ID_NUM]', + 'addr': '10.42.42.53', + 'band': 'GSM-1800', + 'ipa_unit_id': '7', + 'label': 'sysmoCell 5000', + 'osmo_trx': {'clock_reference': 'external', + 'launch_trx': 'False', + 'trx_ip': '10.42.42.112'}, + 'trx_list': [{'nominal_power': '10'}, + {'max_power_red': '1', 'nominal_power': '12'}], + 'type': 'osmo-bts-trx'}, + {'_hash': '07d9c8aaa940b674efcbbabdd69f58a6ce4e94f9', + '_reserved_by': 'test_suite-[ID_NUM]-[ID_NUM]', + 'addr': '10.42.42.114', + 'band': 'GSM-1800', + 'ipa_unit_id': '1', + 'label': 'sysmoBTS 1002', + 'type': 'sysmo'}], + 'ip_address': [{'_hash': 'cde1debf28f07f94f92c761b4b7c6bf35785ced4', + '_reserved_by': 'test_suite-[ID_NUM]-[ID_NUM]', + 'addr': '10.42.42.1'}], + 'modem': [{'_hash': '19c69e45aa090fb511446bd00797690aa82ff52f', + '_reserved_by': 'test_suite-[ID_NUM]-[ID_NUM]', + 'imsi': '901700000007801', + 'ki': 'D620F48487B1B782DA55DF6717F08FF9', + 'label': 'm7801', + 'path': '/wavecom_0'}, + {'_hash': 'e1a46516a1fb493b2617ab14fc1693a9a45ec254', + '_reserved_by': 'test_suite-[ID_NUM]-[ID_NUM]', + 'imsi': '901700000007802', + 'ki': '47FDB2D55CE6A10A85ABDAD034A5B7B3', + 'label': 'm7802', + 'path': '/wavecom_1'}]} + +--------------------------------------------------------------------- +trial test_suite +--------------------------------------------------------------------- + +---------------------------------------------- +trial test_suite test_suite_params.py +---------------------------------------------- +tst test_suite_params.py:[LINENR]: starting test [test_suite↪test_suite_params.py:[LINENR]] [test_suite_params.py:[LINENR]] +tst test_suite: DBG: {combining='config'} [suite.py:[LINENR]] +tst {combining_scenarios='config'}: DBG: {definition_conf={}} [test_suite↪{combining_scenarios='config'}] [suite.py:[LINENR]] +tst {combining_scenarios='config', scenario='foo'}: DBG: {conf={suite={test_suite={some_suite_global_param='heyho', test_suite_params={one_bool_parameter='true', second_list_parameter=['23', '45']}}}}, scenario='foo'} [test_suite↪{combining_scenarios='config', scenario='foo'}] [suite.py:[LINENR]] +tst test_suite_params.py:[LINENR]: SPECIFIC SUITE CONFIG: {'some_suite_global_param': 'heyho', [test_suite↪test_suite_params.py:[LINENR]] [test_suite_params.py:[LINENR]] +tst test_suite_params.py:[LINENR]: 'test_suite_params': {'one_bool_parameter': 'true', [test_suite↪test_suite_params.py:[LINENR]] [test_suite_params.py:[LINENR]] +tst test_suite_params.py:[LINENR]: 'second_list_parameter': ['23', '45']}} [test_suite↪test_suite_params.py:[LINENR]] [test_suite_params.py:[LINENR]] +tst test_suite_params.py:[LINENR]: SPECIFIC TEST CONFIG: {'one_bool_parameter': 'true', 'second_list_parameter': ['23', '45']} [test_suite↪test_suite_params.py:[LINENR]] [test_suite_params.py:[LINENR]] +tst test_suite_params.py:[LINENR] Test passed (N.N sec) [test_suite↪test_suite_params.py] [test.py:[LINENR]] +--------------------------------------------------------------------- +trial test_suite PASS +--------------------------------------------------------------------- +PASS: test_suite (pass: 1, skip: 6) + skip: hello_world.py + skip: mo_mt_sms.py + skip: mo_sms.py + skip: test_error.py + skip: test_fail.py + skip: test_fail_raise.py + pass: test_suite_params.py (N.N sec) - graceful exit. diff --git a/selftest/suite_test/suite_test.py b/selftest/suite_test/suite_test.py index de5c6df..99671c6 100755 --- a/selftest/suite_test/suite_test.py +++ b/selftest/suite_test/suite_test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 import os +import sys import _prep import shutil from osmo_gsm_tester.core import log, config, util, report from osmo_gsm_tester.core import suite -from osmo_gsm_tester.core.schema import generate_schemas +from osmo_gsm_tester.core.schema import generate_schemas, get_all_schema -config.ENV_CONF = './suite_test' +config.ENV_CONF = os.path.join(os.path.dirname(sys.argv[0])) example_trial_dir = os.path.join('test_trial_tmp') @@ -90,6 +91,16 @@ print(repr(s.reserved_resources)) results = s.run_tests('hello_world.py') print(report.suite_to_text(s)) +print('- test with suite-specific config') +trial = FakeTrial() +scenario = config.Scenario('foo', 'bar') +scenario['config'] = {'suite': {s.name(): { 'some_suite_global_param': 'heyho', 'test_suite_params': {'one_bool_parameter': 'true', 'second_list_parameter': ['23', '45']}}}} +s = suite.SuiteRun(trial, 'test_suite', s_def, [scenario]) +s.reserve_resources() +print(repr(s.reserved_resources)) +results = s.run_tests('test_suite_params.py') +print(report.suite_to_text(s)) + print('\n- graceful exit.') #deleting generated tmp trial dir: shutil.rmtree(example_trial_dir, ignore_errors=True) diff --git a/selftest/suite_test/test_suite/suite.conf b/selftest/suite_test/test_suite/suite.conf index 925dedb..4b70be8 100644 --- a/selftest/suite_test/test_suite/suite.conf +++ b/selftest/suite_test/test_suite/suite.conf @@ -9,5 +9,11 @@ resources: modem: - times: 2 +schema: + some_suite_global_param: 'str' + test_suite_params: + one_bool_parameter: 'bool_str' + second_list_parameter: ['uint'] + defaults: timeout: 60s diff --git a/selftest/suite_test/test_suite/test_suite_params.py b/selftest/suite_test/test_suite/test_suite_params.py new file mode 100644 index 0000000..2cb89d7 --- /dev/null +++ b/selftest/suite_test/test_suite/test_suite_params.py @@ -0,0 +1,25 @@ +from osmo_gsm_tester.testenv import * +import pprint + +print('starting test') + +suite_config = tenv.config_suite_specific() +print('SPECIFIC SUITE CONFIG: ' + pprint.pformat(suite_config)) + +test_config = tenv.config_test_specific() +print('SPECIFIC TEST CONFIG: ' + pprint.pformat(test_config)) + +some_suite_global_param = suite_config.get('some_suite_global_param', '') +assert some_suite_global_param == 'heyho' + +assert suite_config[tenv.test().module_name()] == test_config + +one_bool_parameter = test_config.get('one_bool_parameter', '') +assert one_bool_parameter == 'true' + +second_list_parameter = test_config.get('second_list_parameter', []) +assert len(second_list_parameter) == 2 +assert int(second_list_parameter[0]) == 23 +assert int(second_list_parameter[1]) == 45 + +#print('checks successful') diff --git a/src/osmo_gsm_tester/core/schema.py b/src/osmo_gsm_tester/core/schema.py index 0b21e70..9055c5b 100644 --- a/src/osmo_gsm_tester/core/schema.py +++ b/src/osmo_gsm_tester/core/schema.py @@ -325,6 +325,27 @@ def validate(config, schema): nest(None, config, schema) +def config_to_schema_def(src, key_prefix): + 'Converts a yaml parsed config into a schema dictionary used by validate()' + if util.is_dict(src): + out_dict = {} + for key, val in src.items(): + list_token = '' + dict_token = '' + if util.is_list(val): + list_token = '[]' + assert len(val) == 1 + val = val[0] + if util.is_dict(val): + dict_token = '.' + tmp_out = config_to_schema_def(val, "%s%s%s%s" %(key_prefix, key, list_token, dict_token)) + out_dict = {**out_dict, **tmp_out} + return out_dict + + # base case: string + return {key_prefix: str(src)} + + def generate_schemas(): "Generate supported schemas dynamically from objects" obj_dir = '%s/../obj/' % os.path.dirname(os.path.abspath(__file__)) @@ -366,12 +387,13 @@ def register_config_schema(obj_class_str, obj_attr_dict): """Register schema attributes to configure all instances of an object class. For instance: register_resource_schema_attributes('bsc', {'net.codec_list[]': schema.CODEC}) """ - global _CONFIG_SCHEMA + global _CONFIG_SCHEMA, _ALL_SCHEMA tmpdict = {} for key, val in obj_attr_dict.items(): new_key = '%s.%s' % (obj_class_str, key) tmpdict[new_key] = val combine(_CONFIG_SCHEMA, tmpdict) + _ALL_SCHEMA = None # reset _ALL_SCHEMA so it is re-generated next time it's requested. def get_resources_schema(): return _RESOURCES_SCHEMA; diff --git a/src/osmo_gsm_tester/core/suite.py b/src/osmo_gsm_tester/core/suite.py index 1bd6a63..81aab3e 100644 --- a/src/osmo_gsm_tester/core/suite.py +++ b/src/osmo_gsm_tester/core/suite.py @@ -38,8 +38,11 @@ class SuiteDefinition(log.Origin): CONF_FILENAME = 'suite.conf' def __init__(self, suite_dir): + self._suite_name = os.path.basename(suite_dir) + super().__init__(log.C_CNF, self._suite_name) self.suite_dir = suite_dir - super().__init__(log.C_CNF, os.path.basename(self.suite_dir)) + self.conf = None + self._schema = None self.read_conf() def read_conf(self): @@ -47,8 +50,12 @@ class SuiteDefinition(log.Origin): if not os.path.isdir(self.suite_dir): raise RuntimeError('No such directory: %r' % self.suite_dir) self.conf = config.read(os.path.join(self.suite_dir, - SuiteDefinition.CONF_FILENAME), - schema.get_all_schema()) + SuiteDefinition.CONF_FILENAME)) + # Drop schema part since it's dynamically defining content, makes no sense to validate it. + self._schema = self.conf.pop('schema', {}) + sdef = schema.config_to_schema_def(self._schema, "%s." % self._suite_name) + schema.register_config_schema('suite', sdef) + schema.validate(self.conf, schema.get_all_schema()) self.load_test_basenames() def load_test_basenames(self): @@ -58,6 +65,7 @@ class SuiteDefinition(log.Origin): continue self.test_basenames.append(basename) + class SuiteRun(log.Origin): UNKNOWN = 'UNKNOWN' PASS = 'PASS' @@ -79,6 +87,10 @@ class SuiteRun(log.Origin): self.status = SuiteRun.UNKNOWN self.load_tests() + def suite_name(self): + 'Return name of suite without scenarios' + return self.definition.name() + def trial(self): return self._trial @@ -130,6 +142,9 @@ class SuiteRun(log.Origin): self._config = self.combined('config', False) return self._config + def config_suite_specific(self): + return self.config().get('suite', {}).get(self.suite_name(), {}) + def resource_pool(self): return self.resources_pool diff --git a/src/osmo_gsm_tester/core/test.py b/src/osmo_gsm_tester/core/test.py index 8ab124b..7e03b6c 100644 --- a/src/osmo_gsm_tester/core/test.py +++ b/src/osmo_gsm_tester/core/test.py @@ -49,6 +49,11 @@ class Test(log.Origin): self.log_target = None self._report_stdout = None + def module_name(self): + 'Return test name without trailing .py' + assert self.basename.endswith('.py') + return self.basename[:-3] + def get_run_dir(self): if self._run_dir is None: self._run_dir = util.Dir(self.suite_run.get_run_dir().new_dir(self._name)) diff --git a/src/osmo_gsm_tester/testenv.py b/src/osmo_gsm_tester/testenv.py index 42288aa..789e291 100644 --- a/src/osmo_gsm_tester/testenv.py +++ b/src/osmo_gsm_tester/testenv.py @@ -141,6 +141,12 @@ class TestEnv(log_module.Origin): MainLoop.unregister_poll_func(self.poll) self.test_import_modules_cleanup() + def config_suite_specific(self): + return self.suite_run.config_suite_specific() + + def config_test_specific(self): + return self.suite_run.config_suite_specific().get(self._test.module_name(), {}) + def prompt(self, *msgs, **msg_details): 'ask for user interaction. Do not use in tests that should run automatically!' if msg_details: |