resource: Support waiting for reserved resources until available

Before this patch, almost everything was in place to support concurrent
osmo-gsm-tester instances sharing a common state dir. However, during
resource reservation, if the reservation couldn't be done due to too
many resources being in use, osmo-gsm-tester would fail and skip the
test suite.
With this patch, OGT will wait until some reserved resources are
released and then try requesting the reservation again.

Change-Id: I938602ee890712fda82fd3f812d8edb1bcd05e08

2 files changed, 102 insertions, 0 deletions

diff --git a/selftest/resource_test/resource_test.ok b/selftest/resource_test/resource_test.ok
index 91acaaf..5f7d5f6 100644
--- a/selftest/resource_test/resource_test.ok
+++ b/selftest/resource_test/resource_test.ok
@@ -116,6 +116,59 @@ cnf -: DBG: MAIN CONFIG:
 *** end: all resources
 
 - request some resources
+--- testowner: Verifying 2 x arfcn (candidates: 10)
+--- testowner: DBG: Picked - _hash: e620569450f8259b3f0212ec19c285dd07df063c
+  arfcn: '512'
+  band: GSM-1800
+- _hash: 022621e513c5a5bf33b77430a1e9c886be676fa1
+  arfcn: '514'
+  band: GSM-1800
+--- testowner: Verifying 2 x bts (candidates: 3)
+--- testowner: DBG: Picked - _hash: d2aa7c1124943de352351b650ca0c751784da6b6
+  addr: 10.42.42.114
+  band: GSM-1800
+  ciphers:
+  - a5_0
+  - a5_1
+  direct_pcu: 'True'
+  ipa_unit_id: '1'
+  label: sysmoBTS 1002
+  type: osmo-bts-sysmo
+- _hash: 2158317d5e0055070e7174c2498dedf53a2957e9
+  addr: 10.42.42.50
+  band: GSM-1800
+  ciphers:
+  - a5_0
+  - a5_1
+  ipa_unit_id: '6'
+  label: Ettus B200
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'True'
+  type: osmo-bts-trx
+--- testowner: Verifying 1 x ip_address (candidates: 5)
+--- testowner: DBG: Picked - _hash: fd103b22c7cf2480d609150e06f4bbd92ac78d8c
+  addr: 10.42.42.2
+--- testowner: Verifying 2 x modem (candidates: 4)
+--- testowner: DBG: Picked - _hash: 0b538cb6ad799fbd7c2953fd3b4463a76c7cc9c0
+  auth_algo: comp128v1
+  ciphers:
+  - a5_0
+  - a5_1
+  imsi: '901700000009031'
+  ki: 80A37E6FDEA931EAC92FFA5F671EFEAD
+  label: sierra_1
+  path: /sierra_1
+- _hash: 3a6e7747dfe7dfdf817bd3351031bd08051605c3
+  auth_algo: comp128v1
+  ciphers:
+  - a5_0
+  - a5_1
+  imsi: '901700000009029'
+  ki: 00969E283349D354A8239E877F2E0866
+  label: sierra_2
+  path: /sierra_2
+--- testowner: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/resource_test/conf/test_work/state_dir
 --- testowner: Reserving 2 x arfcn (candidates: 10)
 --- testowner: DBG: Picked - _hash: e620569450f8259b3f0212ec19c285dd07df063c
   arfcn: '512'
@@ -168,6 +221,7 @@ cnf -: DBG: MAIN CONFIG:
   ki: 00969E283349D354A8239E877F2E0866
   label: sierra_2
   path: /sierra_2
+--- testowner: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/resource_test/conf/test_work/state_dir, is_recursive=False>
 ~~~ currently reserved:
 arfcn:
 - _hash: e620569450f8259b3f0212ec19c285dd07df063c
@@ -290,3 +344,27 @@ resources(testowner)={'arfcn': [{'_hash': 'e620569450f8259b3f0212ec19c285dd07df0
 3rd subset should not match, pass
 3rd subset should not match, pass
 4th subset should not match, pass
+*** concurrent allocation:
+--- testowner1: Verifying 2 x arfcn (candidates: 10)
+--- testowner1: Verifying 2 x bts (candidates: 3)
+--- testowner1: Verifying 1 x ip_address (candidates: 5)
+--- testowner1: Verifying 2 x modem (candidates: 4)
+--- testowner1: Reserving 2 x arfcn (candidates: 10)
+--- testowner1: Reserving 2 x bts (candidates: 3)
+--- testowner1: Reserving 1 x ip_address (candidates: 5)
+--- testowner1: Reserving 2 x modem (candidates: 4)
+- 2nd instance reserve() start
+--- testowner2: Verifying 2 x arfcn (candidates: 10)
+--- testowner2: Verifying 2 x bts (candidates: 3)
+--- testowner2: Verifying 1 x ip_address (candidates: 5)
+--- testowner2: Verifying 2 x modem (candidates: 4)
+--- testowner2: Reserving 2 x arfcn (candidates: 8)
+--- testowner2: Reserving 2 x bts (candidates: 1)
+--- testowner2: Unable to reserve resources, too many currently reserved. Waiting until some are available again
+- 1st instance free()
+--- testowner2: Reserving 2 x arfcn (candidates: 10)
+--- testowner2: Reserving 2 x bts (candidates: 3)
+--- testowner2: Reserving 1 x ip_address (candidates: 5)
+--- testowner2: Reserving 2 x modem (candidates: 4)
+- 2nd instance reserve() done
+*** end: concurrent allocation
diff --git a/selftest/resource_test/resource_test.py b/selftest/resource_test/resource_test.py
index f18aa73..13cce97 100755
--- a/selftest/resource_test/resource_test.py
+++ b/selftest/resource_test/resource_test.py
@@ -6,6 +6,8 @@ import sys
 import pprint
 import shutil
 import atexit
+import time
+import threading
 import _prep
 from osmo_gsm_tester.core import config, log, util, resource
 from osmo_gsm_tester.core.schema import generate_schemas
@@ -128,4 +130,26 @@ subset =  { 'unordered_list_set': [4] }
 if not resource.item_matches(superset, subset):
     print('4th subset should not match, pass')
 
+print('*** concurrent allocation:')
+origin1 = log.Origin(None, 'testowner1')
+origin2 = log.Origin(None, 'testowner2')
+# We disable dbg() for second thread since FileWatch output result is
+# non-deterministic, since sometimes 1 Modiffied event is triggered, sometimes 2.
+origin1.dbg = origin2.dbg = lambda obj, *messages, _src=3, **named_items: None
+resources2 = None
+def second_ogt_instance():
+    # should block here until "resources" are freed.
+    print('- 2nd instance reserve() start')
+    resources2 = pool.reserve(origin2, config.replicate_times(want), config.replicate_times(modifiers))
+    print('- 2nd instance reserve() done')
+    resources2.free()
+resources = pool.reserve(origin1, config.replicate_times(want), config.replicate_times(modifiers))
+th = threading.Thread(target=second_ogt_instance)
+th.start()
+time.sleep(1.0)
+print('- 1st instance free()')
+resources.free()
+th.join()
+print('*** end: concurrent allocation')
+
 # vim: expandtab tabstop=4 shiftwidth=4