Introduced support for external python tests

The test scripts warn about missing documentation, untested configs,
check common errors, and stub out testing individual VTY commands.
The scripts have been moved to the another osmocom repository,
python/osmo-python-tests

The features were requested by zecke.

7 files changed, 145 insertions, 63 deletions

diff --git a/openbsc/Makefile.am b/openbsc/Makefile.am
index 91299f356..d7329ba33 100644
--- a/openbsc/Makefile.am
+++ b/openbsc/Makefile.am
@@ -7,7 +7,7 @@ pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = openbsc.pc
 
 BUILT_SOURCES = $(top_srcdir)/.version
-EXTRA_DIST = git-version-gen
+EXTRA_DIST = git-version-gen osmoappdesc.py
 $(top_srcdir)/.version:
 	echo $(VERSION) > $@-t && mv $@-t $@
 dist-hook:
diff --git a/openbsc/configure.ac b/openbsc/configure.ac
index cdefcaf04..3dd22124b 100644
--- a/openbsc/configure.ac
+++ b/openbsc/configure.ac
@@ -114,6 +114,21 @@ AC_DEFUN([CHECK_TM_INCLUDES_TM_GMTOFF], [
 
 CHECK_TM_INCLUDES_TM_GMTOFF
 
+AC_ARG_ENABLE([vty_tests],
+		AC_HELP_STRING([--enable-vty-tests],
+				[Include the VTY tests in make check [default=no]]),
+		[enable_vty_tests="$enableval"],[enable_vty_tests="no"])
+if test "x$enable_vty_tests" = "xyes" ; then
+	AM_PATH_PYTHON
+	AC_CHECK_PROG(OSMOTESTVTY_CHECK,osmotestvty.py,yes)
+	 if test "x$OSMOTESTVTY_CHECK" != "xyes" ; then
+		AC_MSG_ERROR([Please install osmocom-python to run the vty tests.])
+	fi
+fi
+AC_MSG_CHECKING([whether to enable VTY tests])
+AC_MSG_RESULT([$enable_vty_tests])
+AM_CONDITIONAL(ENABLE_VTY_TESTS, test "x$enable_vty_tests" = "xyes")
+
 dnl Generate the output
 AM_CONFIG_HEADER(bscconfig.h)
 
diff --git a/openbsc/contrib/dump_all_docs.py b/openbsc/contrib/dump_all_docs.py
deleted file mode 100755
index 2a67cb7f5..000000000
--- a/openbsc/contrib/dump_all_docs.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Start the process and dump the documentation to the doc dir. This is
-copied from the BTS directory and a fix might need to be applied there
-too.
-"""
-
-import socket, subprocess, time,os
-
-
-def dump_doc(end, port, filename):
-	sck = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-	sck.setblocking(1)
-	sck.connect(("localhost", port))
-	sck.recv(4096)
-
-	# Now send the command
-	sck.send("show online-help\r")
-	xml = ""
-	while True:
-		data = sck.recv(4096)
-		xml = "%s%s" % (xml, data)
-		if data.endswith(end):
-			break
-
-	# Now write everything until the end to the file
-	out = open(filename, 'w')
-	out.write(xml[18:len(end)*-1])
-	out.close()
-
-
-apps = [
-	# The same could be done with an empty config file but this way
-	# the example files are properly tested.
-	(4242, "src/osmo-nitb/osmo-nitb", "doc/examples/osmo-nitb/nanobts/openbsc.cfg", "OpenBSC", "nitb"),
-	(4242, "src/osmo-bsc/osmo-bsc", "doc/examples/osmo-bsc/osmo-bsc.cfg", "OsmoBSC", "bsc"),
-	(4243, "src/osmo-bsc_mgcp/osmo-bsc_mgcp", "doc/examples/osmo-bsc_mgcp/mgcp.cfg", "OpenBSC MGCP", "mgcp"),
-	(4244, "src/osmo-bsc_nat/osmo-bsc_nat", "doc/examples/osmo-bsc_nat/osmo-bsc_nat.cfg", "OsmoBSCNAT", "nat"), 
-	(4245, "src/gprs/osmo-sgsn", "doc/examples/osmo-sgsn/osmo-sgsn.cfg", "OsmoSGSN", "sgsn"), 
-	(4246, "src/gprs/osmo-gbproxy", "doc/examples/osmo-gbproxy/osmo-gbproxy.cfg", "OsmoGbProxy", "gbproxy"),
-]
-
-# Dump the config of all our apps
-for app in apps:
-	print "Starting app for %s" % app[4]
-
-	cmd = [app[1], "-c", app[2]]
-	proc = subprocess.Popen(cmd, stdin=None, stdout=None)
-	time.sleep(1)
-	try:
-		dump_doc('\r\n%s> ' % app[3], app[0], 'doc/%s_vty_reference.xml' % app[4])
-	finally:
-		# Clean-up
-		proc.kill()
-		proc.wait()
-
diff --git a/openbsc/doc/examples/osmo-bsc_nat/osmo-bsc_nat.cfg b/openbsc/doc/examples/osmo-bsc_nat/osmo-bsc_nat.cfg
index e254fba7d..93cb98075 100644
--- a/openbsc/doc/examples/osmo-bsc_nat/osmo-bsc_nat.cfg
+++ b/openbsc/doc/examples/osmo-bsc_nat/osmo-bsc_nat.cfg
@@ -1,3 +1,64 @@
-nat
+!
+! OsmoBSCNAT (0.12.0.266-2daa9) configuration saved from vty
+!!
+!
+log stderr
+  logging filter all 1
+  logging color 1
+  logging timestamp 0
+  logging level all everything
+  logging level rll notice
+  logging level cc notice
+  logging level mm notice
+  logging level rr notice
+  logging level rsl notice
+  logging level nm info
+  logging level mncc notice
+  logging level pag notice
+  logging level meas notice
+  logging level sccp notice
+  logging level msc notice
+  logging level mgcp notice
+  logging level ho notice
+  logging level db notice
+  logging level ref notice
+  logging level gprs everything
+  logging level ns info
+  logging level bssgp everything
+  logging level llc everything
+  logging level sndcp everything
+  logging level nat notice
+  logging level ctrl notice
+  logging level smpp everything
+  logging level lglobal notice
+  logging level llapd notice
+  logging level linp notice
+  logging level lmux notice
+  logging level lmi notice
+  logging level lmib notice
+  logging level lsms notice
+!
+line vty
+ no login
+!
 mgcp
- call agent ip 127.0.0.1
+  bind ip 0.0.0.0
+  bind port 2427
+  rtp bts-base 4000
+  rtp net-base 16000
+  rtp ip-dscp 0
+  no rtcp-omit
+  sdp audio-payload number 126
+  sdp audio-payload name AMR/8000
+  loop 0
+  number endpoints 1
+  call-agent ip 127.0.0.1
+  rtp transcoder-base 0
+  transcoder-remote-base 4000
+nat
+ msc ip 127.0.0.1
+ msc port 5000
+ timeout auth 2
+ timeout ping 20
+ timeout pong 5
+ ip-dscp 0
diff --git a/openbsc/doc/examples/osmo-sgsn/osmo-sgsn.cfg b/openbsc/doc/examples/osmo-sgsn/osmo-sgsn.cfg
index 027c03d23..49559830a 100644
--- a/openbsc/doc/examples/osmo-sgsn/osmo-sgsn.cfg
+++ b/openbsc/doc/examples/osmo-sgsn/osmo-sgsn.cfg
@@ -6,8 +6,8 @@ line vty
  no login
 !
 sgsn
- gtp local-ip 10.23.23.23
- ggsn 0 remote-ip 192.168.0.101
+ gtp local-ip 127.0.0.1
+ ggsn 0 remote-ip 127.0.0.1
  ggsn 0 gtp-version 1
 !
 ns
@@ -18,7 +18,7 @@ ns
  timer tns-test 30
  timer tns-alive 3
  timer tns-alive-retries 10
- encapsulation udp local-ip 192.168.0.101
+ encapsulation udp local-ip 127.0.0.1
  encapsulation udp local-port 23000
  encapsulation framerelay-gre enabled 0
 !
diff --git a/openbsc/osmoappdesc.py b/openbsc/osmoappdesc.py
new file mode 100644
index 000000000..19ec6c3c9
--- /dev/null
+++ b/openbsc/osmoappdesc.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+# (C) 2013 by Katerina Barone-Adesi <kat.obsc@gmail.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+
+# Most systems won't be able to use these, so they're separated out
+nitb_e1_configs = [
+    "doc/examples/osmo-nitb/bs11/openbsc-2bts-2trx.cfg",
+    "doc/examples/osmo-nitb/bs11/openbsc-1bts-2trx-hopping.cfg",
+    "doc/examples/osmo-nitb/bs11/openbsc-1bts-2trx.cfg",
+    "doc/examples/osmo-nitb/bs11/openbsc.cfg",
+    "doc/examples/osmo-nitb/nokia/openbsc_nokia_3trx.cfg",
+    "doc/examples/osmo-nitb/nanobts/openbsc-multitrx.cfg",
+    "doc/examples/osmo-nitb/rbs2308/openbsc.cfg"
+]
+
+
+app_configs = {
+    "osmo-bsc": ["doc/examples/osmo-bsc/osmo-bsc.cfg"],
+    "nat": ["doc/examples/osmo-bsc_nat/osmo-bsc_nat.cfg"],
+    "mgcp": ["doc/examples/osmo-bsc_mgcp/mgcp.cfg"],
+    "gbproxy": ["doc/examples/osmo-gbproxy/osmo-gbproxy.cfg"],
+    "sgsn": ["doc/examples/osmo-sgsn/osmo-sgsn.cfg"],
+    "nitb": ["doc/examples/osmo-nitb/hsl/openbsc.cfg",
+             "doc/examples/osmo-nitb/nanobts/openbsc.cfg"]
+}
+
+
+apps = [(4242, "src/osmo-bsc/osmo-bsc", "OsmoBSC", "osmo-bsc"),
+        (4244, "src/osmo-bsc_nat/osmo-bsc_nat",  "OsmoBSCNAT", "nat"),
+        (4243, "src/osmo-bsc_mgcp/osmo-bsc_mgcp", "OpenBSC MGCP", "mgcp"),
+        (4246, "src/gprs/osmo-gbproxy", "OsmoGbProxy", "gbproxy"),
+        (4245, "src/gprs/osmo-sgsn", "OsmoSGSN", "sgsn"),
+        (4242, "src/osmo-nitb/osmo-nitb", "OpenBSC", "nitb")
+        ]
+
+vty_command = ["./src/osmo-nitb/osmo-nitb", "-c",
+               "doc/examples/osmo-nitb/hsl/openbsc.cfg"]
+
+vty_app = apps[-1]
diff --git a/openbsc/tests/Makefile.am b/openbsc/tests/Makefile.am
index c5e0f1fd3..2944ca373 100644
--- a/openbsc/tests/Makefile.am
+++ b/openbsc/tests/Makefile.am
@@ -27,8 +27,19 @@ EXTRA_DIST = testsuite.at $(srcdir)/package.m4 $(TESTSUITE)
 TESTSUITE = $(srcdir)/testsuite
 DISTCLEANFILES = atconfig
 
+if ENABLE_VTY_TESTS
+python-tests: $(BUILT_SOURCES)
+	osmotestvty.py -p $(abs_top_srcdir) -w $(abs_top_builddir) -v
+	osmotestconfig.py -p $(abs_top_srcdir) -w $(abs_top_builddir) -v
+	rm -f $(top_builddir)/hlr.sqlite3
+else
+python-tests: $(BUILT_SOURCES)
+	echo "Not running python-based tests (determined at configure-time)"
+endif
+
 check-local: atconfig $(TESTSUITE)
 	$(SHELL) '$(TESTSUITE)' $(TESTSUITEFLAGS)
+	$(MAKE) $(AM_MAKEFLAGS) python-tests
 
 installcheck-local: atconfig $(TESTSUITE)
 	$(SHELL) '$(TESTSUITE)' AUTOTEST_PATH='$(bindir)' \