Use construct for EF_AD in pySim-{shell, prog, read}.py, cards.py

Also serves as example for RFU (reserved for future use) fields
which should not always be reset to zero in case they have been
set on the uSIM for some reason.
See pySim/ts_51_011.py, class EF_AD.

* Add definitions for RFU {Flag, Bits, Byte, Bytes}
* Use IntEnum for OP_MODE (convenient auto completion)
* Remove obsolete definitions and imports
* Update test results for all SIMs (opmode strings are shortened)

Change-Id: I65e0a426f80a619fec38856a30e590f0e726b554

12 files changed, 90 insertions, 58 deletions

diff --git a/pySim-prog.py b/pySim-prog.py
index 0abd190..4c85be7 100755
--- a/pySim-prog.py
+++ b/pySim-prog.py
@@ -146,7 +146,7 @@ def parse_options():
 	parser.add_option("--opmode", dest="opmode", type="choice",
 			help="Set UE Operation Mode in EF.AD (Administrative Data)",
 			default=None,
-			choices=['{:02X}'.format(m) for m in list(EF_AD.OP_MODE.keys())],
+			choices=['{:02X}'.format(int(m)) for m in EF_AD.OP_MODE],
 		)
 	parser.add_option("--epdgid", dest="epdgid",
 			help="Set Home Evolved Packet Data Gateway (ePDG) Identifier. (Only FQDN format supported)",
diff --git a/pySim-read.py b/pySim-read.py
index 59c5762..8e4a512 100755
--- a/pySim-read.py
+++ b/pySim-read.py
@@ -28,7 +28,7 @@ import os
 import random
 import re
 import sys
-from pySim.ts_51_011 import EF, DF, EF_SST_map, EF_AD_mode_map
+from pySim.ts_51_011 import EF, DF, EF_SST_map, EF_AD
 from pySim.ts_31_102 import EF_UST_map, EF_USIM_ADF_map
 from pySim.ts_31_103 import EF_IST_map, EF_ISIM_ADF_map
 
@@ -230,11 +230,10 @@ if __name__ == '__main__':
 	(res, sw) = card.read_binary('AD')
 	if sw == '9000':
 		print("Administrative data: %s" % (res,))
-		if res[:2] in EF_AD_mode_map:
-			print("\tMS operation mode: %s" % (EF_AD_mode_map[res[:2]],))
-		else:
-			print("\tMS operation mode: (unknown 0x%s)" % (res[:2],))
-		if int(res[4:6], 16) & 0x01:
+		ad = EF_AD()
+		decoded_data = ad.decode_hex(res)
+		print("\tMS operation mode: %s" % decoded_data['ms_operation_mode'])
+		if decoded_data['ofm']:
 			print("\tCiphering Indicator: enabled")
 		else:
 			print("\tCiphering Indicator: disabled")
diff --git a/pySim-shell.py b/pySim-shell.py
index 2a7c377..a8db263 100755
--- a/pySim-shell.py
+++ b/pySim-shell.py
@@ -30,7 +30,7 @@ import os
 import sys
 from pathlib import Path
 
-from pySim.ts_51_011 import EF, DF, EF_SST_map, EF_AD_mode_map
+from pySim.ts_51_011 import EF, DF, EF_SST_map
 from pySim.ts_31_102 import EF_UST_map, EF_USIM_ADF_map
 from pySim.ts_31_103 import EF_IST_map, EF_ISIM_ADF_map
 
diff --git a/pySim/cards.py b/pySim/cards.py
index 5a39bda..9a19eed 100644
--- a/pySim/cards.py
+++ b/pySim/cards.py
@@ -175,29 +175,27 @@ class Card(object):
 
 		# read from card
 		raw_hex_data, sw = self._scc.read_binary(EF['AD'], length=None, offset=0)
-		raw_bin_data = h2b(raw_hex_data)
-		abstract_data = ad.decode_bin(raw_bin_data)
+		abstract_data = ad.decode_hex(raw_hex_data)
 
 		# perform updates
-		if mnc:
+		if mnc and abstract_data['extensions']:
 			mnclen = len(str(mnc))
 			if mnclen == 1:
 				mnclen = 2
 			if mnclen > 3:
 				raise RuntimeError('invalid length of mnc "{}"'.format(mnc))
-			abstract_data['len_of_mnc_in_imsi'] = mnclen
+			abstract_data['extensions']['mnc_len'] = mnclen
 		if opmode:
-			opmode_symb = ad.OP_MODE.get(int(opmode, 16))
-			if opmode_symb:
-				abstract_data['ms_operation_mode'] = opmode_symb
+			opmode_num = int(opmode, 16)
+			if opmode_num in [int(v) for v in EF_AD.OP_MODE]:
+				abstract_data['ms_operation_mode'] = opmode_num
 			else:
 				raise RuntimeError('invalid opmode "{}"'.format(opmode))
 		if ofm:
-			abstract_data['specific_facilities']['ofm'] = bool(int(ofm, 16))
+			abstract_data['ofm'] = bool(int(ofm, 16))
 
 		# write to card
-		raw_bin_data = ad.encode_bin(abstract_data)
-		raw_hex_data = b2h(raw_bin_data)
+		raw_hex_data = ad.encode_hex(abstract_data)
 		data, sw = self._scc.update_binary(EF['AD'], raw_hex_data)
 		return sw
 
diff --git a/pySim/construct.py b/pySim/construct.py
index 839497c..b0f03b7 100644
--- a/pySim/construct.py
+++ b/pySim/construct.py
@@ -47,3 +47,44 @@ def filter_dict(d, exclude_prefix='_'):
 
 # here we collect some shared / common definitions of data types
 LV = Prefixed(Int8ub, HexAdapter(GreedyBytes))
+
+# Default value for Reserved for Future Use (RFU) bits/bytes
+# See TS 31.101 Sec. "3.4 Coding Conventions"
+__RFU_VALUE = 0
+
+# Field that packs Reserved for Future Use (RFU) bit
+FlagRFU = Default(Flag, __RFU_VALUE)
+
+# Field that packs Reserved for Future Use (RFU) byte
+ByteRFU = Default(Byte, __RFU_VALUE)
+
+# Field that packs all remaining Reserved for Future Use (RFU) bytes
+GreedyBytesRFU = Default(GreedyBytes, b'')
+
+def BitsRFU(n=1):
+    '''
+    Field that packs Reserved for Future Use (RFU) bit(s)
+    as defined in TS 31.101 Sec. "3.4 Coding Conventions"
+
+    Use this for (currently) unused/reserved bits whose contents
+    should be initialized automatically but should not be cleared
+    in the future or when restoring read data (unlike padding).
+
+    Parameters:
+        n (Integer): Number of bits (default: 1)
+    '''
+    return Default(BitsInteger(n), __RFU_VALUE)
+
+def BytesRFU(n=1):
+    '''
+    Field that packs Reserved for Future Use (RFU) byte(s)
+    as defined in TS 31.101 Sec. "3.4 Coding Conventions"
+
+    Use this for (currently) unused/reserved bytes whose contents
+    should be initialized automatically but should not be cleared
+    in the future or when restoring read data (unlike padding).
+
+    Parameters:
+        n (Integer): Number of bytes (default: 1)
+    '''
+    return Default(Bytes(n), __RFU_VALUE)
diff --git a/pySim/ts_51_011.py b/pySim/ts_51_011.py
index c21e86c..48649cd 100644
--- a/pySim/ts_51_011.py
+++ b/pySim/ts_51_011.py
@@ -319,22 +319,12 @@ EF_SST_map = {
 	59: 'MMS User Connectivity Parameters',
 }
 
-# 10.3.18 "EF.AD (Administrative data) "
-EF_AD_mode_map = {
-	'00' : 'normal operation',
-	'80' : 'type approval operations',
-	'01' : 'normal operation + specific facilities',
-	'81' : 'type approval operations + specific facilities',
-	'02' : 'maintenance (off line)',
-	'04' : 'cell test operation',
-}
-
-
 from pySim.utils import *
 from struct import pack, unpack
 from construct import *
 from construct import Optional as COptional
-from pySim.construct import HexAdapter, BcdAdapter
+from pySim.construct import HexAdapter, BcdAdapter, FlagRFU, ByteRFU, GreedyBytesRFU, BitsRFU, BytesRFU
+import enum
 
 from pySim.filesystem import *
 import pySim.ts_102_221
@@ -553,30 +543,34 @@ class EF_LOCI(TransparentEF):
 
 # TS 51.011 Section 10.3.18
 class EF_AD(TransparentEF):
-    OP_MODE = {
-            0x00: 'normal',
-            0x80: 'type_approval',
-            0x01: 'normal_and_specific_facilities',
-            0x81: 'type_approval_and_specific_facilities',
-            0x02: 'maintenance_off_line',
-            0x04: 'cell_test',
-        }
-    OP_MODE_reverse = dict(map(reversed, OP_MODE.items()))
+    class OP_MODE(enum.IntEnum):
+        normal                                  = 0x00
+        type_approval                           = 0x80
+        normal_and_specific_facilities          = 0x01
+        type_approval_and_specific_facilities   = 0x81
+        maintenance_off_line                    = 0x02
+        cell_test                               = 0x04
+    #OP_MODE_DICT = {int(v) : str(v) for v in EF_AD.OP_MODE}
+    #OP_MODE_DICT_REVERSED = {str(v) : int(v) for v in EF_AD.OP_MODE}
+
     def __init__(self, fid='6fad', sfid=None, name='EF.AD', desc='Administrative Data', size={3,4}):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-    def _decode_bin(self, raw_bin):
-        u = unpack('!BH', raw_bin[:3])
-        ofm = True if u[1] & 1 else False
-        res = {'ms_operation_mode': self.OP_MODE.get(u[0], u[0]), 'specific_facilities': { 'ofm': ofm } }
-        if len(raw_bin) > 3:
-            res['len_of_mnc_in_imsi'] = int(raw_bin[3]) & 0xf
-        return res
-    def _encode_bin(self, abstract):
-        op_mode = self.OP_MODE_reverse[abstract['ms_operation_mode']]
-        res = pack('!BH', op_mode, abstract['specific_facilities']['ofm'])
-        if 'len_of_mnc_in_imsi' in abstract:
-            res += pack('!B', abstract['len_of_mnc_in_imsi'])
-        return res
+        self._construct = BitStruct(
+            # Byte 1
+            'ms_operation_mode'/Bytewise(Enum(Byte, EF_AD.OP_MODE)),
+            # Byte 2
+            'rfu1'/Bytewise(ByteRFU),
+            # Byte 3
+            'rfu2'/BitsRFU(7),
+            'ofm'/Flag,
+            # Byte 4 (optional),
+            'extensions'/COptional(Struct(
+                'rfu3'/BitsRFU(4),
+                'mnc_len'/BitsInteger(4),
+                # Byte 5..N-4 (optional, RFU)
+                'extensions'/Bytewise(GreedyBytesRFU)
+            ))
+        )
 
 # TS 51.011 Section 10.3.20 / 10.3.22
 class EF_VGCS(TransRecEF):
diff --git a/pysim-testdata/Fairwaves-SIM.ok b/pysim-testdata/Fairwaves-SIM.ok
index e6fcfe3..f83f415 100644
--- a/pysim-testdata/Fairwaves-SIM.ok
+++ b/pysim-testdata/Fairwaves-SIM.ok
@@ -43,7 +43,7 @@ HPLMNAcT:
 ACC: 0008
 MSISDN: Not available
 Administrative data: 00000002
-	MS operation mode: normal operation
+	MS operation mode: normal
 	Ciphering Indicator: disabled
 SIM Service Table: ff3cc3ff030fff0f000fff03f0c0
 	Service 1 - CHV1 disable function
diff --git a/pysim-testdata/Wavemobile-SIM.ok b/pysim-testdata/Wavemobile-SIM.ok
index 1c78cc9..0682a70 100644
--- a/pysim-testdata/Wavemobile-SIM.ok
+++ b/pysim-testdata/Wavemobile-SIM.ok
@@ -50,7 +50,7 @@ HPLMNAcT: Can't read file -- SW match failed! Expected 9000 and got 6a82.
 ACC: abce
 MSISDN: Not available
 Administrative data: 00000102
-	MS operation mode: normal operation
+	MS operation mode: normal
 	Ciphering Indicator: enabled
 SIM Service Table: ff33ff0f3c00ff0f000cf0c0f0030000
 	Service 1 - CHV1 disable function
diff --git a/pysim-testdata/fakemagicsim.ok b/pysim-testdata/fakemagicsim.ok
index 1d1714f..11296f5 100644
--- a/pysim-testdata/fakemagicsim.ok
+++ b/pysim-testdata/fakemagicsim.ok
@@ -17,7 +17,7 @@ HPLMNAcT: Can't read file -- SW match failed! Expected 9000 and got 9404.
 ACC: ffff
 MSISDN: Not available
 Administrative data: 000000
-	MS operation mode: normal operation
+	MS operation mode: normal
 	Ciphering Indicator: disabled
 SIM Service Table: ff3fff0f0300f003000c
 	Service 1 - CHV1 disable function
diff --git a/pysim-testdata/sysmoISIM-SJA2.ok b/pysim-testdata/sysmoISIM-SJA2.ok
index ae332a8..dc7b865 100644
--- a/pysim-testdata/sysmoISIM-SJA2.ok
+++ b/pysim-testdata/sysmoISIM-SJA2.ok
@@ -55,7 +55,7 @@ HPLMNAcT:
 ACC: 0200
 MSISDN (NPI=1 ToN=3): 6766266
 Administrative data: 00000002
-	MS operation mode: normal operation
+	MS operation mode: normal
 	Ciphering Indicator: disabled
 SIM Service Table: ff33ffff3f003f0f300cf0c3f00000
 	Service 1 - CHV1 disable function
diff --git a/pysim-testdata/sysmoUSIM-SJS1.ok b/pysim-testdata/sysmoUSIM-SJS1.ok
index 95f6967..bce3c9d 100644
--- a/pysim-testdata/sysmoUSIM-SJS1.ok
+++ b/pysim-testdata/sysmoUSIM-SJS1.ok
@@ -55,7 +55,7 @@ HPLMNAcT:
 ACC: 0008
 MSISDN (NPI=1 ToN=1): +77776336143
 Administrative data: 00000002
-	MS operation mode: normal operation
+	MS operation mode: normal
 	Ciphering Indicator: disabled
 SIM Service Table: ff3fffff3f003f1ff00c00c0f00000
 	Service 1 - CHV1 disable function
diff --git a/pysim-testdata/sysmosim-gr1.ok b/pysim-testdata/sysmosim-gr1.ok
index f4b09c8..3aff2a3 100644
--- a/pysim-testdata/sysmosim-gr1.ok
+++ b/pysim-testdata/sysmosim-gr1.ok
@@ -17,7 +17,7 @@ HPLMNAcT: Can't read file -- SW match failed! Expected 9000 and got 9404.
 ACC: 0008
 MSISDN: Not available
 Administrative data: 000000
-	MS operation mode: normal operation
+	MS operation mode: normal
 	Ciphering Indicator: disabled
 SIM Service Table: ff3fff0f0f0000030000
 	Service 1 - CHV1 disable function