BER-TLV EF support (command, filesystem, shell)

This adds support for a new EF file type: BER-TLV files.  They are
different from transparent and linear fixed EFs in that they neither
operate on a byte stream nor fixed-sized records, but on BER-TLV encoded
objects.  One can specify a tag value, and the card will return the
entire TLV for that tag.

As indicated in the spec, the magic tag value 0x5C (92) will return a
list of tags existing in the file.

Change-Id: Ibfcce757dcd477fd0d6857f64fbb4346d6d62e63

6 files changed, 301 insertions, 7 deletions

diff --git a/docs/shell.rst b/docs/shell.rst
index 4cdf9e0..5170999 100644
--- a/docs/shell.rst
+++ b/docs/shell.rst
@@ -334,6 +334,43 @@ to the SIM card.
 This allows for easy interactive modification of file contents.
 
 
+
+BER-TLV EF commands
+-------------------
+
+BER-TLV EFs are files that contain BER-TLV structured data.  Every file can contain any number
+of variable-length IEs (DOs).  The tag within a BER-TLV EF must be unique within the file.
+
+The commands below become enabled only when your currently selected file is of *BER-TLV EF* type.
+
+retrieve_tags
+~~~~~~~~~~~~~
+
+Retrieve a list of all tags present in the currently selected file.
+
+
+retrieve_data
+~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: BerTlvEF.ShellCommands.retrieve_data_parser
+
+
+set_data
+~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: BerTlvEF.ShellCommands.set_data_parser
+
+
+del_data
+~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: BerTlvEF.ShellCommands.del_data_parser
+
+
+
 USIM commands
 -------------
 
diff --git a/pySim-shell.py b/pySim-shell.py
index 659c12c..5b5768a 100755
--- a/pySim-shell.py
+++ b/pySim-shell.py
@@ -38,7 +38,7 @@ from pySim.exceptions import *
 from pySim.commands import SimCardCommands
 from pySim.transport import init_reader, ApduTracer, argparse_add_reader_args
 from pySim.cards import card_detect, Card
-from pySim.utils import h2b, swap_nibbles, rpad, h2s, JsonEncoder
+from pySim.utils import h2b, swap_nibbles, rpad, b2h, h2s, JsonEncoder, bertlv_parse_one
 from pySim.utils import dec_st, sanitize_pin_adm, tabulate_str_list, is_hex, boxed_heading_str
 from pySim.card_handler import card_handler
 
@@ -256,11 +256,19 @@ class PySimCommands(CommandSet):
 			if structure == 'transparent':
 				result = self._cmd.rs.read_binary()
 				self._cmd.poutput("update_binary " + str(result[0]))
-			if structure == 'cyclic' or structure == 'linear_fixed':
+			elif structure == 'cyclic' or structure == 'linear_fixed':
 				num_of_rec = fd['num_of_rec']
 				for r in range(1, num_of_rec + 1):
 					result = self._cmd.rs.read_record(r)
 					self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
+			elif structure == 'ber_tlv':
+				tags = self._cmd.rs.retrieve_tags()
+				for t in tags:
+					result = self._cmd.rs.retrieve_data(t)
+					(tag, l, val) = bertlv_parse_one(h2b(result[0]))
+					self._cmd.poutput("set_data 0x%02x %s" % (t, b2h(val)))
+			else:
+				raise RuntimeError('Unsupported structure "%s" of file "%s"' % (structure, filename))
 		except Exception as e:
 			bad_file_str = '/'.join(df_path_list) + "/" + str(filename) + ", " + str(e)
 			self._cmd.poutput("# bad file: %s" % bad_file_str)
diff --git a/pySim/commands.py b/pySim/commands.py
index 33aec12..0b3d9b6 100644
--- a/pySim/commands.py
+++ b/pySim/commands.py
@@ -5,7 +5,7 @@
 
 #
 # Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
-# Copyright (C) 2010       Harald Welte <laforge@gnumonks.org>
+# Copyright (C) 2010-2021  Harald Welte <laforge@gnumonks.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,7 +23,7 @@
 
 from construct import *
 from pySim.construct import LV
-from pySim.utils import rpad, b2h, sw_match
+from pySim.utils import rpad, b2h, h2b, sw_match, bertlv_encode_len
 from pySim.exceptions import SwMatchError
 
 class SimCardCommands(object):
@@ -269,6 +269,76 @@ class SimCardCommands(object):
 		r = self.select_path(ef)
 		return self.__len(r)
 
+	# TS 102 221 Section 11.3.1 low-level helper
+	def _retrieve_data(self, tag:int, first:bool=True):
+		if first:
+			pdu = '80cb008001%02x' % (tag)
+		else:
+			pdu = '80cb000000'
+		return self._tp.send_apdu_checksw(pdu)
+
+	# TS 102 221 Section 11.3.1
+	def retrieve_data(self, ef, tag:int):
+		"""Execute RETRIEVE DATA.
+
+		Args
+			ef : string or list of strings indicating name or path of transparent EF
+			tag : BER-TLV Tag of value to be retrieved
+		"""
+		r = self.select_path(ef)
+		if len(r[-1]) == 0:
+			return (None, None)
+		total_data = ''
+		# retrieve first block
+		data, sw = self._retrieve_data(tag, first=True)
+		total_data += data
+		while sw == '62f1' or sw == '62f2':
+			data, sw = self._retrieve_data(tag, first=False)
+			total_data += data
+		return total_data, sw
+
+	# TS 102 221 Section 11.3.2 low-level helper
+	def _set_data(self, data:str, first:bool=True):
+		if first:
+			p1 = 0x80
+		else:
+			p1 = 0x00
+		if isinstance(data, bytes) or isinstance(data, bytearray):
+			data = b2h(data)
+		pdu = '80db00%02x%02x%s' % (p1, len(data)//2, data)
+		return self._tp.send_apdu_checksw(pdu)
+
+	def set_data(self, ef, tag:int, value:str, verify:bool=False, conserve:bool=False):
+		"""Execute SET DATA.
+
+		Args
+			ef : string or list of strings indicating name or path of transparent EF
+			tag : BER-TLV Tag of value to be stored
+			value : BER-TLV value to be stored
+		"""
+		r = self.select_path(ef)
+		if len(r[-1]) == 0:
+			return (None, None)
+
+		# in case of deleting the data, we only have 'tag' but no 'value'
+		if not value:
+			return self._set_data('%02x' % tag, first=True)
+
+		# FIXME: proper BER-TLV encode
+		tl = '%02x%s' % (tag, b2h(bertlv_encode_len(len(value)//2)))
+		tlv = tl + value
+		tlv_bin = h2b(tlv)
+
+		first = True
+		total_len = len(tlv_bin)
+		remaining = tlv_bin
+		while len(remaining) > 0:
+			fragment = remaining[:255]
+			rdata, sw = self._set_data(fragment, first=first)
+			first = False
+			remaining = remaining[255:]
+		return rdata, sw
+
 	def run_gsm(self, rand:str):
 		"""Execute RUN GSM ALGORITHM."""
 		if len(rand) != 32:
diff --git a/pySim/filesystem.py b/pySim/filesystem.py
index edfe85d..b3e28ef 100644
--- a/pySim/filesystem.py
+++ b/pySim/filesystem.py
@@ -34,7 +34,7 @@ import argparse
 
 from typing import cast, Optional, Iterable, List, Any, Dict, Tuple
 
-from pySim.utils import sw_match, h2b, b2h, is_hex
+from pySim.utils import sw_match, h2b, b2h, is_hex, auto_int, bertlv_parse_one
 from pySim.construct import filter_dict
 from pySim.exceptions import *
 from pySim.jsonpath import js_path_find, js_path_modify
@@ -914,7 +914,7 @@ class TransRecEF(TransparentEF):
         return b''.join(chunks)
 
 
-class BerTlvEF(TransparentEF):
+class BerTlvEF(CardEF):
     """BER-TLV EF (Entry File) in the smart card filesystem.
     A BER-TLV EF is a binary file with a BER (Basic Encoding Rules) TLV structure
 
@@ -922,6 +922,61 @@ class BerTlvEF(TransparentEF):
     around TransparentEF as a place-holder, so we can already define EFs of BER-TLV
     type without fully supporting them."""
 
+    @with_default_category('BER-TLV EF Commands')
+    class ShellCommands(CommandSet):
+        """Shell commands specific for BER-TLV EFs."""
+        def __init__(self):
+            super().__init__()
+
+        retrieve_data_parser = argparse.ArgumentParser()
+        retrieve_data_parser.add_argument('tag', type=auto_int, help='BER-TLV Tag of value to retrieve')
+        @cmd2.with_argparser(retrieve_data_parser)
+        def do_retrieve_data(self, opts):
+            """Retrieve (Read) data from a BER-TLV EF"""
+            (data, sw) = self._cmd.rs.retrieve_data(opts.tag)
+            self._cmd.poutput(data)
+
+        def do_retrieve_tags(self, opts):
+            """List tags available in a given BER-TLV EF"""
+            tags = self._cmd.rs.retrieve_tags()
+            self._cmd.poutput(tags)
+
+        set_data_parser = argparse.ArgumentParser()
+        set_data_parser.add_argument('tag', type=auto_int, help='BER-TLV Tag of value to set')
+        set_data_parser.add_argument('data', help='Data bytes (hex format) to write')
+        @cmd2.with_argparser(set_data_parser)
+        def do_set_data(self, opts):
+            """Set (Write) data for a given tag in a BER-TLV EF"""
+            (data, sw) = self._cmd.rs.set_data(opts.tag, opts.data)
+            if data:
+                self._cmd.poutput(data)
+
+        del_data_parser = argparse.ArgumentParser()
+        del_data_parser.add_argument('tag', type=auto_int, help='BER-TLV Tag of value to set')
+        @cmd2.with_argparser(del_data_parser)
+        def do_delete_data(self, opts):
+            """Delete  data for a given tag in a BER-TLV EF"""
+            (data, sw) = self._cmd.rs.set_data(opts.tag, None)
+            if data:
+                self._cmd.poutput(data)
+
+
+    def __init__(self, fid:str, sfid:str=None, name:str=None, desc:str=None, parent:CardDF=None,
+                 size={1,None}):
+        """
+        Args:
+            fid : File Identifier (4 hex digits)
+            sfid : Short File Identifier (2 hex digits, optional)
+            name : Brief name of the file, lik EF_ICCID
+            desc : Description of the file
+            parent : Parent CardFile object within filesystem hierarchy
+            size : tuple of (minimum_size, recommended_size)
+        """
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent)
+        self._construct = None
+        self.size = size
+        self.shell_commands = [self.ShellCommands()]
+
 
 class RuntimeState(object):
     """Represent the runtime state of a session with a card."""
@@ -1172,6 +1227,43 @@ class RuntimeState(object):
         data_hex = self.selected_file.encode_record_hex(data)
         return self.update_record(rec_nr, data_hex)
 
+    def retrieve_data(self, tag:int=0):
+        """Read a DO/TLV as binary data.
+
+        Args:
+            tag : Tag of TLV/DO to read
+        Returns:
+            hex string of full BER-TLV DO including Tag and Length
+        """
+        if not isinstance(self.selected_file, BerTlvEF):
+            raise TypeError("Only works with BER-TLV EF")
+        # returns a string of hex nibbles
+        return self.card._scc.retrieve_data(self.selected_file.fid, tag)
+
+    def retrieve_tags(self):
+        """Retrieve tags available on BER-TLV EF.
+
+        Returns:
+            list of integer tags contained in EF
+        """
+        if not isinstance(self.selected_file, BerTlvEF):
+            raise TypeError("Only works with BER-TLV EF")
+        data, sw = self.card._scc.retrieve_data(self.selected_file.fid, 0x5c)
+        tag, length, value = bertlv_parse_one(h2b(data))
+        return list(value)
+
+    def set_data(self, tag:int, data_hex:str):
+        """Update a TLV/DO with given binary data
+
+        Args:
+            tag : Tag of TLV/DO to be written
+            data_hex : Hex string binary data to be written (value portion)
+        """
+        if not isinstance(self.selected_file, BerTlvEF):
+            raise TypeError("Only works with BER-TLV EF")
+        return self.card._scc.set_data(self.selected_file.fid, tag, data_hex, conserve=self.conserve_write)
+
+
 
 
 class FileData(object):
diff --git a/pySim/ts_102_221.py b/pySim/ts_102_221.py
index 2c335a6..bb49fe5 100644
--- a/pySim/ts_102_221.py
+++ b/pySim/ts_102_221.py
@@ -113,10 +113,14 @@ def interpret_file_descriptor(in_hex):
         1: 'transparent',
         2: 'linear_fixed',
         6: 'cyclic',
+     0x39: 'ber_tlv',
     }
     fdb = in_bin[0]
     ftype = (fdb >> 3) & 7
-    fstruct = fdb & 7
+    if fdb & 0xbf == 0x39:
+        fstruct = 0x39
+    else:
+        fstruct = fdb & 7
     out['shareable'] = True if fdb & 0x40 else False
     out['file_type'] = ft_dict[ftype] if ftype in ft_dict else ftype
     out['structure'] = fs_dict[fstruct] if fstruct in fs_dict else fstruct
diff --git a/pySim/utils.py b/pySim/utils.py
index 3e67386..a3cd1b5 100644
--- a/pySim/utils.py
+++ b/pySim/utils.py
@@ -89,6 +89,85 @@ def lpad(s:str, l:int, c='f') -> str:
 def half_round_up(n:int) -> int:
 	return (n + 1)//2
 
+#########################################################################
+# poor man's BER-TLV decoder. To be a more sophisticated OO library later
+#########################################################################
+
+def bertlv_parse_tag(binary:bytes) -> Tuple[dict, bytes]:
+	"""Parse a single Tag value according to ITU-T X.690 8.1.2
+	Args:
+		binary : binary input data of BER-TLV length field
+	Returns:
+		Tuple of ({class:int, constructed:bool, tag:int}, remainder:bytes)
+	"""
+	cls = binary[0] >> 6
+	constructed = True if binary[0] & 0x20 else False
+	tag = binary[0] & 0x1f
+	if tag <= 30:
+		return ({'class':cls, 'constructed':constructed, 'tag': tag}, binary[1:])
+	else: # multi-byte tag
+		tag = 0
+		i = 1
+		last = False
+		while not last:
+			last = False if binary[i] & 0x80 else True
+			tag <<= 7
+			tag |= binary[i] & 0x7f
+			i += 1
+		return ({'class':cls, 'constructed':constructed, 'tag':tag}, binary[i:])
+
+def bertlv_parse_len(binary:bytes) -> Tuple[int, bytes]:
+	"""Parse a single Length value according to ITU-T X.690 8.1.3;
+	only the definite form is supported here.
+	Args:
+		binary : binary input data of BER-TLV length field
+	Returns:
+		Tuple of (length, remainder)
+	"""
+	if binary[0] < 0x80:
+		return (binary[0], binary[1:])
+	else:
+		num_len_oct = binary[0] & 0x7f
+		length = 0
+		for i in range(1, 1+num_len_oct):
+			length <<= 8
+			length |= binary[i]
+		return (length, binary[num_len_oct:])
+
+def bertlv_encode_len(length:int) -> bytes:
+	"""Encode a single Length value according to ITU-T X.690 8.1.3;
+	only the definite form is supported here.
+	Args:
+		length : length value to be encoded
+	Returns:
+		binary output data of BER-TLV length field
+	"""
+	if length < 0x80:
+		return length.to_bytes(1, 'big')
+	elif length <= 0xff:
+		return b'\x81' + length.to_bytes(1, 'big')
+	elif length <= 0xffff:
+		return b'\x82' + length.to_bytes(2, 'big')
+	elif length <= 0xffffff:
+		return b'\x83' + length.to_bytes(3, 'big')
+	elif length <= 0xffffffff:
+		return b'\x84' + length.to_bytes(4, 'big')
+	else:
+		raise ValueError("Length > 32bits not supported")
+
+def bertlv_parse_one(binary:bytes) -> (dict, int, bytes):
+	"""Parse a single TLV IE at the start of the given binary data.
+	Args:
+		binary : binary input data of BER-TLV length field
+	Returns:
+		Tuple of (tag:dict, len:int, remainder:bytes)
+	"""
+	(tagdict, remainder) = bertlv_parse_tag(binary)
+	(length, remainder) = bertlv_parse_len(remainder)
+	return (tagdict, length, remainder)
+
+
+
 # IMSI encoded format:
 # For IMSI 0123456789ABCDE:
 #
@@ -894,6 +973,10 @@ def tabulate_str_list(str_list, width:int = 79, hspace:int = 2, lspace:int = 1,
 		table.append(format_str_row % tuple(str_list_row))
 	return '\n'.join(table)
 
+def auto_int(x):
+    """Helper function for argparse to accept hexadecimal integers."""
+    return int(x, 0)
+
 class JsonEncoder(json.JSONEncoder):
     """Extend the standard library JSONEncoder with support for more types."""
     def default(self, o):