add more fields to map-descriptions, remove ild united dict stuff

  * removed ild_storage_data.py. The concept was flawed as it
  did not work for unknown legacy maps.

  * got read_xed_db.py to work with map_info_rdr.py so that it finds
  legacy maps and the opcodes in the patterns of legacy maps

  * had to add the opcode position in line to the map descriptions

  * had to sort the map_info_t objecdts so that map 1 was not
  before maps 2/3/amd-3dnow, so that the searching would work
  properly.

Change-Id: Ia87bfede141199371314d5e8a68213d4536ebfe2
(cherry picked from commit b6c0897bd6e02b2d6fbd7552fc5ce085239727e8)

datafiles/amd-3dnow-maps.txt

@@ -16,5 +16,5 @@
 #  
 #END_LEGAL
 
-#  name     space   esc  mapopc mapno modrm imm8 imm32
-amd-3dnow    legacy  0x0F 0x0F   N/A   yes   yes  no
+#  name     space   esc  mapopc mapno modrm imm8 imm32 opcpos
+amd-3dnow    legacy  0x0F 0x0F   AMD   yes   yes  no   -1

datafiles/amdxop/amd-xop-maps.txt

@@ -16,7 +16,7 @@
 #  
 #END_LEGAL
 
-#  name     space   esc  mapopc mapno modrm imm8 imm32
-amd-xop8     xop     N/A  N/A    8     yes   yes  no
-amd-xop9     xop     N/A  N/A    9     yes   no   no
-amd-xopA     xop     N/A  N/A    0xA   yes   no   yes
+#  name     space   esc  mapopc mapno modrm imm8 imm32 opcpos
+amd-xop8     xop     N/A  N/A    8     yes   yes  no   1
+amd-xop9     xop     N/A  N/A    9     yes   no   no   1
+amd-xopA     xop     N/A  N/A    0xA   yes   no   yes  1

datafiles/avx/vex-maps.txt

@@ -16,7 +16,7 @@
 #  
 #END_LEGAL
 
-#  name     space   esc  mapopc mapno modrm imm8 imm32
-vex-map1     vex     N/A  N/A    1     yes   no   no
-vex-map2     vex     N/A  N/A    2     yes   no   no
-vex-map3     vex     N/A  N/A    3     yes   yes  no
+#  name     space   esc  mapopc mapno modrm imm8 imm32 opcpos
+vex-map1     vex     N/A  N/A    1     yes   no   no   1
+vex-map2     vex     N/A  N/A    2     yes   no   no   1
+vex-map3     vex     N/A  N/A    3     yes   yes  no   1

datafiles/avx512f/evex-maps.txt

@@ -16,7 +16,7 @@
 #  
 #END_LEGAL
 
-#  name     space   esc  mapopc mapno modrm imm8 imm32
-evex-map1    evex    N/A  N/A    1     yes   no   no
-evex-map2    evex    N/A  N/A    2     yes   no   no
-evex-map3    evex    N/A  N/A    3     yes   yes  no
+#  name     space   esc  mapopc mapno modrm imm8 imm32 opcpos
+evex-map1    evex    N/A  N/A    1     yes   no   no   1
+evex-map2    evex    N/A  N/A    2     yes   no   no   2
+evex-map3    evex    N/A  N/A    3     yes   yes  no   3

datafiles/xed-base-maps.txt

@@ -16,8 +16,8 @@
 #  
 #END_LEGAL
 
-#  name     space   esc  mapopc mapno modrm imm8 int32
-legacy-map0  legacy  N/A  N/A    N/A   var   var  no
-legacy-map1  legacy  0x0F N/A    N/A   var   var  no
-legacy-map2  legacy  0x0F 0x38   N/A   yes   no   no
-legacy-map3  legacy  0x0F 0x3A   N/A   yes   yes  no
+#  name     space   esc  mapopc mapno modrm imm8 int32  opcpos
+legacy-map0  legacy  N/A  N/A    0     var   var  no    0
+legacy-map1  legacy  0x0F N/A    1     var   var  no    1
+legacy-map2  legacy  0x0F 0x38   2     yes   no   no    2
+legacy-map3  legacy  0x0F 0x3A   3     yes   yes  no    2

pysrc/gen_chip_list.py

@@ -49,10 +49,7 @@ def work(args):  # main function
     msgb("READING XED DB")
     (chips, chip_db) = chipmodel.read_database(args.chip_filename)
 
-    xeddb = read_xed_db.xed_reader_t(args.state_bits_filename,
-                                     args.instructions_filename,
-                                     args.widths_filename,
-                                     args.element_types_filename)
+    xeddb = gen_setup.read_db(args)
 
     isasets = set()
     for r in xeddb.recs:

pysrc/gen_cpuid.py

@@ -33,12 +33,7 @@ def msgb(b,s=''):
 def work(args):  # main function
     msgb("READING XED DB")
 
-
-    xeddb = read_xed_db.xed_reader_t(args.state_bits_filename,
-                                     args.instructions_filename,
-                                     args.widths_filename,
-                                     args.element_types_filename,
-                                     args.cpuid_filename)
+    xeddb = gen_setup.read_db(args)
 
     xeddb.recs.sort(key=lambda x:x.iclass)
     for r in xeddb.recs:

pysrc/gen_dump.py

@@ -38,11 +38,7 @@ def msge(b,s=''):
 def work(args):  # main function
     msge("READING XED DB")
 
-    xeddb = read_xed_db.xed_reader_t(args.state_bits_filename,
-                                     args.instructions_filename,
-                                     args.widths_filename,
-                                     args.element_types_filename,
-                                     args.cpuid_filename)
+    xeddb = gen_setup.read_db(args)
 
     xeddb.recs.sort(key=lambda x:x.iclass)
     for r in xeddb.recs:

pysrc/gen_flags.py

@@ -35,10 +35,8 @@ def msg(b):
 def work(args):  # main function
     msgb("READING XED DB")
 
-    xeddb = read_xed_db.xed_reader_t(args.state_bits_filename,
-                                     args.instructions_filename,
-                                     args.widths_filename,
-                                     args.element_types_filename)
+    xeddb = gen_setup.read_db(args)
+
     d = {}
     for r in xeddb.recs:
         if hasattr(r,'flags'):

pysrc/gen_inst_list.py

@@ -102,11 +102,7 @@ def work(args):  # main function
     msgb("READING XED DB")
     (chips, chip_db) = chipmodel.read_database(args.chip_filename)
 
-    xeddb = read_xed_db.xed_reader_t(args.state_bits_filename,
-                                     args.instructions_filename,
-                                     args.widths_filename,
-                                     args.element_types_filename)
-
+    xeddb = gen_setup.read_db(args)
 
     (insts,undoc) = check(args.chip, xeddb, chip_db)
     ilist = list(set( [ x.iclass for x in insts ] ))

pysrc/gen_newer_inst_list.py

@@ -40,10 +40,7 @@ def work(args):  # main function
     gen_setup.msge("READING XED DB")
     (chips, chip_db) = chipmodel.read_database(args.chip_filename)
 
-    xeddb = read_xed_db.xed_reader_t(args.state_bits_filename,
-                                     args.instructions_filename,
-                                     args.widths_filename,
-                                     args.element_types_filename)
+    xeddb = gen_setup.read_db(args)
 
     # base chip instr
     bi = chip_list(args.basechip, xeddb, chip_db)

pysrc/gen_operands.py

@@ -28,10 +28,7 @@
 def work(args):  # main function
     gen_setup.msge("READING XED DB")
 
-    xeddb = read_xed_db.xed_reader_t(args.state_bits_filename,
-                                     args.instructions_filename,
-                                     args.widths_filename,
-                                     args.element_types_filename)
+    xeddb = gen_setup.read_db(args)
 
     histo = collections.defaultdict(int)
     for r in xeddb.recs:

pysrc/gen_setup.py

@@ -54,13 +54,15 @@ def _check_jn(x,y):
     args.chip_filename          = _check_jn(args.prefix, 'all-chip-models.txt')
     args.widths_filename        = _check_jn(args.prefix, 'all-widths.txt')
     args.element_types_filename = _check_jn(args.prefix, 'all-element-types.txt')
+    args.map_descriptions       = _check_jn(args.prefix, 'all-map-descriptions.txt')
 
 def read_db(args):
     xeddb = read_xed_db.xed_reader_t(args.state_bits_filename,
                                      args.instructions_filename,
                                      args.widths_filename,
                                      args.element_types_filename,
-                                     args.cpuid_filename)
+                                     args.cpuid_filename,
+                                     args.map_descriptions)
     return xeddb
 
 

pysrc/generator.py

@@ -266,11 +266,6 @@ def setup_arg_parser():
                           dest='map_descriptions_input_fn', 
                           default='',
                           help='map descriptions input file')
-    arg_parser.add_option('--gen-ild-storage',
-                          action='store_true', 
-                          dest='gen_ild_storage', 
-                          default=False,
-                          help='Dump the ILD storage file.')
     arg_parser.add_option("--compress-operands", 
                           action="store_true",
                           dest="compress_operands",
@@ -1242,7 +1237,7 @@ def read_structured_flexible(self,lines):
       structured_input_dict = dict(zip(list(structured_input_tags.keys()),
                                        len(structured_input_tags)*[False]))
       found_operands = False
-      filling_extra = False
+      filling_extra = False # when there is more than one pattern/operand/iform per {...} definition
       while 1:
          line = read_str_simple(lines)
          if debug:

pysrc/genutil.py

@@ -259,7 +259,7 @@ def round_up_power_of_two(x):
 
 
 
-make_numeric_decimal_pattern = re.compile(r'^[0-9]+$')
+make_numeric_decimal_pattern = re.compile(r'^[-]?[0-9]+$')
 make_numeric_hex_pattern = re.compile(r'^0[xX][0-9A-Fa-f]+$')
 make_numeric_binary_pattern = re.compile(r'^0b[01_]+$') 
 

pysrc/ild.py

@@ -23,7 +23,6 @@
 import mbuild
 import ild_info
 import ild_storage
-import ild_storage_data
 import ild_eosz
 import ild_easz
 import ild_imm
@@ -54,8 +53,6 @@
 #the debug file
 debug = None
 
-storage_fn = 'ild_storage_data.py'
-
 #FIXME: can we get it from generator.py?
 _xed_3dnow_category = '3DNOW'
 
@@ -183,7 +180,6 @@ def work(agi):
     is_3dnow = _is_amd3dnow(agi)
 
     debug.write("state_space:\n %s" % agi.common.state_space)
-    debug.write("DUMP STORAGE %s\n" % agi.common.options.gen_ild_storage)
 
     # Collect up interesting NT names.
     # We are going to use them when we generate pattern_t objects
@@ -240,43 +236,37 @@ def work(agi):
                                 disp_nts, brdisp_nts, all_state_space)
 
     if ild_patterns:
-        if agi.common.options.gen_ild_storage:
-            #dump the ild_storage_data.py file
-            emit_gen_info_lookup(agi, ild_patterns, is_3dnow, debug)
-            reload(ild_storage_data)
 
         #get ild_storage_t object - the main data structure for ILD
         #essentially a 2D dictionary:
-        #united_lookup[map][opcode] == [ ild_info_t ]
-        #the ild_info_t objects are obtained both from grammar and
-        #ild_storage_data.py file, so that if ild_info_t objects are
-        #defined in ild_storage_data.py file, ILD will have information
-        #about illegal map-opcodes too.
-        united_lookup = _get_united_lookup(ild_patterns,is_3dnow)
+        #ild_tbl[map][opcode] == [ ild_info_t ]
+        #the ild_info_t objects are obtained the grammar
+
+        ild_tbl = _convert_to_ild_storage(ild_patterns, is_3dnow)
 
         #generate modrm lookup tables
-        ild_modrm.work(agi, united_lookup, debug)
+        ild_modrm.work(agi, ild_tbl, debug)
 
         #dump_patterns is for debugging
         if verbosity.vild():
             dump_patterns(ild_patterns,
                           mbuild.join(ild_gendir, 'all_patterns.txt'))
 
 
-        eosz_dict = ild_eosz.work(agi, united_lookup,
+        eosz_dict = ild_eosz.work(agi, ild_tbl,
                     eosz_nts, ild_gendir, debug)
 
-        easz_dict = ild_easz.work(agi, united_lookup,
+        easz_dict = ild_easz.work(agi, ild_tbl,
                     easz_nts, ild_gendir, debug)
 
         #dump operand accessor functions
         agi.operand_storage.dump_operand_accessors(agi)
 
 
         if eosz_dict and easz_dict:
-            ild_imm.work(agi, united_lookup, imm_nts, ild_gendir,
+            ild_imm.work(agi, ild_tbl, imm_nts, ild_gendir,
                          eosz_dict, debug)
-            ild_disp.work(agi, united_lookup, disp_nts, brdisp_nts,
+            ild_disp.work(agi, ild_tbl, disp_nts, brdisp_nts,
                           ild_gendir, eosz_dict, easz_dict, debug)
 
 
@@ -357,38 +347,17 @@ def get_patterns(agi, eosz_nts, easz_nts,
                     patterns.extend(expanded_ptrns)
     return patterns
 
-# FIXME: 2019-10-03 can remove the "uniting" functionality as we do
-# not use the "from_storage" aspect. We still need the stuff stored by
-# map/opcode though. The "from_storage" thing does not work for
-# undefined maps for example. It only workes for undefined instr in
-# exising maps.
-def _get_united_lookup(ptrn_list,is_3dnow):
-    """
-    Combine storage obtained from grammar and from ILD storage
-    @return: ild_info.storage_t object
-    """
-    #build an ild_info_storage_t object from grammar
-    from_grammar = get_info_storage(ptrn_list, 0, is_3dnow)
-
-    #get an ild_info_storage_t object from ild python-based storage
-    from_storage = ild_storage_data.gen_ild_info()
-
-    #FIXME: should we make is_amd=(is_3dnow or from_storage.is_3dnow)?
-    united_lookup = ild_storage.ild_storage_t(is_amd=is_3dnow)
-
-    #unite the lookups, conflicts will be resolved by priority
-    for insn_map in ild_info.get_maps(is_3dnow):
-        for op in range(0, 256):
-            ulist = (from_grammar.get_info_list(insn_map, hex(op)) +
-                     from_storage.get_info_list(insn_map, hex(op)))
-            united_lookup.set_info_list(insn_map, hex(op), ulist)
-    return united_lookup
+def _convert_to_ild_storage(ptrn_list,is_3dnow):
+    """ Store ILD objects by map/opcode
+    @return: ild_info.storage_t object"""
+
+    #convert ptrns to ild_info_t and put in a dict of lists indexed by map/opcode
+    return get_info_storage(ptrn_list, 0, is_3dnow)
 
 
 def get_info_storage(ptrn_list, priority, is_3dnow):
-    """
-    convert list of pattern_t objects to ild_storage_t object
-    """
+    """convert list of pattern_t objects to ild_storage_t object"""
+
     storage = ild_storage.ild_storage_t(is_amd=is_3dnow)
 
     for p in ptrn_list:
@@ -398,37 +367,6 @@ def get_info_storage(ptrn_list, priority, is_3dnow):
     return storage
 
 
-def emit_gen_info_lookup(agi, ptrn_list, is_3dnow, debug):
-    debug.write("DUMPING ILD STORAGE\n")
-    f = codegen.xed_file_emitter_t(agi.common.options.xeddir,
-                                   agi.common.options.xeddir,
-                                   storage_fn,
-                                   shell_file=True)
-
-    storage = get_info_storage(ptrn_list, ild_info.storage_priority, is_3dnow)
-    #list_name = "info_lookup['%s']['%s']"
-    list_name = 'info_list'
-    indent = ' ' * 4
-    f.start()
-    f.add_code("import ild_info")
-    f.add_code("import ild_storage\n\n\n")
-
-    f.add_code("#GENERATED FILE - DO NOT EDIT\n\n\n")
-    f.write("def gen_ild_info():\n")
-    f.write(indent + "storage = ild_storage.ild_storage_t(is_amd=%s)\n" %
-               is_3dnow)
-
-    for insn_map in ild_info.get_dump_maps():
-        for op in range(0, 256):
-            for info in storage.get_info_list(insn_map, hex(op)):
-                f.write("%s#MAP:%s OPCODE:%s\n" %
-                   (indent, info.insn_map, info.opcode))
-                f.write("%sinfo_list = storage.get_info_list('%s','%s')\n" %
-                           (indent, insn_map, hex(op)))
-                emit_add_info_call(info, list_name,
-                                   f, indent)
-    f.write(indent + "return storage\n")
-    f.close()
 
 def emit_add_info_call(info, list_name, f, indent=''):
     s = []