Update legacy XLDST evaluation (ConvLab#186)

* remove transformer cache dir * process data * fix "book" slots processing in MultiWOZ-zh SUMBT, update evaluation results ConvLab#185 * Revert "process data" This reverts commit d17602c23cccb482827d8892554f32eb69dde297. * Revert "remove transformer cache dir" This reverts commit 35873129eb8d45a5bebada63b4549de88b665873.
khalo-sa · Apr 2, 2021 · 67bf789 · 67bf789
1 parent 06f53fd
commit 67bf789
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 14 deletions.
diff --git a/README.md b/README.md
@@ -204,9 +204,9 @@ evaluation of our pre-trained models are: (joint acc.)
 
 | type  | CrossWOZ-en | MultiWOZ-zh |
 | ----- | ----------- | ----------- |
-| val   | 12.4%       | 45.1%       |
-| test  | 12.4%       | 43.5%       |
-| human_val | 10.6%       | 49.4%       |
+| val   | 12.4%       | 48.5%       |
+| test  | 12.4%       | 46.0%       |
+| human_val | 10.6%       | 47.4%       |
 
 `human_val` option will make the model evaluate on the validation set translated by human. 
 

diff --git a/convlab2/dst/sumbt/multiwoz_zh/sumbt.py b/convlab2/dst/sumbt/multiwoz_zh/sumbt.py
@@ -582,28 +582,20 @@ def update(self, user_act=None):
         new_belief_state = copy.deepcopy(prev_state['belief_state'])
         for state in pred_states:
             domain, slot, value = state.split('-', 2)
-
-            if slot not in ['name', 'book']:
-                if domain not in new_belief_state:
-                    if domain == 'bus':
-                        continue
-                    else:
-                        raise Exception(
-                            'Error: domain <{}> not in belief state'.format(domain))
             # slot = REF_SYS_DA[domain.capitalize()].get(slot, slot)
             assert 'semi' in new_belief_state[domain]
             assert 'book' in new_belief_state[domain]
+            domain_dic = new_belief_state[domain]
             if '预订' in slot:
                 assert slot.startswith('预订')
+                slot = slot[2:]
+                assert slot in domain_dic['book']
 
-            domain_dic = new_belief_state[domain]
             if slot in domain_dic['semi']:
                 new_belief_state[domain]['semi'][slot] = value
                 # normalize_value(self.value_dict, domain, slot, value)
             elif slot in domain_dic['book']:
                 new_belief_state[domain]['book'][slot] = value
-            elif slot.lower() in domain_dic['book']:
-                new_belief_state[domain]['book'][slot.lower()] = value
             else:
                 with open('trade_tracker_unknown_slot.log', 'a+') as f:
                     f.write(