readme

.gitignore

@@ -4,4 +4,5 @@ results/*
 *-bk.py
 *.sh
 trial.py
-.DS_store
+.DS_stores
+*/.DS_stores

README.md

@@ -20,12 +20,16 @@
 [![MIT License][license-shield]][license-url]
 <!-- [![LinkedIn][linkedin-shield]][linkedin-url] -->
 <!-- :magic_wand: -->
-<!-- $\ddot{a}$ -->
+<!-- $\ddot{a}$ 🪄-->
 
 <br />
 <div align="center">
+
+<a href="https://github.com/othneildrew/Best-README-Template">
+    <img src="imgs/logo.png" alt="Logo" width="200" height="200">
+</a>
 <h2 align="center">
-🪄MAg&IumlC: Investigation of Large Language Model Powered Multi-Agent in Cognition, Adaptability, Rationality and Collaboration</h2>
+MAg&IumlC: Investigation of Large Language Model Powered Multi-Agent in Cognition, Adaptability, Rationality and Collaboration</h2>
 
   <p align="center">
     A competition-based benchmark with quantitative metrics for Large Language Model Powered Multi-agent system.
@@ -37,7 +41,7 @@
     <a href="https://arxiv.org/abs/2311.08562"> 📊 Leaderboard</a>
   </p>
 </div>
-
+[![Product Name Screen Shot][tease]]()
 
 <!-- ## 📌Introducing MAgIC Benchmark🎉
 1. Run our bench with 
@@ -66,12 +70,10 @@
   </ol>
 </details>
 
-
-
 <!-- ABOUT THE PROJECT -->
 ## About The Project
 
-[![Product Name Screen Shot][tease]]()
+
 <!-- [![ghaha][pgm]]() -->
 
 MAgIC provides a benchmark that can quantitatively measure the abilities of Cognition, Adaptability, Rationality and Collaboration of Large Language Models within multi-agent sytems. Our benchmark are based competition on 5 scenarios:
@@ -180,7 +182,9 @@ Lin Xu- [@Lin_Xu_](https://twitter.com/twitter_handle) - [email protected]
 [linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=for-the-badge&logo=linkedin&colorB=555
 [linkedin-url]: https://linkedin.com/in/linkedin_username
 [tease]: imgs/tease_horizontal.jpg
+[demo]: imgs/magic-demo.mp4
 [pgm]: imgs/pgm.jpg
+[logo]: imgs/logo.png
 [leaderboard]: imgs/leaderboard.png
 [Next.js]: https://img.shields.io/badge/next.js-000000?style=for-the-badge&logo=nextdotjs&logoColor=white
 [Next-url]: https://nextjs.org/

chatarena/environments/undercover_competition_pgm.py

@@ -255,29 +255,38 @@ def get_observation(self, player_name=None) -> List[Message]:
             return messages
 
             # return self.message_pool.get_visible_messages(player_name, turn=self._current_turn)
+
+    def _text2vote_rule(self, text):
+        text = text.lower()
+        for name in self.player_names:
+            candidates = [name.lower(), name.lower().replace(" ", ""), name.lower().replace(" ", "_")]
+            if any([candidate in text for candidate in candidates]):
+                return name
+        return None
 
     def _text2vote(self, text) -> str:
         """
         convert text to vote, return a player's name
         """
         # lower = text.lower().replace("[", "").replace("]", "").replace(".", "")
 
-        text = text.lower()
-        for name in self.player_names:
-            candidates = [name.lower(), name.lower().replace(" ", ""), name.lower().replace(" ", "_")]
-            if any([candidate in text for candidate in candidates]):
-                return name
+        ans = self._text2vote_rule(text)
+        if ans is not None:
+            return ans
 
-        prompt = text+"According to the above text, help me identify which Player the text is voting for or accusing as the undercover? reply with \"Player xx\""
+        prompt = text+"According to the above text, help me identify which Player the text is voting for or accusing as the undercover? You must reply with \"Player xx\""
 
         response = openai.ChatCompletion.create(model="gpt-3.5-turbo",
                                         messages=[{'role':'user','content':prompt}],                                    
                                         temperature = 0,
                                         n=3,
                                         max_tokens=5)
         ans = response['choices'][0]['message']['content']
-        # print(ans)
-        return ans
+
+        n_ans = self._text2vote_rule(ans)
+        if n_ans is None:
+            return "Player 1"
+        return n_ans
 
     def _text2pgm(self, text):
 

run_competition_chameleon.py

@@ -25,23 +25,6 @@ def __init__(self):
 
         self.win_count = {"chameleon":0, "non-chameleon":0}
     def run(self,config_dir, competition, path, test_player_model_name, num_of_game=20):
-
-        # if len(sys.argv) > 1:
-        #     if len(sys.argv) != 4:
-        #         print("require 4 arguments!")
-        #     else:
-        #         config_dir=sys.argv[1]
-        #         competition=sys.argv[2]
-        #         save_root=sys.argv[3]
-        #         test_player_model_name =sys.argv[4]
-        #         postfix=""
-
-        # else:
-        #     config_dir="config_release"
-        #     competition = "competition_as_chameleon"
-        #     save_root = "results/chameleon"
-        #     test_player_model_name = "gpt-3.5-turbo"
-        #     postfix="-test"
 
         config_dir=config_dir
         competition = competition