Merge pull request #70 from MycroftAI/feature/pymediawiki

Major refactor including switch to pymediawiki

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
+.pytest_cache/
 settings.json

dialog/ca-es/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+volíeu dir
+us agradaria sentir alguna cosa sobre

dialog/da-dk/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+mente du
+kunne du tænke dig at høre om

dialog/de-de/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+meinst du
+möchtest du etwas über

dialog/el-gr/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+μήπως εννοούσες
+θα ήθελες να ακούσεις για

dialog/en-us/connection-error.dialog

@@ -0,0 +1 @@
+I can't connect to Wikipedia. Please try again later.

dialog/en-us/disambiguate-exists.dialog

@@ -0,0 +1 @@
+Did I get that right?

dialog/en-us/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+did you mean
+would you like to hear about

dialog/en-us/no entry found.dialog

@@ -1,2 +1,2 @@
-I can't find a related Wikipedia article
-I'm afraid there isn't any article about that
+I can't find a Wikipedia article about {topic}
+I'm afraid there isn't any article about {topic}

dialog/en-us/question_words.list

@@ -2,3 +2,4 @@ who
 whom
 what
 when
+where

dialog/es-es/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+quieres decir
+quieres escuchar sobre

dialog/es-lm/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+quieres decir
+quieres escuchar sobre

dialog/fa-fa/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+آیا منظورتون?
+می خوای درباره?

dialog/fr-fr/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+voulez-vous dire
+voulez-vous entendre parler de

dialog/gl-es/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+quixeches dicir
+gustaríache oír sobre

dialog/it-it/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+volevi dire
+vuoi sentire qualcosa su

dialog/nl-nl/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+bedoel je
+wil je meer weten over

dialog/pl-pl/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+czy chodziło Ci o
+czy chciałbyś usłyszeć o

dialog/pt-br/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+você quis dizer
+você gostaria de ouvir sobre

dialog/ro-ro/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+vrei să spui
+doriți să aflați despre

dialog/ru-ru/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+ты имел ввиду
+хотел бы ты услышать про

dialog/sv-se/disambiguate-intro.dialog

@@ -0,0 +1,2 @@
+menade du
+vill för höra om

requirements.txt

@@ -1 +1,2 @@
-wikipedia==1.4.0
+pymediawiki~=0.7.0
+requests

test/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2021, Mycroft AI Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

test/behave/steps/wiki.py

@@ -1,41 +1,24 @@
-import time 
+import time
 
-from behave import then, when
+from behave import then
 from mycroft.messagebus import Message
-from mycroft.audio import wait_while_speaking
 from test.integrationtests.voight_kampff import then_wait
 
+
 @then('dialog is stopped')
 def dialog_is_stopped(context):
     def check_dialog_tts_stop(message):
         who = message.data.get('by', '')
         return (who == 'TTS', '')
 
-    def check_dialog_mycroft_stop(message):
-        return True, ''
-
     context.bus.emit(Message('mycroft.audio.speech.stop',
                              data={},
                              context={}))
-    status, debug = then_wait("mycroft.stop.handled", check_dialog_tts_stop, context, 5)
-    if status:
-        return status, debug
-
-    return then_wait("mycroft.stop", check_dialog_mycroft_stop, context, 5)
-
+    status, debug = then_wait("mycroft.stop.handled",
+                              check_dialog_tts_stop, context, 5)
+    return status, debug
 
-@then('"{skill}" should reply with dialog "{dialog}"')
-def then_dialog(context, skill, dialog):
-    def check_dialog(message):
-        utt_dialog = message.data.get('utterance', '')
-        return (utt_dialog == 'Here is your answer from wiki peedia', '')
 
+@then('there will be a short delay')
+def short_sleep(context):
     time.sleep(1)
-    passed, debug = then_wait('speak', check_dialog, context)
-    if not passed:
-        assert_msg = debug
-
-    assert passed, assert_msg or 'Mycroft didn\'t respond'
-
-
-

test/behave/wiki.feature

@@ -4,6 +4,7 @@ Feature: Wikipedia Skill
     Given an english speaking user
      When the user says "<tell me about a person>"
      Then "skill-wiki" should reply with dialog from "searching.dialog"
+     Then there will be a short delay
      And mycroft reply should contain "<person>"
      Then dialog is stopped
 
@@ -23,6 +24,7 @@ Feature: Wikipedia Skill
     Given an english speaking user
      When the user says "<tell me about a person>"
      Then "skill-wiki" should reply with dialog from "searching.dialog"
+     Then there will be a short delay
      And mycroft reply should contain "<person>"
      Then dialog is stopped
 
@@ -34,6 +36,7 @@ Feature: Wikipedia Skill
     Given an english speaking user
      When the user says "<tell me about a place>"
      Then "mycroft-wiki" should reply with dialog from "searching.dialog"
+     Then there will be a short delay
      And mycroft reply should contain "<place>"
      Then dialog is stopped
 
@@ -49,6 +52,7 @@ Feature: Wikipedia Skill
     Given an english speaking user
      When the user says "<tell me about a thing>"
      Then "mycroft-wiki" should reply with dialog from "searching.dialog"
+     Then there will be a short delay
      And mycroft reply should contain "<thing>"
      Then dialog is stopped
 
@@ -64,6 +68,7 @@ Feature: Wikipedia Skill
     Given an english speaking user
      When the user says "<tell me about an idea>"
      Then "mycroft-wiki" should reply with dialog from "searching.dialog"
+     Then there will be a short delay
      And mycroft reply should contain "<idea>"
      Then dialog is stopped
 

test/unit/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2021, Mycroft AI Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

test/unit/test_util.py

@@ -0,0 +1,30 @@
+# Copyright 2021, Mycroft AI Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from wiki.util import remove_nested_parentheses
+
+class TestUtil(unittest.TestCase):
+
+    def test_remove_nested_parentheses(self):
+        test_strings = [
+            ["No change", "No change"],
+            ["a (simple) one", "a  one"],
+            ["Ləmurs (/ˈliːmər/ (listen) LEE-mər)", "Ləmurs "],
+            ["No (end (parentheses)", "No "]
+        ]
+        for input, expected in test_strings:
+            output = remove_nested_parentheses(input)
+            self.assertEqual(output, expected)

test/unit/test_wiki.py

@@ -0,0 +1,115 @@
+# Copyright 2021, Mycroft AI Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from wiki import Wiki, DisambiguationError, MediaWikiPage
+
+class TestWiki(unittest.TestCase):
+    def setUp(self):
+        self.wiki = Wiki('en', auto_more=False)
+        self.test_pages = {}
+        test_titles = ['Elon Musk', 'Lemur', 'Car', 'Nike, Inc.']
+        for title in test_titles:
+            self.test_pages[title] = self.wiki.get_page(title)
+
+    def test_wiki_search(self):
+        results = self.wiki.search('cars')
+        self.assertIsInstance(results, list)
+        self.assertTrue(len(results) > 0)
+        self.assertEqual(results[0], 'Car')
+
+    def test_wiki_search_returns_disambiguation(self):
+        results = self.wiki.search('george church')
+        with self.assertRaises(DisambiguationError):
+            _ = self.wiki.get_page(results[0])
+
+    def test_get_best_image(self):
+        for page in self.test_pages.values():
+            image = self.wiki.get_best_image_url(page, 50)
+            self.assertIsInstance(image, str)
+            self.assertEqual(image[:5], 'https')
+            self.assertEqual(image[-4:], '.jpg')
+
+    # def test_get_disambiguation_from_results(self):
+    #     results = self.wiki.search('george church')
+    #     title = self.wiki.get_disambiguation_page(results)
+    #     self.assertIsInstance(title, str)
+    #     self.assertTrue(len(title) > 0)
+    #     with self.assertRaises(DisambiguationError):
+    #         _ = self.wiki.get_page(title)
+
+    def test_get_page(self):
+        bitcoin_page = self.wiki.get_page('bitcoin')
+        self.assertIsInstance(bitcoin_page, MediaWikiPage)
+        self.assertEqual(bitcoin_page.title, 'Bitcoin')
+        self.assertTrue('crypto' in bitcoin_page.summarize())
+
+    def test_get_random_page(self):
+        random_page = self.wiki.get_random_page()
+        self.assertIsInstance(random_page, MediaWikiPage)
+        self.assertIsInstance(random_page.title, str)
+        self.assertTrue(len(random_page.title) > 0)
+
+    def test_set_language(self):
+        changed = self.wiki.set_language('es')
+        self.assertTrue(changed)
+        self.assertEqual(self.wiki.wiki.language, 'es')
+        page = self.wiki.get_page('barcelona')
+        self.assertEqual(page.title, 'Barcelona')
+        summary = page.summarize(sentences=1)
+        expected_start = 'Barcelona es una ciudad española'
+        self.assertEqual(summary[:len(expected_start)], expected_start)
+
+        # Try to change to unsupported language
+        changed = self.wiki.set_language('notlang')
+        self.assertTrue(not changed)
+        self.assertEqual(self.wiki.wiki.language, 'es')
+
+        # Change to same language
+        changed = self.wiki.set_language('es')
+        self.assertTrue(not changed)
+        self.assertEqual(self.wiki.wiki.language, 'es')
+
+        # Change to default
+        changed = self.wiki.set_language('en')
+        self.assertTrue(changed)
+        self.assertEqual(self.wiki.wiki.language, 'en')
+
+        # Set to default language
+        self.wiki.set_language('fr')
+        self.assertEqual(self.wiki.wiki.language, 'fr')
+        self.wiki.set_language()
+        self.assertEqual(self.wiki.wiki.language, 'en')
+
+    def test_summarize_page(self):
+        for page in self.test_pages.values():
+            summary = self.wiki.summarize_page(page, sentences=2)
+            self.assertIsInstance(summary, str)
+            self.assertTrue('(' not in summary)
+            self.assertTrue(')' not in summary)
+            self.assertTrue('  ' not in summary)
+            self.assertTrue(0 < len(summary) < 500)
+
+    def test_summary_next_lines(self):
+        for page in self.test_pages.values():
+            summary_intro, intro_length = self.wiki.get_summary_intro(page)
+            new_lines = 2
+            summary_follow_up, total_lines = self.wiki.get_summary_next_lines(page, intro_length, new_lines)
+            self.assertIsInstance(summary_follow_up, str)
+            self.assertTrue('(' not in summary_follow_up)
+            self.assertTrue(')' not in summary_follow_up)
+            self.assertTrue('  ' not in summary_follow_up)
+            self.assertTrue(summary_intro not in summary_follow_up)
+            self.assertEqual(total_lines, intro_length + new_lines)