Prefer full match for user agent.

Author: serhiy-storchaka

Lib/test/test_robotparser.py

@@ -260,22 +260,25 @@ class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
 
 
 class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase):
-    # the order of User-agent should be correct. note
-    # that this file is incorrect because "Googlebot" is a
-    # substring of "Googlebot-Mobile"
+    # the order of User-agent should not matter
     robots_txt = """\
 User-agent: Googlebot
 Disallow: /
+Allow: /folder1/
 
 User-agent: Googlebot-Mobile
 Allow: /
+Disallow: /folder1/
     """
     agent = 'Googlebot'
     bad = ['/something.jpg']
+    good = ['/folder1/myfile.html']
 
 
 class UserAgentGoogleMobileTest(UserAgentOrderingTest):
-    agent = 'Googlebot-Mobile'
+    agent = 'Googlebot-mobile'
+    bad = ['/folder1/myfile.html']
+    good = ['/something.jpg']
 
 
 class LongestMatchTest(BaseRobotTest, unittest.TestCase):

Lib/urllib/robotparser.py

@@ -76,6 +76,7 @@ def read(self):
     def _add_entry(self, entry):
         self.entries.append(entry)
         for agent in entry.useragents:
+            agent = agent.lower()
             if agent not in self.groups:
                 self.groups[agent] = entry
             else:
@@ -156,6 +157,9 @@ def parse(self, lines):
             self._add_entry(entry)
 
     def _find_entry(self, useragent):
+        entry = self.groups.get(useragent.lower())
+        if entry is not None:
+            return entry
         for entry in self.groups.values():
             if entry.applies_to(useragent):
                 return entry