Page MenuHomePhabricator

APIError during test_search_where
Closed, ResolvedPublic

Description

_______________________ SearchTestCase.test_search_where _______________________
self = <tests.site_tests.SearchTestCase testMethod=test_search_where>
    def test_search_where(self):
        """Test the site.search() method with 'where' parameter."""
        self.assertEqual(list(self.site.search('wiki', total=10)),
                         list(self.site.search('wiki', total=10, where='text')))
        self.assertLessEqual(len(list(self.site.search('wiki', total=10,
                                                       where='nearmatch'))),
                             len(list(self.site.search('wiki', total=10))))
        for hit in self.site.search('wiki', namespaces=0, total=10,
>                                   get_redirects=True, where='title'):
tests/site_tests.py:1410: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
pywikibot/data/api.py:2753: in __iter__
    self.data = self.request.submit()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = pywikibot.data.api.Request<wpbeta:en->'/w/api.php?inprop=protection&gsrwhat=ti...&iiprop=timestamp|user|comment|url|size|sha1|metadata&uiprop=blockinfo|hasmsg'>
    def submit(self):
        """
            Submit a query and parse the response.
    
            @return: a dict containing data retrieved from api.php
            @rtype: dict
            """
        self._add_defaults()
        if (not config.enable_GET_without_SSL and
                self.site.protocol() != 'https' or
                self.site.is_oauth_token_available()):  # work around T108182
            use_get = False
        elif self.use_get is None:
            if self.action == 'query':
                # for queries check the query module
                modules = set()
                for mod_type_name in ('list', 'prop', 'generator'):
                    modules.update(self._params.get(mod_type_name, []))
            else:
                modules = set([self.action])
            if modules:
                self.site._paraminfo.fetch(modules)
                use_get = all('mustbeposted' not in self.site._paraminfo[mod]
                              for mod in modules)
            else:
                # If modules is empty, just 'meta' was given, which doesn't
                # require POSTs, and is required for ParamInfo
                use_get = True
        else:
            use_get = self.use_get
        while True:
            paramstring = self._http_param_string()
            simulate = self._simulate(self.action)
            if simulate:
                return simulate
            if self.throttle:
                self.site.throttle(write=self.write)
            else:
                pywikibot.log(
                    "Submitting unthrottled action '{0}'.".format(self.action))
            uri = self.site.scriptpath() + "/api.php"
            try:
                if self.mime:
                    (headers, body) = Request._build_mime_request(
                        self._encoded_items(), self.mime_params)
                    use_get = False  # MIME requests require HTTP POST
                else:
                    headers = {'Content-Type': 'application/x-www-form-urlencoded'}
                    if (not self.site.maximum_GET_length() or
                            self.site.maximum_GET_length() < len(paramstring)):
                        use_get = False
                    if use_get:
                        uri = '{0}?{1}'.format(uri, paramstring)
                        body = None
                    else:
                        body = paramstring
    
                pywikibot.debug('API request to {0} (uses get: {1}):\n'
                                'Headers: {2!r}\nURI: {3!r}\n'
                                'Body: {4!r}'.format(self.site, use_get,
                                                     headers, uri, body),
                                _logger)
    
                rawdata = http.request(
                    site=self.site, uri=uri, method='GET' if use_get else 'POST',
                    body=body, headers=headers)
            except Server504Error:
                pywikibot.log(u"Caught HTTP 504 error; retrying")
                self.wait()
                continue
            except Server414Error:
                if use_get:
                    pywikibot.log('Caught HTTP 414 error; retrying')
                    use_get = False
                    self.wait()
                    continue
                else:
                    pywikibot.warning('Caught HTTP 414 error, although not '
                                      'using GET.')
                    raise
            except FatalServerError:
                # This error is not going to be fixed by just waiting
                pywikibot.error(traceback.format_exc())
                raise
            # TODO: what other exceptions can occur here?
            except Exception:
                # for any other error on the http request, wait and retry
                pywikibot.error(traceback.format_exc())
                pywikibot.log(u"%s, %s" % (uri, paramstring))
                self.wait()
                continue
            if not isinstance(rawdata, unicode):
                rawdata = rawdata.decode(self.site.encoding())
            pywikibot.debug((u"API response received from %s:\n" % self.site) +
                            rawdata, _logger)
            if rawdata.startswith(u"unknown_action"):
                raise APIError(rawdata[:14], rawdata[16:])
            try:
                result = json.loads(rawdata)
            except ValueError:
                # if the result isn't valid JSON, there must be a server
                # problem. Wait a few seconds and try again
                pywikibot.warning(
                    "Non-JSON response received from server %s; the server may be down."
                    % self.site)
                # there might also be an overflow, so try a smaller limit
                for param in self._params:
                    if param.endswith("limit"):
                        # param values are stored a list of str
                        value = self._params[param][0]
                        try:
                            self._params[param] = [str(int(value) // 2)]
                            pywikibot.output(u"Set %s = %s"
                                             % (param, self._params[param]))
                        except:
                            pass
                self.wait()
                continue
            if not result:
                result = {}
            if not isinstance(result, dict):
                raise APIError("Unknown",
                               "Unable to process query response of type %s."
                               % type(result),
                               data=result)
    
            if self.action == 'query' and 'userinfo' in result.get('query', ()):
                # if we get passed userinfo in the query result, we can confirm
                # that we are logged in as the correct user. If this is not the
                # case, force a re-login.
                username = result['query']['userinfo']['name']
                if self.site.user() is not None and self.site.user() != username:
                    pywikibot.error(
                        "Logged in as '{actual}' instead of '{expected}'. "
                        "Forcing re-login.".format(
                            actual=username,
                            expected=self.site.user()
                        )
                    )
                    self.site._relogin()
                    continue
    
            self._handle_warnings(result)
    
            if "error" not in result:
                return result
    
            error = result['error'].copy()
            for key in result:
                if key in ('error', 'warnings'):
                    continue
                assert key not in error
                assert isinstance(result[key], basestring), \
                    'Unexpected %s: %r' % (key, result[key])
                error[key] = result[key]
    
            if "*" in result["error"]:
                # help text returned
                result['error']['help'] = result['error'].pop("*")
            code = result['error'].setdefault('code', 'Unknown')
            info = result['error'].setdefault('info', None)
    
            # Older wikis returned an error instead of a warning when
            # the request asked for too many values. If we get this
            # error, assume we are not logged in (we can't check this
            # because the userinfo data is not present) and force
            # a re-login
            if code.endswith('limit'):
                pywikibot.error("Received API limit error. Forcing re-login")
                self.site._relogin()
                continue
    
            # If the user assertion failed, we're probably logged out as well.
            if code == 'assertuserfailed':
                pywikibot.error("User assertion failed. Forcing re-login.")
                self.site._relogin()
                continue
    
            # Lastly, the purge module require a POST if used as anonymous user,
            # but we normally send a GET request. If the API tells us the request
            # has to be POSTed, we're probably logged out.
            if code == 'mustbeposted' and self.action == 'purge':
                pywikibot.error("Received unexpected 'mustbeposted' error. "
                                "Forcing re-login.")
                self.site._relogin()
                continue
    
            if code == "maxlag":
                lag = lagpattern.search(info)
                if lag:
                    pywikibot.log(
                        u"Pausing due to database lag: " + info)
                    self.site.throttle.lag(int(lag.group("lag")))
                    continue
            elif code == 'help' and self.action == 'help':
                # The help module returns an error result with the complete
                # API information. As this data was requested, return the
                # data instead of raising an exception.
                return {'help': {'mime': 'text/plain',
                                 'help': result['error']['help']}}
    
            pywikibot.warning('API error %s: %s' % (code, info))
    
            if code.startswith(u'internal_api_error_'):
                class_name = code[len(u'internal_api_error_'):]
    
                del error['code']  # is added via class_name
                e = APIMWException(class_name, **error)
    
                retry = class_name in ['DBConnectionError',  # T64974
                                       'DBQueryError',  # T60158
                                       'ReadOnlyError'  # T61227
                                       ]
    
                pywikibot.error("Detected MediaWiki API exception %s%s"
                                % (e,
                                   "; retrying" if retry else "; raising"))
                # Due to bug T66958, Page's repr may return non ASCII bytes
                # Get as bytes in PY2 and decode with the console encoding as
                # the rest should be ASCII anyway.
                param_repr = str(self._params)
                if PY2:
                    param_repr = param_repr.decode(config.console_encoding)
                pywikibot.log(u"MediaWiki exception %s details:\n"
                              u"          query=\n%s\n"
                              u"          response=\n%s"
                              % (class_name,
                                 pprint.pformat(param_repr),
                                 result))
    
                if retry:
                    self.wait()
                    continue
    
                raise e
    
            # Phab. tickets T48535, T64126, T68494, T68619
            if code == "failed-save" and \
               self.action == 'wbeditentity' and \
               self._is_wikibase_error_retryable(result["error"]):
                self.wait()
                continue
            # If readapidenied is returned try to login
            if code == 'readapidenied' and self.site._loginstatus in (-3, -1):
                self.site.login()
                continue
            if code == 'badtoken':
                user_tokens = self.site.tokens._tokens[self.site.user()]
                # all token values mapped to their type
                tokens = dict((token, t_type)
                              for t_type, token in user_tokens.items())
                # determine which tokens are bad
                invalid_param = {}
                for name, param in self._params.items():
                    if len(param) == 1 and param[0] in tokens:
                        invalid_param[name] = tokens[param[0]]
                # doesn't care about the cache so can directly load them
                if invalid_param:
                    pywikibot.log(
                        u'Bad token error for {0}. Tokens for "{1}" used in '
                        u'request; invalidated them.'.format(
                            self.site.user(),
                            '", "'.join(sorted(set(invalid_param.values())))))
                    self.site.tokens.load_tokens(set(invalid_param.values()))
                    # fix parameters; lets hope that it doesn't mistake actual
                    # parameters as tokens
                    for name, t_type in invalid_param.items():
                        self[name] = self.site.tokens[t_type]
                    continue
                else:
                    # otherwise couldn't find any … weird there is nothing what
                    # can be done here because it doesn't know which parameters
                    # to fix
                    pywikibot.log(
                        u'Bad token error for {0} but no parameter is using a '
                        u'token. Current tokens: {1}'.format(
                            self.site.user(),
                            ', '.join('{0}: {1}'.format(*e)
                                      for e in user_tokens.items())))
            if 'mwoauth-invalid-authorization' in code:
                if 'Nonce already used' in info:
                    pywikibot.error(
                        'Retrying failed OAuth authentication for {0}: {1}'
                        .format(self.site, info))
                    continue
                raise NoUsername('Failed OAuth authentication for %s: %s'
                                 % (self.site, info))
            # raise error
            try:
                # Due to bug T66958, Page's repr may return non ASCII bytes
                # Get as bytes in PY2 and decode with the console encoding as
                # the rest should be ASCII anyway.
                param_repr = str(self._params)
                if PY2:
                    param_repr = param_repr.decode(config.console_encoding)
                pywikibot.log(u"API Error: query=\n%s"
                              % pprint.pformat(param_repr))
                pywikibot.log(u"           response=\n%s"
                              % result)
    
>               raise APIError(**result['error'])
E               APIError: gsrsearch-title-disabled: title search is disabled [help:See https://en.wikipedia.beta.wmflabs.org/w/api.php for API usage]
pywikibot/data/api.py:2189: APIError

see: https://travis-ci.org/wikimedia/pywikibot-core/jobs/177955830#L1740-L2065

Event Timeline

Magul created this task.Nov 22 2016, 7:16 PM

Change 322974 had a related patch set uploaded (by Magul):
Handle disable error during title search test.

https://gerrit.wikimedia.org/r/322974

Magul added a subscriber: Dalba.Nov 29 2016, 6:28 PM

@Dalba are there any issues with this review? Or can we proceed with merge?

Dalba added a comment.Dec 4 2016, 1:36 AM

@Magul I only see a minor formatting issue. See my comment on gerrit.

Magul added a comment.Dec 7 2016, 10:49 PM

@Dalba patch rebased (merge conflict resolved) and You doubt should be also addressed.

Dalba added a comment.EditedDec 8 2016, 1:05 AM

@Magul

From T152598:

Error codes returned by some modules have changed. Notably, codes from query submodules will no longer include the module prefix, e.g. if you supply a bad continuation parameter for prop=revisions it will report error code 'badcontinue' instead of 'rvbadcontinue'. Other codes may have changed as well, e.g. for list=allpages the error for attempting to use redirects=1 in generator mode is now 'invalidparammix' instead of 'params'.

Or have a look at this error code:
*https://en.wikipedia.beta.wmflabs.org/w/api.php?gsrsearch=wiki&generator=search&gsrwhat=title&action=query

I still think the error code has changed.

Dalba closed this task as Resolved.Dec 8 2016, 10:13 PM

Change 322974 merged by jenkins-bot:
Handle disabled error during title search test

https://gerrit.wikimedia.org/r/322974