[Freeipa-users] concurrent requests to ipalib app giving network error

[Oğuz Yarımtepe]

Hi,

I have a REST API that is using the ipalib and written with Falcon.
Below is the code or you can check it online here:
http://paste.ubuntu.com/15966308/

from __future__ import print_function
from bson import json_util
import json
import falcon

from ipalib import api as ipaapi
from api.utils.utils import parse_json, check_connection
from api import settings

class Calls(object):

    #@falcon.before(check_connection)
    def on_post(self, req, resp):

        result_json = parse_json(req)
        command_name = result_json["command_name"]
        params = result_json["params"]

        if not hasattr(ipaapi.env, "conf"):
            #TODO: add kinit oguz for exceptional case
            ipaapi.bootstrap_with_global_options(context='satcloud_api')
            ipaapi.finalize()

            if ipaapi.env.in_server:
                ipaapi.Backend.ldap2.connect()
            else:
                ipaapi.Backend.rpcclient.connect()

        #import ipdb
        #ipdb.set_trace()

        command=ipaapi.Command
        command_result=getattr(command,command_name)

        #resp.set_cookie('api_status_cookie', 'True')
        if not params:
            resp.body = json.dumps(command_result())
            resp.status = falcon.HTTP_200
        else:
            if type(params) == dict:
                arguments = []
                kwargs = dict()
                for key, value in params.iteritems():
                    if "arg" in key:
                        arguments.append(value)
                    else:
                        kwargs[key]=value
                try:
                    #for datetime serialization problems better to use bson
                    dump = command_result(*arguments, **kwargs)
                    resp.body = json.dumps(dump, default=json_util.default)
                    #resp.body = json.dumps(command_result(*arguments,
**kwargs))
                    resp.status = falcon.HTTP_200
                except UnicodeDecodeError:
                    resp.body = json.dumps(dump, default=json_util.default,
encoding='latin1')
                    resp.status = falcon.HTTP_200
                except Exception as e:
                    resp.status = falcon.HTTP_BAD_REQUEST
                    resp.body = json.dumps({"description": e.message,
"title": "Dublicate entry"})
                            #raise falcon.HTTPBadRequest(title="Dublicate
entry",
                                        #            description=e,
                                        #            href=settings.__docs__)
            else:
                dump = command_result(params)
                resp.body = json.dumps(dump, default=json_util.default)
                #resp.body = json.dumps(command_result(params))
                resp.status = falcon.HTTP_200


Basically i am making concurrent calls to this rest api and i am getting

Network error: http://paste.ubuntu.com/15966347/

ipa: INFO: Forwarding 'user_find' to json server '
https://ipa.foo.com/ipa/json'
ipa: INFO: Forwarding 'netgroup_find' to json server '
https://ipa.foo.com/ipa/json'
[pid: 5450|app: 0|req: 9/14] 10.102.235.77 () {34 vars in 463 bytes} [Thu
Apr 21 17:43:22 2016] POST /v1/ipa/calls => generated 2324 bytes in 227
msecs (HTTP/1.1 200) 8 headers in 459 bytes (1 switches on core 0)
Traceback (most recent call last):
  File "falcon/api.py", line 213, in falcon.api.API.__call__
(falcon/api.c:2521)
  File "falcon/api.py", line 182, in falcon.api.API.__call__
(falcon/api.c:2118)
  File "./api/resources/ipa/calls.py", line 38, in on_post
    resp.body = json.dumps(command_result())
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 443, in
__call__
    ret = self.run(*args, **options)
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 761, in
run
    return self.forward(*args, **options)
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 782, in
forward
    return self.Backend.rpcclient.forward(self.name, *args, **kw)
  File "/usr/lib/python2.7/site-packages/ipalib/rpc.py", line 935, in
forward
    raise NetworkError(uri=server, error=e.errmsg)
ipalib.errors.NetworkError: cannot connect to 'https://ipa.foo.com/ipa/json':
Internal Server Error
[pid: 5451|app: 0|req: 3/15] 10.102.235.77 () {34 vars in 463 bytes} [Thu
Apr 21 17:43:22 2016] POST /v1/ipa/calls => generated 0 bytes in 1421 msecs
(HTTP/1.1 500) 0 headers in 0 bytes (0 switches on core 0)


This is how a concurrent request is being sent:
#!/usr/bin/env python

from multiprocessing import Process, Pool
import time
import urllib2

def millis():
  return int(round(time.time() * 1000))

def http_get(url):
  start_time = millis()
  request = urllib2.Request(url, headers={"Content-Type":
"application/json", "Origin": "http://ipa.foo.com", "Authorization":
"{'token':
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzcnYiOiJpcGEuc2F0Y2xvdWQuY29tLnRyIiwic3ViIjoiMGU1ZGZkNDc3N2I2NmNhOTU3ZTc4ZmJhZjMxNjYxMmEifQ.cr8cNy7zgQkY-q7UUyTCNPCjGlmz-LCCzUYSUV9P694'}"})
  result = {"url": url, "data": urllib2.urlopen(request,
timeout=10).read()[:100]}
  #result = {"url": url, "data": urllib2.urlopen(request, timeout=5).read()}
  print url + " took " + str(millis() - start_time) + " ms"
  return result


urls = ['http://api.foo.com:8888/v1/users', '
http://api.foo.com:8888/v1/organizations']

pool = Pool(processes=2)

start_time = millis()
results = pool.map(http_get, urls)

print "\nTotal took " + str(millis() - start_time) + " ms\n"

for result in results:
  print result

I am confused about the reason of the error. Any idea?


-- 
Oğuz Yarımtepe
http://about.me/oguzy
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://listman.redhat.com/archives/freeipa-users/attachments/20160421/820a9c3a/attachment.htm>