MiscUtil: CERN LDAP plugin improvement

* Improves the search function and introduces paged search to
  avoid exceeding the size limit of the CERN LDAP server.

Signed-off-by: Jochen Klein <j.klein@cern.ch>

Author: jochenklein

modules/miscutil/lib/ldap_cern.py

@@ -1,5 +1,5 @@
 ## This file is part of Invenio.
-## Copyright (C) 2009, 2010, 2011, 2014 CERN.
+## Copyright (C) 2009, 2010, 2011, 2014, 2015 CERN.
 ##
 ## Invenio is free software; you can redistribute it and/or
 ## modify it under the terms of the GNU General Public License as
@@ -22,9 +22,11 @@
 
 import ldap
 import ldap.filter
+from ldap.controls import SimplePagedResultsControl
 
 CFG_CERN_LDAP_URI = "ldap://xldap.cern.ch:389"
 CFG_CERN_LDAP_BASE = "OU=Users,OU=Organic Units,DC=cern,DC=ch"
+CFG_CERN_LDAP_PAGESIZE = 250
 
 _ldap_connection_pool = {}
 
@@ -48,6 +50,47 @@ def _sanitize_input(query):
     return query
 
 
+def _paged_search(ldap_connection, query_filter):
+    """
+    Search the CERN LDAP server.
+    Return a list of user dictionaries (or empty list).
+    """
+    req_ctrl = SimplePagedResultsControl(True, CFG_CERN_LDAP_PAGESIZE, "")
+    msgid = ldap_connection.search_ext(
+        CFG_CERN_LDAP_BASE,
+        ldap.SCOPE_SUBTREE,
+        query_filter,
+        attrlist=None,
+        attrsonly=0,
+        serverctrls=[req_ctrl])
+    result_pages = 0
+    results = []
+
+    while True:
+        rtype, rdata, rmsgid, rctrls = ldap_connection.result3(msgid)
+        results.extend(rdata)
+        result_pages += 1
+
+        pctrls = [
+            c
+            for c in rctrls
+            if c.controlType == SimplePagedResultsControl.controlType
+        ]
+        if pctrls:
+            if pctrls[0].cookie:
+                req_ctrl.cookie = pctrls[0].cookie
+                msgid = ldap_connection.search_ext(
+                    CFG_CERN_LDAP_BASE,
+                    ldap.SCOPE_SUBTREE,
+                    query_filter,
+                    attrlist=None,
+                    attrsonly=0,
+                    serverctrls=[req_ctrl])
+            else:
+                break
+    return results
+
+
 def get_users_info_by_displayName(displayName):
     """
     Query the CERN LDAP server for information about all users whose name
@@ -71,15 +114,13 @@ def get_users_info_by_displayName(displayName):
         return []
 
     try:
-        results = connection.search_st(CFG_CERN_LDAP_BASE, ldap.SCOPE_SUBTREE,
-                                       query_filter, timeout=5)
+        results = _paged_search(connection, query_filter)
     except ldap.LDAPError:
         ## Mmh.. connection error? Let's reconnect at least once just in case
         sleep(1)
         connection = _cern_ldap_login()
         try:
-            results = connection.search_st(CFG_CERN_LDAP_BASE, ldap.SCOPE_SUBTREE,
-                                           query_filter, timeout=5)
+            results = _paged_search(connection, query_filter)
         except ldap.LDAPError:
             # Another error (maybe the LDAP query size is too big, etc.)
             # TODO, if it's needed, here we can return various different
@@ -114,15 +155,13 @@ def get_users_info_by_displayName_or_email(name):
         return []
 
     try:
-        results = connection.search_st(CFG_CERN_LDAP_BASE, ldap.SCOPE_SUBTREE,
-                                       query_filter, timeout=5)
+        results = _paged_search(connection, query_filter)
     except ldap.LDAPError:
         ## Mmh.. connection error? Let's reconnect at least once just in case
         sleep(1)
         connection = _cern_ldap_login()
         try:
-            results = connection.search_st(CFG_CERN_LDAP_BASE, ldap.SCOPE_SUBTREE,
-                                           query_filter, timeout=5)
+            results = _paged_search(connection, query_filter)
         except ldap.LDAPError:
             # Another error (maybe the LDAP query size is too big, etc.)
             # TODO, if it's needed, here we can return various different