Merge pull request #27 from pglauner/master-faster-weblinkback

Master faster weblinkback

Author: egabancho

modules/weblinkback/lib/weblinkback.py

@@ -157,20 +157,25 @@ def send_pending_linkbacks_notification(linkback_type):
     Send notification emails to all linkback moderators for all pending linkbacks
     @param linkback_type: of CFG_WEBLINKBACK_LIST_TYPE
     """
-    pending_linkbacks = get_all_linkbacks(linkback_type=CFG_WEBLINKBACK_TYPE['TRACKBACK'], status=CFG_WEBLINKBACK_STATUS['PENDING'])
+    pending_linkbacks = get_all_linkbacks(linkback_type=CFG_WEBLINKBACK_TYPE['TRACKBACK'],
+                                          status=CFG_WEBLINKBACK_STATUS['PENDING'],
+                                          limit=CFG_WEBLINKBACK_MAX_LINKBACKS_IN_EMAIL)
 
     if pending_linkbacks:
-        pending_count = len(pending_linkbacks)
+        pending_count = get_all_linkbacks(linkback_type=CFG_WEBLINKBACK_TYPE['TRACKBACK'],
+                                          status=CFG_WEBLINKBACK_STATUS['PENDING'],
+                                          full_count_only=True)
+
         cutoff_text = ''
         if pending_count > CFG_WEBLINKBACK_MAX_LINKBACKS_IN_EMAIL:
-            cutoff_text = ' (Printing only the first %s requests)' % CFG_WEBLINKBACK_MAX_LINKBACKS_IN_EMAIL
+            cutoff_text = '(Printing only the first %s requests)' % CFG_WEBLINKBACK_MAX_LINKBACKS_IN_EMAIL
 
-        content = """There are %(count)s new %(linkback_type)s requests which you should approve or reject%(cutoff)s:
+        content = """There are %(count)s new %(linkback_type)s requests which you should approve or reject %(cutoff)s:
                   """ % {'count': pending_count,
                          'linkback_type': linkback_type,
                          'cutoff': cutoff_text}
 
-        for pending_linkback in pending_linkbacks[0:CFG_WEBLINKBACK_MAX_LINKBACKS_IN_EMAIL]:
+        for pending_linkback in pending_linkbacks:
             content += """
                        For %(recordURL)s from %(origin_url)s.
                        """ % {'recordURL': generate_redirect_url(pending_linkback[2]),
@@ -180,15 +185,14 @@ def send_pending_linkbacks_notification(linkback_type):
             send_email(CFG_SITE_ADMIN_EMAIL, email, 'Pending ' + linkback_type + ' requests', content)
 
 
-def infix_exists_for_url_in_list(url, list_type):
+def infix_exists_for_url_in_list(url, url_list):
     """
     Check if an infix of a url exists in a list
     @param url
-    @param list_type, of CFG_WEBLINKBACK_LIST_TYPE
+    @param url_list
     @return True, False
     """
-    urls = get_url_list(list_type)
-    for current_url in urls:
+    for current_url in url_list:
         if current_url in url:
             return True
     return False
@@ -257,8 +261,8 @@ def perform_sendtrackback(req, recid, url, title, excerpt, blog_name, blog_id, s
                              <message>%s</message>
                          """
 
-    blacklist_match = infix_exists_for_url_in_list(url, CFG_WEBLINKBACK_LIST_TYPE['BLACKLIST'])
-    whitelist_match = infix_exists_for_url_in_list(url, CFG_WEBLINKBACK_LIST_TYPE['WHITELIST'])
+    blacklist_match = infix_exists_for_url_in_list(url, get_url_list(CFG_WEBLINKBACK_LIST_TYPE['BLACKLIST']))
+    whitelist_match = infix_exists_for_url_in_list(url, get_url_list(CFG_WEBLINKBACK_LIST_TYPE['WHITELIST']))
 
     # faulty request, url argument not set
     if url in (CFG_WEBLINKBACK_SUBSCRIPTION_DEFAULT_ARGUMENT_NAME, None, ''):
@@ -336,6 +340,7 @@ def delete_linkbacks_on_blacklist():
     linkbacks.extend(list(get_all_linkbacks(status=CFG_WEBLINKBACK_STATUS['REJECTED'])))
     linkbacks.extend(list(get_all_linkbacks(status=CFG_WEBLINKBACK_STATUS['BROKEN'])))
 
+    blacklist = get_url_list(CFG_WEBLINKBACK_LIST_TYPE['BLACKLIST'])
     for linkback in linkbacks:
-        if infix_exists_for_url_in_list(linkback[1], CFG_WEBLINKBACK_LIST_TYPE['BLACKLIST']):
+        if infix_exists_for_url_in_list(linkback[1], blacklist):
             remove_linkback(linkback[0])

modules/weblinkback/lib/weblinkback_dblayer.py

@@ -27,21 +27,30 @@
 from invenio.textutils import xml_entities_to_utf8
 
 
-def get_all_linkbacks(recid=None, status=None, order=CFG_WEBLINKBACK_ORDER_BY_INSERTION_TIME["ASC"], linkback_type=None):
+def get_all_linkbacks(recid=None,
+                      status=None,
+                      order=CFG_WEBLINKBACK_ORDER_BY_INSERTION_TIME["ASC"],
+                      linkback_type=None,
+                      limit=None,
+                      full_count_only=False):
     """
     Get all linkbacks
     @param recid: of one record, of all if None
     @param status: with a certain status, of all if None
     @param order: order by insertion time either "ASC" or "DESC"
     @param linkback_type: of a certain type, of all if None
+    @param limit: maximum result count, all if None
+    @param full_count_only: return only full result count (does not consider "limit"), result set if False
     @return [(linkback_id,
               origin_url,
               recid,
               additional_properties,
               linkback_type,
               linkback_status,
               insert_time)]
-              in order by id
+              in order by id, up to "limited" results
+
+            OR integer if count_only
     """
 
     header_sql = """SELECT id,
@@ -52,9 +61,10 @@ def get_all_linkbacks(recid=None, status=None, order=CFG_WEBLINKBACK_ORDER_BY_IN
                             status,
                             insert_time
                      FROM lnkENTRY"""
-    conditions = []
-    order_sql = "ORDER by id %s" % order
+    if full_count_only:
+        header_sql = 'SELECT count(id) FROM lnkENTRY'
 
+    conditions = []
     params = []
 
     def add_condition(column, value):
@@ -69,7 +79,17 @@ def add_condition(column, value):
     add_condition('status', status)
     add_condition('type', linkback_type)
 
-    return run_sql(header_sql + ' ' + ' '.join(conditions) + ' ' + order_sql, tuple(params))
+    order_sql = 'ORDER by id %s' % order
+
+    limit_sql = ''
+    if limit:
+        limit_sql = 'LIMIT %s' % limit
+
+    res = run_sql('%s %s %s %s' % (header_sql, ' '.join(conditions), order_sql, limit_sql), tuple(params))
+    if full_count_only:
+        return int(res[0][0])
+    else:
+        return res
 
 
 def approve_linkback(linkbackid, user_info):

modules/weblinkback/lib/weblinkback_regression_tests.py

@@ -224,6 +224,16 @@ def test_get_all_linkbacks2(self):
         """weblinkback - get all linkbacks with a certain status"""
         self.assertEqual(9, len(get_all_linkbacks(status=CFG_WEBLINKBACK_STATUS['PENDING'])))
         self.assertEqual(0, len(get_all_linkbacks(status=CFG_WEBLINKBACK_STATUS['APPROVED'])))
+        self.assertEqual(9, get_all_linkbacks(status=CFG_WEBLINKBACK_STATUS['PENDING'], full_count_only=True))
+        self.assertEqual(0, get_all_linkbacks(status=CFG_WEBLINKBACK_STATUS['APPROVED'], full_count_only=True))
+
+    def test_get_limited_count_of_linkbacks(self):
+        """weblinkback - get limited count of linkbacks"""
+        self.assertEqual(5, len(get_all_linkbacks(status=CFG_WEBLINKBACK_STATUS['PENDING'], limit=5)))
+        self.assertEqual(2, len(get_all_linkbacks(recid=42, status=CFG_WEBLINKBACK_STATUS['PENDING'], limit=2)))
+        # Ignores limit
+        self.assertEqual(9, get_all_linkbacks(status=CFG_WEBLINKBACK_STATUS['PENDING'], limit=5, full_count_only=True))
+        self.assertEqual(5, get_all_linkbacks(recid=42, status=CFG_WEBLINKBACK_STATUS['PENDING'], limit=2, full_count_only=True))
 
     def test_approve_linkback(self):
         """weblinkback - approve linkback"""