From 678f6722c471cd5be714ced3bfe860f9309a2b95 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 9 Dec 2015 14:33:42 -0600 Subject: [PATCH 01/15] adding 'load_unshipped_traders() --- pucauto.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pucauto.py b/pucauto.py index f4c252c..8aa612d 100755 --- a/pucauto.py +++ b/pucauto.py @@ -145,6 +145,22 @@ def send_card(card, add_on=False): return True +def load_unshipped_traders(): + """Build and return a list of members for which we have unshipped cards. + Will be a dictionary from "trader id" : "trader profile name". + """ + + print("Loading unshipped traders...") + DRIVER.get("https://pucatrade.com/trades/active") + DRIVER.find_element_by_css_selector("div.dataTables_filter input").send_keys('Unshipped') + # Wait a bit for the DOM to update after filtering + time.sleep(5) + soup = BeautifulSoup(DRIVER.page_source, "html.parser") + unshipped = dict() + for trader in soup.find_all("a", class_="trader"): + unshipped[trader["href"].replace("/profiles/show/", "")] = trader.contents[0].strip() + return unshipped + def find_and_send_add_ons(): """Build a list of members that have unshipped cards and then send them any @@ -371,6 +387,8 @@ def find_trades(): print_pucauto() print("Logging in...") log_in() + unshipped = load_unshipped_traders() + print("Loading trades page...") goto_trades() wait_for_load() From 7818c9bfd2d907248345a6524b4c83a6c59af1b0 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 9 Dec 2015 18:17:49 -0600 Subject: [PATCH 02/15] track unshipped through all trades; allow add_on arg for complete_trades --- pucauto.py | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/pucauto.py b/pucauto.py index 8aa612d..4029648 100755 --- a/pucauto.py +++ b/pucauto.py @@ -328,40 +328,44 @@ def find_highest_value_bundle(trades): return None -def complete_trades(highest_value_bundle): +def complete_trades(bundle, add_on=False): """Sort the cards by highest value first and then send them all. Args: - highest_value_bundle - The result tuple from find_highest_value_bundle + bundle - tuple of trades for a single trader. + add_on - are these add-on trades for an unshipped bundle? + + return the number of cards successfully sent """ - if not highest_value_bundle: + if not bundle: # No valid bundle was found, give up and restart the main loop - return + return 0 - cards = highest_value_bundle[1]["cards"] + cards = bundle[1]["cards"] # Sort the cards by highest value to make the most valuable trades first. sorted_cards = sorted(cards, key=lambda k: k["value"], reverse=True) - member_name = highest_value_bundle[1]["name"] - member_points = highest_value_bundle[1]["points"] - bundle_value = highest_value_bundle[1]["value"] - print("Found {} card(s) worth {} points to trade to {} who has {} points...".format( - len(sorted_cards), bundle_value, member_name, member_points)) + member_name = bundle[1]["name"] + member_points = bundle[1]["points"] + bundle_value = bundle[1]["value"] + print("Found {}{} card(s) worth {} points to trade to {} who has {} points...".format( + len(sorted_cards), [""," additional"][add_on], + bundle_value, member_name, member_points)) success_count = 0 success_value = 0 for card in sorted_cards: - if send_card(card): + if send_card(card, add_on): success_value += card["value"] success_count += 1 - print("Successfully sent {} out of {} cards worth {} points!".format( - success_count, len(sorted_cards), success_value)) - + print("Successfully {} {} out of {} cards worth {} points!".format( + ["sent","added"][add_on], success_count, len(sorted_cards), success_value)) + return success_count -def find_trades(): +def find_trades(unshipped): """The special sauce. Read the docstrings for the individual functions to figure out how this works.""" @@ -370,13 +374,20 @@ def find_trades(): if CONFIG.get("find_add_ons") and should_check_add_ons(): find_and_send_add_ons() LAST_ADD_ON_CHECK = datetime.now() + + if CONFIG["DEBUG"]: + print("current unshipped list:") + for (id, name) in unshipped.iteritems(): + print(" '{}' -> '{}'".format(id,name)) + goto_trades() wait_for_load() load_trade_list(True) soup = BeautifulSoup(DRIVER.page_source, "html.parser") trades = build_trades_dict(soup) highest_value_bundle = find_highest_value_bundle(trades) - complete_trades(highest_value_bundle) + if complete_trades(highest_value_bundle) >= 1: + unshipped[highest_value_bundle[0]] = highest_value_bundle[1]["name"] # Slow down to not hit PucaTrade refresh limit time.sleep(5) @@ -407,5 +418,7 @@ def find_trades(): wait_for_load() print("Finding trades...") while check_runtime(): - find_trades() + find_trades(unshipped) DRIVER.close() + + # TODO Need to remove filter on low value traders when add-ons allowed. From eb6d886da91d7722bdf6e2949ab27c6594309658 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Mon, 21 Dec 2015 14:47:35 -0600 Subject: [PATCH 03/15] move unshipped trade debug into load_unshipped() --- pucauto.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pucauto.py b/pucauto.py index 4029648..06c68f4 100755 --- a/pucauto.py +++ b/pucauto.py @@ -159,6 +159,10 @@ def load_unshipped_traders(): unshipped = dict() for trader in soup.find_all("a", class_="trader"): unshipped[trader["href"].replace("/profiles/show/", "")] = trader.contents[0].strip() + + if CONFIG.get("DEBUG"): + print("Unshipped Traders List:\n{}".format(pprint.pformat(unshipped))) + return unshipped @@ -311,10 +315,10 @@ def find_highest_value_bundle(trades): """Find the highest value bundle in the trades dictionary. Args: - trades - The result dictionary from build_trades_dict + trades - The result dictionary from build_trades_dict. Returns the highest value bundle, which is a tuple of the (k, v) from - trades. + trades, or None. """ if len(trades) == 0: @@ -374,12 +378,6 @@ def find_trades(unshipped): if CONFIG.get("find_add_ons") and should_check_add_ons(): find_and_send_add_ons() LAST_ADD_ON_CHECK = datetime.now() - - if CONFIG["DEBUG"]: - print("current unshipped list:") - for (id, name) in unshipped.iteritems(): - print(" '{}' -> '{}'".format(id,name)) - goto_trades() wait_for_load() load_trade_list(True) From e0708089eddc4384ccd88e0cd67dae47d97c5232 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Mon, 21 Dec 2015 17:54:50 -0600 Subject: [PATCH 04/15] Additional handling/rescan of unshipped traders list. - Add debug() call that depneds upon DEBUG flag; may reuse when LOGGER fixed. - Add check and reload of unshipped traders list; default: 60 m; min: 5 m. - Add option to control reload/research of trades; default: 60 s; min: 5 s. - Add two new config options; documented in config.example.json: # Interval between reload/rescan for trades (seconds) "reload_trades_interval_s" : 60, # Interval between reloading unshipped traders list (minutes) "reload_unshipped_interval_m" : 60, - Trailing Whitespace cleanup --- config.example.json | 4 ++++ pucauto.py | 47 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/config.example.json b/config.example.json index 3d6025d..de57a32 100644 --- a/config.example.json +++ b/config.example.json @@ -7,6 +7,10 @@ "password" : "yourpassword", # Minimum value of a bundle of cards before PucAuto will auto-ship "min_value" : 500, + # Interval between reload/rescan for trades (seconds) + "reload_trades_interval_s" : 60, + # Interval between reloading unshipped traders list (minutes) + "reload_unshipped_interval_m" : 60, # Look for add ons periodically. # - note that the minimum value has no affect for add-ons. "find_add_ons" : true, diff --git a/pucauto.py b/pucauto.py index 06c68f4..b9778ce 100755 --- a/pucauto.py +++ b/pucauto.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python from __future__ import print_function @@ -10,17 +10,14 @@ from datetime import datetime from bs4 import BeautifulSoup - with open("config.json") as config: CONFIG = json.load(config) - DRIVER = webdriver.Firefox() - START_TIME = datetime.now() LAST_ADD_ON_CHECK = START_TIME - +LAST_UNSHIPPED_CHECK = START_TIME def print_pucauto(): """Print logo and version number.""" @@ -39,6 +36,10 @@ def print_pucauto(): """) +def debug(str): + if CONFIG.get("DEBUG"): + print("DEBUG: ", str) + def wait_for_load(): """Wait for PucaTrade's loading spinner to dissappear.""" @@ -145,11 +146,23 @@ def send_card(card, add_on=False): return True + +def unshipped_reload_due(interval_minutes): + """Return True if we should reload unshipped traders list. + Presumably, we want to do this periodically, especially when we are physically shipping cards. + """ + + global LAST_UNSHIPPED_CHECK + return (datetime.now() - LAST_UNSHIPPED_CHECK).total_seconds() / 60 >= interval_minutes + + def load_unshipped_traders(): """Build and return a list of members for which we have unshipped cards. Will be a dictionary from "trader id" : "trader profile name". """ + global LAST_UNSHIPPED_CHECK + print("Loading unshipped traders...") DRIVER.get("https://pucatrade.com/trades/active") DRIVER.find_element_by_css_selector("div.dataTables_filter input").send_keys('Unshipped') @@ -160,9 +173,8 @@ def load_unshipped_traders(): for trader in soup.find_all("a", class_="trader"): unshipped[trader["href"].replace("/profiles/show/", "")] = trader.contents[0].strip() - if CONFIG.get("DEBUG"): - print("Unshipped Traders List:\n{}".format(pprint.pformat(unshipped))) - + debug("Unshipped Traders List:\n{}".format(pprint.pformat(unshipped))) + LAST_UNSHIPPED_CHECK = datetime.now() return unshipped @@ -386,14 +398,20 @@ def find_trades(unshipped): highest_value_bundle = find_highest_value_bundle(trades) if complete_trades(highest_value_bundle) >= 1: unshipped[highest_value_bundle[0]] = highest_value_bundle[1]["name"] - # Slow down to not hit PucaTrade refresh limit - time.sleep(5) if __name__ == "__main__": """Start Pucauto.""" print_pucauto() + + # sleep for refresh interval (seconds); default: 60; min: 5 + refresh_interval = max(5,CONFIG.get("reload_trades_interval_s") or 60) + # interval for reloading unshipped traders (minutes); default: 60; min 5 + unshipped_interval = max(5,CONFIG.get("reload_unshipped_interval_m") or 60) + # interval for chekcing for add-on trades (minutes); default: 20; min 5 + addon_check_interval = max(5,CONFIG.get("minutes_between_add_ons_check") or 20) + print("Logging in...") log_in() unshipped = load_unshipped_traders() @@ -414,9 +432,16 @@ def find_trades(unshipped): wait_for_load() sort_by_member_points() wait_for_load() - print("Finding trades...") + print("Finding trades ({} sec interval)...".format(refresh_interval)) while check_runtime(): + # reload unshipped traders periodically + if unshipped_reload_due(unshipped_interval): + unshipped = load_unshipped_traders() + # find and send trades find_trades(unshipped) + # sleep for refresh interval (seconds) + time.sleep(refresh_interval) + DRIVER.close() # TODO Need to remove filter on low value traders when add-ons allowed. From 6434ff21e7bb87612f54b309a290a67a03a4f032 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Mon, 21 Dec 2015 18:09:32 -0600 Subject: [PATCH 05/15] unify trade output; more debugging --- pucauto.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pucauto.py b/pucauto.py index b9778ce..ec56013 100755 --- a/pucauto.py +++ b/pucauto.py @@ -127,22 +127,16 @@ def send_card(card, add_on=False): try: DRIVER.find_element_by_id("confirm-trade-button") except Exception: - if not add_on: - reason = DRIVER.find_element_by_tag_name("h3").text - # Indented for readability because this is part of a bundle and there - # are header/footer messages - print(" Failed to send {}. Reason: {}".format(card["name"], reason)) + # FAILED - output indented for readability w.r.t header/footer messages from elsewhere. + reason = DRIVER.find_element_by_tag_name("h3").text + print(" Failed to send '{}'. Reason: {}".format(card["name"], reason)) return False # Then go to the /trades/confirm/******* page to confirm the trade DRIVER.get(card["href"].replace("sendcard", "confirm")) - if add_on: - print("Added on {} to an unshipped trade for {} PucaPoints!".format(card["name"], card["value"])) - else: - # Indented for readability because this is part of a bundle and there - # are header/footer messages - print(" Sent {} for {} PucaPoints!".format(card["name"], card["value"])) + # SUCCESS - output indented for readability w.r.t header/footer messages from elsewhere. + print(" {} '{}' for {} PucaPoints!".format(["Sent","Added"][add_on], card["name"], card["value"])) return True @@ -236,15 +230,19 @@ def load_trade_list(partial=False): old_scroll_y = 0 while True: + debug("Scrolling trades table") if partial: try: lowest_visible_points = int( DRIVER.find_element_by_css_selector(".cards-show tbody tr:last-of-type td.points").text) + debug("Lowest member points visible in trades table: {}".format(lowest_visible_points)) except: # We reached the bottom lowest_visible_points = -1 if lowest_visible_points < CONFIG["min_value"]: # Stop loading because there are no more members with points above min_value + debug("Curtail loading trades table; lowest: {} <= {} minimum trade.".format( + lowest_visible_points, CONFIG["min_value"])) break DRIVER.execute_script("window.scrollBy(0, 5000);") @@ -255,6 +253,7 @@ def load_trade_list(partial=False): break else: old_scroll_y = new_scroll_y + debug("Finished scrolling trades table") def build_trades_dict(soup): @@ -337,6 +336,7 @@ def find_highest_value_bundle(trades): return None highest_value_bundle = max(six.iteritems(trades), key=lambda x: x[1]["value"]) + #debug("Highest value bundle:\n{}".format(pprint.pformat(highest_value_bundle))) if highest_value_bundle[1]["value"] >= CONFIG["min_value"]: return highest_value_bundle From 5644e3b106c3934cb74791cb1fa15ab82ce40898 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 23 Dec 2015 00:06:36 -0600 Subject: [PATCH 06/15] use unshipped traders to find add-ons while seraching for new bundles too. --- pucauto.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pucauto.py b/pucauto.py index ec56013..bb667b6 100755 --- a/pucauto.py +++ b/pucauto.py @@ -256,7 +256,7 @@ def load_trade_list(partial=False): debug("Finished scrolling trades table") -def build_trades_dict(soup): +def build_trades_dict(soup, unshipped): """Iterate through the rows in the table on the /trades page and build up a dictionary. @@ -292,12 +292,12 @@ def build_trades_dict(soup): for row in soup.find_all("tr", id=lambda x: x and x.startswith("uc_")): member_points = int(row.find("td", class_="points").text) - if member_points < CONFIG["min_value"]: - # This member doesn't have enough points so move on to next row - continue member_link = row.find("td", class_="member").find("a", href=lambda x: x and x.startswith("/profiles")) - member_name = member_link.text.strip() member_id = member_link["href"].replace("/profiles/show/", "") + member_name = member_link.text.strip() + if (member_id not in unshipped and member_points < CONFIG["min_value"]) : + # This member isn't possible add on and doesn't have enough points so move on to next row + continue card_name = row.find("a", class_="cl").text card_value = int(row.find("td", class_="value").text) card_href = "https://pucatrade.com" + row.find("a", class_="fancybox-send").get("href") @@ -306,6 +306,8 @@ def build_trades_dict(soup): "value": card_value, "href": card_href } + if member_id in unshipped: + debug("found add-on card for '{}':\n{}".format(member_name,pprint.pformat(card))) if trades.get(member_id): # Seen this member before in another row so just add another card trades[member_id]["cards"].append(card) @@ -394,7 +396,7 @@ def find_trades(unshipped): wait_for_load() load_trade_list(True) soup = BeautifulSoup(DRIVER.page_source, "html.parser") - trades = build_trades_dict(soup) + trades = build_trades_dict(soup, unshipped) highest_value_bundle = find_highest_value_bundle(trades) if complete_trades(highest_value_bundle) >= 1: unshipped[highest_value_bundle[0]] = highest_value_bundle[1]["name"] @@ -415,6 +417,7 @@ def find_trades(unshipped): print("Logging in...") log_in() unshipped = load_unshipped_traders() + print("Loading trades page...") goto_trades() wait_for_load() From 99026120a5301e3c7953d9f2fe396bdda9f43c36 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 23 Dec 2015 11:25:04 -0600 Subject: [PATCH 07/15] always check and send add-ons if we have any unshipped traders --- pucauto.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/pucauto.py b/pucauto.py index bb667b6..ba68ee5 100755 --- a/pucauto.py +++ b/pucauto.py @@ -383,23 +383,40 @@ def complete_trades(bundle, add_on=False): return success_count +def find_add_on_bundles(trades, unshipped): + """Return subset of 'trades' for which we are have unshipped cards + to those traders in the 'unshipped' dictionary. + """ + + # interesting syntactic alternatives: http://stackoverflow.com/questions/2844516 + return {id: b for id, b in trades.iteritems() if id in unshipped} + + def find_trades(unshipped): """The special sauce. Read the docstrings for the individual functions to figure out how this works.""" - global LAST_ADD_ON_CHECK - - if CONFIG.get("find_add_ons") and should_check_add_ons(): - find_and_send_add_ons() - LAST_ADD_ON_CHECK = datetime.now() +# global LAST_ADD_ON_CHECK +# +# if CONFIG.get("find_add_ons") and should_check_add_ons(): +# find_and_send_add_ons() +# LAST_ADD_ON_CHECK = datetime.now() + debug("Looking for bundles...") goto_trades() wait_for_load() - load_trade_list(True) + load_trade_list(len(unshipped) <= 0) soup = BeautifulSoup(DRIVER.page_source, "html.parser") trades = build_trades_dict(soup, unshipped) + # Send higest value bundle, and track recipient in unshipped highest_value_bundle = find_highest_value_bundle(trades) if complete_trades(highest_value_bundle) >= 1: unshipped[highest_value_bundle[0]] = highest_value_bundle[1]["name"] + # remove from the trades dict, if it happens to be an add-on trade too + trades.pop(highest_value_bundle[0] + # Send add-on bundles + for bundle in find_add_on_bundles(trades, unshipped).iteritems(): + debug("Add-on bundle found:\n{}".format(pprint.pformat(bundle))) + complete_trades(bundle, True) if __name__ == "__main__": From bbfc09bafa008db9ec30fd47110c53b69c7ae0d5 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 23 Dec 2015 11:57:46 -0600 Subject: [PATCH 08/15] only do a full check for add ons if set in the config file --- pucauto.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/pucauto.py b/pucauto.py index ba68ee5..08980b8 100755 --- a/pucauto.py +++ b/pucauto.py @@ -97,14 +97,14 @@ def check_runtime(): return True -def should_check_add_ons(): - """Return True if we should check for add on trades.""" +def full_addon_check_due(interval_minutes): + """Return True if we should do a FULL check for add on trades.""" - minutes_between_add_ons_check = CONFIG.get("minutes_between_add_ons_check") - if minutes_between_add_ons_check: - return (datetime.now() - LAST_ADD_ON_CHECK).total_seconds() / 60 >= minutes_between_add_ons_check + global LAST_ADD_ON_CHECK + if CONFIG.get("find_add_ons"): + return (datetime.now() - LAST_ADD_ON_CHECK).total_seconds() / 60 >= interval_minutes else: - return True + return False def send_card(card, add_on=False): @@ -218,7 +218,7 @@ def find_and_send_add_ons(): send_card(card, True) -def load_trade_list(partial=False): +def load_trade_list(partial=True): """Scroll to the bottom of the page until we can't scroll any further. PucaTrade's /trades page implements an infinite scroll table. Without this function, we would only see a portion of the cards available for trade. @@ -392,19 +392,18 @@ def find_add_on_bundles(trades, unshipped): return {id: b for id, b in trades.iteritems() if id in unshipped} -def find_trades(unshipped): +def find_trades(unshipped, full_addon_check=False): """The special sauce. Read the docstrings for the individual functions to figure out how this works.""" -# global LAST_ADD_ON_CHECK -# -# if CONFIG.get("find_add_ons") and should_check_add_ons(): -# find_and_send_add_ons() -# LAST_ADD_ON_CHECK = datetime.now() debug("Looking for bundles...") goto_trades() wait_for_load() - load_trade_list(len(unshipped) <= 0) + load_trade_list(not (full_addon_check and len(unshipped) > 0)) + if full_addon_check and len(unshipped) > 0: + debug("looked for add ons - now done") + global LAST_ADD_ON_CHECK + LAST_ADD_ON_CHECK = datetime.now() soup = BeautifulSoup(DRIVER.page_source, "html.parser") trades = build_trades_dict(soup, unshipped) # Send higest value bundle, and track recipient in unshipped @@ -412,8 +411,8 @@ def find_trades(unshipped): if complete_trades(highest_value_bundle) >= 1: unshipped[highest_value_bundle[0]] = highest_value_bundle[1]["name"] # remove from the trades dict, if it happens to be an add-on trade too - trades.pop(highest_value_bundle[0] - # Send add-on bundles + trades.pop(highest_value_bundle[0]) + # Send add-on bundles; this always happens, even if full_addon_check is false. for bundle in find_add_on_bundles(trades, unshipped).iteritems(): debug("Add-on bundle found:\n{}".format(pprint.pformat(bundle))) complete_trades(bundle, True) @@ -457,8 +456,8 @@ def find_trades(unshipped): # reload unshipped traders periodically if unshipped_reload_due(unshipped_interval): unshipped = load_unshipped_traders() - # find and send trades - find_trades(unshipped) + # find and send trades, and perhaps add-ons + find_trades(unshipped, full_addon_check_due(addon_check_interval)) # sleep for refresh interval (seconds) time.sleep(refresh_interval) From 83ae3ca9b60a61171e56521898eca59cf34a90b8 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 23 Dec 2015 14:49:16 -0600 Subject: [PATCH 09/15] reduce minimum add-on-internval to zero --- pucauto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pucauto.py b/pucauto.py index 08980b8..3c95c34 100755 --- a/pucauto.py +++ b/pucauto.py @@ -427,8 +427,8 @@ def find_trades(unshipped, full_addon_check=False): refresh_interval = max(5,CONFIG.get("reload_trades_interval_s") or 60) # interval for reloading unshipped traders (minutes); default: 60; min 5 unshipped_interval = max(5,CONFIG.get("reload_unshipped_interval_m") or 60) - # interval for chekcing for add-on trades (minutes); default: 20; min 5 - addon_check_interval = max(5,CONFIG.get("minutes_between_add_ons_check") or 20) + # interval for chekcing for add-on trades (minutes); default: 20; min 0 + addon_check_interval = max(0.1,CONFIG.get("minutes_between_add_ons_check") or 20) print("Logging in...") log_in() From 8d5b1c17878423865b8284ba88d2c7965dfc5e43 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Sat, 26 Dec 2015 13:27:28 -0600 Subject: [PATCH 10/15] remove dead code; minor debugging updates; ready for review/merge --- pucauto.py | 50 ++------------------------------------------------ 1 file changed, 2 insertions(+), 48 deletions(-) diff --git a/pucauto.py b/pucauto.py index 3c95c34..0f2b2fe 100755 --- a/pucauto.py +++ b/pucauto.py @@ -165,6 +165,7 @@ def load_unshipped_traders(): soup = BeautifulSoup(DRIVER.page_source, "html.parser") unshipped = dict() for trader in soup.find_all("a", class_="trader"): + debug(pprint.pformat(trader.contents)); unshipped[trader["href"].replace("/profiles/show/", "")] = trader.contents[0].strip() debug("Unshipped Traders List:\n{}".format(pprint.pformat(unshipped))) @@ -172,52 +173,6 @@ def load_unshipped_traders(): return unshipped -def find_and_send_add_ons(): - """Build a list of members that have unshipped cards and then send them any - new cards that they may want. Card value is ignored because they are already - being shipped to. So it's fine to add any and all cards on. - """ - - DRIVER.get("https://pucatrade.com/trades/active") - DRIVER.find_element_by_css_selector("div.dataTables_filter input").send_keys('Unshipped') - # Wait a bit for the DOM to update after filtering - time.sleep(5) - - soup = BeautifulSoup(DRIVER.page_source, "html.parser") - - unshipped = set() - for a in soup.find_all("a", class_="trader"): - unshipped.add(a.get("href")) - - goto_trades() - wait_for_load() - load_trade_list() - soup = BeautifulSoup(DRIVER.page_source, "html.parser") - - # Find all rows containing traders from the unshipped set we found earlier - rows = [r.find_parent("tr") for r in soup.find_all("a", href=lambda x: x and x in unshipped)] - - cards = [] - - for row in rows: - card_name = row.find("a", class_="cl").text - card_value = int(row.find("td", class_="value").text) - card_href = "https://pucatrade.com" + row.find("a", class_="fancybox-send").get("href") - card = { - "name": card_name, - "value": card_value, - "href": card_href - } - cards.append(card) - - # Sort by highest value to send those cards first - sorted_cards = sorted(cards, key=lambda k: k["value"], reverse=True) - - - for card in sorted_cards: - send_card(card, True) - - def load_trade_list(partial=True): """Scroll to the bottom of the page until we can't scroll any further. PucaTrade's /trades page implements an infinite scroll table. Without this @@ -401,7 +356,7 @@ def find_trades(unshipped, full_addon_check=False): wait_for_load() load_trade_list(not (full_addon_check and len(unshipped) > 0)) if full_addon_check and len(unshipped) > 0: - debug("looked for add ons - now done") + debug("Completed FULL serach for add ons; updating timer...") global LAST_ADD_ON_CHECK LAST_ADD_ON_CHECK = datetime.now() soup = BeautifulSoup(DRIVER.page_source, "html.parser") @@ -463,4 +418,3 @@ def find_trades(unshipped, full_addon_check=False): DRIVER.close() - # TODO Need to remove filter on low value traders when add-ons allowed. From 49c128ef7f86fe2b91a4be86a9588904a6056e57 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 6 Jan 2016 09:52:46 -0600 Subject: [PATCH 11/15] always print unshipped traders list on refresh of same --- pucauto.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pucauto.py b/pucauto.py index 0f2b2fe..1e0f3a4 100755 --- a/pucauto.py +++ b/pucauto.py @@ -168,7 +168,11 @@ def load_unshipped_traders(): debug(pprint.pformat(trader.contents)); unshipped[trader["href"].replace("/profiles/show/", "")] = trader.contents[0].strip() - debug("Unshipped Traders List:\n{}".format(pprint.pformat(unshipped))) + #debug("Unshipped Traders List:\n{}".format(pprint.pformat(unshipped))) + if unshipped: + print("Unshipped Traders List:\n - {}" + .format("\n - ".join( sorted( map(lambda (k,v): v+" (id: "+k+")", unshipped.iteritems()) ) ))) + LAST_UNSHIPPED_CHECK = datetime.now() return unshipped From 3e8248e4bdb7ed34e35d2af37da7b0e97b17a6f4 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 6 Jan 2016 10:06:24 -0600 Subject: [PATCH 12/15] make highest value bundle send output look like add-on if appropriate --- pucauto.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pucauto.py b/pucauto.py index 1e0f3a4..918680a 100755 --- a/pucauto.py +++ b/pucauto.py @@ -284,7 +284,8 @@ def build_trades_dict(soup, unshipped): def find_highest_value_bundle(trades): - """Find the highest value bundle in the trades dictionary. + """Find the highest value bundle in the trades dictionary + with a trade total greater than our minimum threshold. Args: trades - The result dictionary from build_trades_dict. @@ -367,9 +368,10 @@ def find_trades(unshipped, full_addon_check=False): trades = build_trades_dict(soup, unshipped) # Send higest value bundle, and track recipient in unshipped highest_value_bundle = find_highest_value_bundle(trades) - if complete_trades(highest_value_bundle) >= 1: - unshipped[highest_value_bundle[0]] = highest_value_bundle[1]["name"] - # remove from the trades dict, if it happens to be an add-on trade too + if highest_value_bundle: + if complete_trades(highest_value_bundle, highest_value_bundle[0] in unshipped) >= 1: + unshipped[highest_value_bundle[0]] = highest_value_bundle[1]["name"] + # remove from the trades dictionary regardless - we've already tried. trades.pop(highest_value_bundle[0]) # Send add-on bundles; this always happens, even if full_addon_check is false. for bundle in find_add_on_bundles(trades, unshipped).iteritems(): From 2e661d8eeef2e799927ecabcf6da26611b591afd Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Wed, 2 Mar 2016 13:23:44 -0600 Subject: [PATCH 13/15] fix stupid mistake removing '#' from shebang --- pucauto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pucauto.py b/pucauto.py index 4c2ae29..eb4cbf2 100755 --- a/pucauto.py +++ b/pucauto.py @@ -1,4 +1,4 @@ -!/usr/bin/env python +#!/usr/bin/env python from __future__ import print_function From f058a00a38eff0148e8db6a23cdaadd688440a6a Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Thu, 3 Mar 2016 14:30:38 -0600 Subject: [PATCH 14/15] make new try/except block in unshipped return empty dictionary if failed --- pucauto.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pucauto.py b/pucauto.py index edbff2b..ef800fa 100755 --- a/pucauto.py +++ b/pucauto.py @@ -159,17 +159,18 @@ def load_unshipped_traders(): global LAST_UNSHIPPED_CHECK print("Loading unshipped traders...") + unshipped = dict() + DRIVER.get("https://pucatrade.com/trades/active") try: DRIVER.find_element_by_css_selector("div.dataTables_filter input").send_keys('Unshipped') except NoSuchElementException: - return + return unshipped # Wait a bit for the DOM to update after filtering time.sleep(5) soup = BeautifulSoup(DRIVER.page_source, "html.parser") - unshipped = dict() for trader in soup.find_all("a", class_="trader"): debug(pprint.pformat(trader.contents)); unshipped[trader["href"].replace("/profiles/show/", "")] = trader.contents[0].strip() From cc2671c8cb920080bab01157c2be29f9ff847574 Mon Sep 17 00:00:00 2001 From: Eric Engstrom Date: Thu, 3 Mar 2016 14:52:29 -0600 Subject: [PATCH 15/15] fix debug issue and improve full/partial logic for ease of understanding --- pucauto.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pucauto.py b/pucauto.py index ef800fa..e1a3515 100755 --- a/pucauto.py +++ b/pucauto.py @@ -38,7 +38,7 @@ def print_pucauto(): """) def debug(str): - if CONFIG.get("DEBUG"): + if CONFIG.get("debug"): print("DEBUG: ", str) @@ -184,20 +184,20 @@ def load_unshipped_traders(): return unshipped -def load_trade_list(partial=True): +def load_trade_list(full=False): """Scroll to the bottom of the page until we can't scroll any further. - PucaTrade's /trades page implements an infinite scroll table. Without this + PucaTrade's trades page implements an infinite scroll table. Without this function, we would only see a portion of the cards available for trade. Args: - partial - When True, only loads rows above min_value, thus speeding up - this function + full - When True, load ALL possible trades; otherwise, only load rows + above min_value, thus speeding up the search. """ old_scroll_y = 0 while True: debug("Scrolling trades table") - if partial: + if not full: try: lowest_visible_points = int( DRIVER.find_element_by_css_selector(".cards-show tbody tr:last-of-type td.points").text) @@ -366,11 +366,16 @@ def find_trades(unshipped, full_addon_check=False): debug("Looking for bundles...") goto_trades() wait_for_load() - load_trade_list(not (full_addon_check and len(unshipped) > 0)) - if full_addon_check and len(unshipped) > 0: + + # Do a complete check only when we want to and when we have unshipped trades + if (full_addon_check and len(unshipped) > 0): + load_trade_list(True) debug("Completed FULL serach for add ons; updating timer...") global LAST_ADD_ON_CHECK LAST_ADD_ON_CHECK = datetime.now() + else: + load_trade_list(False) + soup = BeautifulSoup(DRIVER.page_source, "html.parser") trades = build_trades_dict(soup, unshipped) # Send higest value bundle, and track recipient in unshipped