diff --git a/pyteal/ast/scratch.py b/pyteal/ast/scratch.py index bc1c43f29..a75a7841f 100644 --- a/pyteal/ast/scratch.py +++ b/pyteal/ast/scratch.py @@ -1,5 +1,7 @@ from typing import cast, TYPE_CHECKING +import traceback # TODO: this is temporary!!! + from pyteal.types import TealType, require_type from pyteal.config import NUM_SLOTS from pyteal.errors import TealInputError, TealInternalError @@ -28,6 +30,30 @@ def __init__(self, requestedSlotId: int | None = None): requestedSlotId (optional): A scratch slot id that the compiler must store the value. This id may be a Python int in the range [0-256). """ + # Compilation ID: + self._cid: int | None = None + # Temporary: + tb = traceback.extract_stack() + + pt_indices = [i + 1 for i, b in enumerate(reversed(tb)) if "/pyteal/" in b[0]] + self.creator_tb_idx = max(pt_indices) + + def back(n): + filename, line_number, function_name, _ = tb[-n] + return { + "loc": f"{filename}:{line_number}:", + "function_name": function_name, + } + + self.creator = back(self.creator_tb_idx) + + def breadcrumb(n): + return back(n)["loc"].split("/")[-1] + + self.breadcrumbs = [ + breadcrumb(i) for i in range(self.creator_tb_idx, 0, -1) + ] + if requestedSlotId is None: self.id = ScratchSlot.nextSlotId ScratchSlot.nextSlotId += 1 @@ -71,7 +97,14 @@ def __repr__(self): return "ScratchSlot({})".format(self.id) def __str__(self): - return "slot#{}".format(self.id) + code = "?" + if (bc := self.breadcrumbs[1]) == 'router.py:1099:': + code = "C" + elif bc == 'router.py:1095:': + code = "B" + arrows = "-".join(x.split(":")[1] for x in self.breadcrumbs) + pid = hash(arrows) + return "({}{})slot#{}:{}".format(code, pid, self.id, arrows) ScratchSlot.__module__ = "pyteal" diff --git a/pyteal/compiler/compiler.py b/pyteal/compiler/compiler.py index c2bd058dc..32f6a6182 100644 --- a/pyteal/compiler/compiler.py +++ b/pyteal/compiler/compiler.py @@ -1,4 +1,6 @@ from dataclasses import dataclass +import threading +import time from typing import Dict, Final, List, Optional, Set, Tuple, cast from algosdk.v2client.algod import AlgodClient @@ -297,6 +299,10 @@ def get_results(self) -> CompileResults: class Compilation: + ready_to_unpause_lock: threading.Lock = threading.Lock() + ready_to_unpause: int = 0 + paused: bool = True + def __init__( self, ast: Expr, @@ -419,6 +425,11 @@ def _compile_impl( ) for start in subroutine_start_blocks.values(): apply_global_optimizations(start, options.optimize, self.version) + + with self.ready_to_unpause_lock: + self.ready_to_unpause += 1 + while self.paused: + time.sleep(0.1) localSlotAssignments: Dict[ Optional[SubroutineDefinition], Set[int] diff --git a/pyteal/compiler/scratchslots.py b/pyteal/compiler/scratchslots.py index 3d316b269..860f114ae 100644 --- a/pyteal/compiler/scratchslots.py +++ b/pyteal/compiler/scratchslots.py @@ -40,7 +40,8 @@ def collectSlotsFromBlock(block: TealBlock): def collectScratchSlots( - subroutineBlocks: Dict[Optional[SubroutineDefinition], TealBlock] + subroutineBlocks: Dict[Optional[SubroutineDefinition], TealBlock], + starting_cid: int | None = None, ) -> Tuple[Set[ScratchSlot], Dict[Optional[SubroutineDefinition], Set[ScratchSlot]]]: """Find and return all referenced ScratchSlots for each subroutine. @@ -53,6 +54,20 @@ def collectScratchSlots( same as subroutineBlocks, and whose values are the local slots of that subroutine. """ + cid: int | None = starting_cid + assert cid is None or cid >= 0, f"negative id not allowed but {starting_cid=}" + + def incrementing() -> bool: + nonlocal cid + return cid is not None + + def next_cid() -> int: + nonlocal cid + if not incrementing(): + return -1 + curr = cid + cid += 1 + return curr subroutineSlots: Dict[Optional[SubroutineDefinition], Set[ScratchSlot]] = dict() @@ -60,6 +75,9 @@ def collectSlotsFromBlock(block: TealBlock, slots: Set[ScratchSlot]): for op in block.ops: for slot in op.getSlots(): slots.add(slot) + if slot._cid is None and (nid := next_cid()) >= 0: + slot._cid = nid + for subroutine, start in subroutineBlocks.items(): slots: Set[ScratchSlot] = set() @@ -107,7 +125,9 @@ def assignScratchSlotsToSubroutines( integers representing the assigned IDs of slots which appear only in that subroutine (subroutine local slots). """ - global_slots, local_slots = collectScratchSlots(subroutineBlocks) + from pyteal import ScratchSlot + + global_slots, local_slots = collectScratchSlots(subroutineBlocks, starting_cid=0) # all scratch slots referenced by the program allSlots: Set[ScratchSlot] = global_slots | cast(Set[ScratchSlot], set()).union( *local_slots.values() @@ -146,7 +166,11 @@ def assignScratchSlotsToSubroutines( raise TealInternalError(msg) from errors[0] nextSlotIndex = 0 - for slot in sorted(allSlots, key=lambda slot: slot.id): + # sorted_slots = sorted(allSlots, key=lambda slot: slot._cid) + sorted_slots = sorted(allSlots, key=lambda slot: slot.id) + x = 42 + print(f"\n{ScratchSlot.nextSlotId=}") + for slot in sorted_slots: # Find next vacant slot that compiler can assign to while nextSlotIndex in slotIds: nextSlotIndex += 1 diff --git a/pyteal/compiler/subroutines.py b/pyteal/compiler/subroutines.py index 403549b1f..ce43572ba 100644 --- a/pyteal/compiler/subroutines.py +++ b/pyteal/compiler/subroutines.py @@ -100,7 +100,7 @@ def spillLocalSlotsDuringRecursion( "Spill to stack" means loading all local slots onto the stack, invoking the subroutine which may result in recursion, then restoring all local slots from the stack. This prevents the local - slots from being modifying by a new recursive invocation of the current subroutine. + slots from being modified by a new recursive invocation of the current subroutine. Args: version: The current program version being assembled. diff --git a/pyteal/ir/tealblock.py b/pyteal/ir/tealblock.py index ebb6bdebf..5402a68a0 100644 --- a/pyteal/ir/tealblock.py +++ b/pyteal/ir/tealblock.py @@ -15,7 +15,6 @@ class TealBlock(ABC): """Represents a basic block of TealComponents in a graph.""" def __init__(self, ops: List[TealOp], root_expr: Optional["Expr"] = None) -> None: - # TODO: do I still need root_expr? self.ops = ops self.incoming: List[TealBlock] = [] self._root_expr = root_expr diff --git a/tests/compile_asserts.py b/tests/compile_asserts.py index 85d660818..4b198ff8e 100644 --- a/tests/compile_asserts.py +++ b/tests/compile_asserts.py @@ -25,29 +25,35 @@ def compile_and_save(approval, version: int, test_name: str) -> tuple[Path, str, return tealdir, name, compiled -def assert_teal_as_expected(path2actual: Path, path2expected: Path): +def assert_teal_as_expected( + path2actual: Path, path2expected: Path, skip_final_assertion=False +) -> tuple[list[str], list[str]]: with open(path2actual, "r") as f: actual_lines = f.readlines() with open(path2expected, "r") as f: expected_lines = f.readlines() - diff = list( - unified_diff( - expected_lines, - actual_lines, - fromfile=str(path2expected), - tofile=str(path2actual), - n=3, + if not skip_final_assertion: + diff = list( + unified_diff( + expected_lines, + actual_lines, + fromfile=str(path2expected), + tofile=str(path2actual), + n=3, + ) ) - ) - assert ( - len(diff) == 0 - ), f"Difference between expected and actual TEAL code:\n\n{''.join(diff)}" + assert ( + len(diff) == 0 + ), f"Difference between expected and actual TEAL code:\n\n{''.join(diff)}" + return expected_lines, actual_lines -def assert_new_v_old(approve_func, version: int, test_name: str): +def assert_new_v_old( + approve_func, version: int, test_name: str, skip_final_assertion=False +) -> tuple[list[str], list[str]]: tealdir, name, compiled = compile_and_save(approve_func, version, test_name) print( @@ -58,4 +64,6 @@ def assert_new_v_old(approve_func, version: int, test_name: str): path2actual = tealdir / (name + ".teal") path2expected = FIXTURES / test_name / (name + ".teal") - assert_teal_as_expected(path2actual, path2expected) + return assert_teal_as_expected( + path2actual, path2expected, skip_final_assertion=skip_final_assertion + ) diff --git a/tests/integration/graviton_test.py b/tests/integration/graviton_test.py index 7f80e70a3..25e38f4fd 100644 --- a/tests/integration/graviton_test.py +++ b/tests/integration/graviton_test.py @@ -24,8 +24,7 @@ GENERATED = PATH / "generated" # TODO: remove these skips after the following issue has been fixed https://github.com/algorand/pyteal/issues/199 -STABLE_SLOT_GENERATION = False -SKIP_SCRATCH_ASSERTIONS = not STABLE_SLOT_GENERATION +SKIP_SCRATCH_ASSERTIONS = True # ---- Helper ---- # @@ -134,11 +133,12 @@ def fib(n): # ---- Blackbox pure unit tests (Skipping for now due to flakiness) ---- # -@pytest.mark.skipif(not STABLE_SLOT_GENERATION, reason="cf. #199") +ISSUE_199_CASES = [exp, square_byref, square, swap, string_mult, oldfac, slow_fibonacci] +@pytest.mark.skipif(SKIP_SCRATCH_ASSERTIONS, reason="cf. #199") @pytest.mark.parametrize( "subr, mode", product( - [exp, square_byref, square, swap, string_mult, oldfac, slow_fibonacci], + ISSUE_199_CASES, [pt.Mode.Application, pt.Mode.Signature], ), ) diff --git a/tests/integration/teal/stability/app_exp.teal b/tests/integration/teal/stability/app_exp_v6.teal similarity index 100% rename from tests/integration/teal/stability/app_exp.teal rename to tests/integration/teal/stability/app_exp_v6.teal diff --git a/tests/integration/teal/stability/app_oldfac.teal b/tests/integration/teal/stability/app_oldfac_v6.teal similarity index 100% rename from tests/integration/teal/stability/app_oldfac.teal rename to tests/integration/teal/stability/app_oldfac_v6.teal diff --git a/tests/integration/teal/stability/app_slow_fibonacci.teal b/tests/integration/teal/stability/app_slow_fibonacci_v6.teal similarity index 100% rename from tests/integration/teal/stability/app_slow_fibonacci.teal rename to tests/integration/teal/stability/app_slow_fibonacci_v6.teal diff --git a/tests/integration/teal/stability/app_square_byref.teal b/tests/integration/teal/stability/app_square_byref_v6.teal similarity index 87% rename from tests/integration/teal/stability/app_square_byref.teal rename to tests/integration/teal/stability/app_square_byref_v6.teal index b6c39b9df..25393281a 100644 --- a/tests/integration/teal/stability/app_square_byref.teal +++ b/tests/integration/teal/stability/app_square_byref_v6.teal @@ -1,15 +1,15 @@ #pragma version 6 txna ApplicationArgs 0 btoi -store 1 -pushint 1 // 1 +store 0 +pushint 0 // 0 callsub squarebyref_0 pushint 1337 // 1337 -store 0 -load 0 +store 1 +load 1 itob log -load 0 +load 1 return // square_byref diff --git a/tests/integration/teal/stability/app_square.teal b/tests/integration/teal/stability/app_square_v6.teal similarity index 100% rename from tests/integration/teal/stability/app_square.teal rename to tests/integration/teal/stability/app_square_v6.teal diff --git a/tests/integration/teal/stability/app_string_mult.teal b/tests/integration/teal/stability/app_string_mult_v6.teal similarity index 87% rename from tests/integration/teal/stability/app_string_mult.teal rename to tests/integration/teal/stability/app_string_mult_v6.teal index cf135fc8e..2da35ce7b 100644 --- a/tests/integration/teal/stability/app_string_mult.teal +++ b/tests/integration/teal/stability/app_string_mult_v6.teal @@ -1,37 +1,37 @@ #pragma version 6 intcblock 1 txna ApplicationArgs 0 -store 1 -intc_0 // 1 +store 0 +pushint 0 // 0 txna ApplicationArgs 1 btoi callsub stringmult_0 -store 0 -load 0 +store 1 +load 1 log -load 0 +load 1 len return // string_mult stringmult_0: -store 3 store 2 +store 3 intc_0 // 1 store 4 -load 2 +load 3 loads store 5 -load 2 +load 3 pushbytes 0x // "" stores stringmult_0_l1: load 4 -load 3 +load 2 <= bz stringmult_0_l3 -load 2 -load 2 +load 3 +load 3 loads load 5 concat @@ -42,6 +42,6 @@ intc_0 // 1 store 4 b stringmult_0_l1 stringmult_0_l3: -load 2 +load 3 loads retsub \ No newline at end of file diff --git a/tests/integration/teal/stability/app_swap.teal b/tests/integration/teal/stability/app_swap_v6.teal similarity index 90% rename from tests/integration/teal/stability/app_swap.teal rename to tests/integration/teal/stability/app_swap_v6.teal index bbcef87ff..63c719e3d 100644 --- a/tests/integration/teal/stability/app_swap.teal +++ b/tests/integration/teal/stability/app_swap_v6.teal @@ -1,31 +1,31 @@ #pragma version 6 txna ApplicationArgs 0 -store 1 +store 0 txna ApplicationArgs 1 -store 2 +store 1 +pushint 0 // 0 pushint 1 // 1 -pushint 2 // 2 callsub swap_0 pushint 1337 // 1337 -store 0 -load 0 +store 2 +load 2 itob log -load 0 +load 2 return // swap swap_0: -store 4 store 3 -load 3 +store 4 +load 4 loads store 5 -load 3 load 4 +load 3 loads stores -load 4 +load 3 load 5 stores retsub \ No newline at end of file diff --git a/tests/integration/teal/stability/lsig_exp.teal b/tests/integration/teal/stability/lsig_exp_v6.teal similarity index 100% rename from tests/integration/teal/stability/lsig_exp.teal rename to tests/integration/teal/stability/lsig_exp_v6.teal diff --git a/tests/integration/teal/stability/lsig_oldfac.teal b/tests/integration/teal/stability/lsig_oldfac_v6.teal similarity index 100% rename from tests/integration/teal/stability/lsig_oldfac.teal rename to tests/integration/teal/stability/lsig_oldfac_v6.teal diff --git a/tests/integration/teal/stability/lsig_slow_fibonacci.teal b/tests/integration/teal/stability/lsig_slow_fibonacci_v6.teal similarity index 100% rename from tests/integration/teal/stability/lsig_slow_fibonacci.teal rename to tests/integration/teal/stability/lsig_slow_fibonacci_v6.teal diff --git a/tests/integration/teal/stability/lsig_square_byref.teal b/tests/integration/teal/stability/lsig_square_byref_v6.teal similarity index 81% rename from tests/integration/teal/stability/lsig_square_byref.teal rename to tests/integration/teal/stability/lsig_square_byref_v6.teal index bb2445ae6..d3f1ad2ae 100644 --- a/tests/integration/teal/stability/lsig_square_byref.teal +++ b/tests/integration/teal/stability/lsig_square_byref_v6.teal @@ -1,19 +1,19 @@ #pragma version 6 arg 0 btoi -store 1 -pushint 1 // 1 +store 0 +pushint 0 // 0 callsub squarebyref_0 pushint 1337 // 1337 return // square_byref squarebyref_0: -store 0 -load 0 -load 0 +store 1 +load 1 +load 1 loads -load 0 +load 1 loads * stores diff --git a/tests/integration/teal/stability/lsig_square.teal b/tests/integration/teal/stability/lsig_square_v6.teal similarity index 100% rename from tests/integration/teal/stability/lsig_square.teal rename to tests/integration/teal/stability/lsig_square_v6.teal diff --git a/tests/integration/teal/stability/lsig_string_mult.teal b/tests/integration/teal/stability/lsig_string_mult_v6.teal similarity index 85% rename from tests/integration/teal/stability/lsig_string_mult.teal rename to tests/integration/teal/stability/lsig_string_mult_v6.teal index 6f9cfcb71..c30988953 100644 --- a/tests/integration/teal/stability/lsig_string_mult.teal +++ b/tests/integration/teal/stability/lsig_string_mult_v6.teal @@ -1,8 +1,8 @@ #pragma version 6 intcblock 1 arg 0 -store 4 -pushint 4 // 4 +store 0 +pushint 0 // 0 arg 1 btoi callsub stringmult_0 @@ -12,32 +12,32 @@ return // string_mult stringmult_0: store 1 -store 0 -intc_0 // 1 store 2 -load 0 -loads +intc_0 // 1 store 3 -load 0 +load 2 +loads +store 4 +load 2 pushbytes 0x // "" stores stringmult_0_l1: -load 2 +load 3 load 1 <= bz stringmult_0_l3 -load 0 -load 0 +load 2 +load 2 loads -load 3 +load 4 concat stores -load 2 +load 3 intc_0 // 1 + -store 2 +store 3 b stringmult_0_l1 stringmult_0_l3: -load 0 +load 2 loads retsub \ No newline at end of file diff --git a/tests/integration/teal/stability/lsig_swap.teal b/tests/integration/teal/stability/lsig_swap_v6.teal similarity index 74% rename from tests/integration/teal/stability/lsig_swap.teal rename to tests/integration/teal/stability/lsig_swap_v6.teal index c8d8ccd6d..b08b9bd7e 100644 --- a/tests/integration/teal/stability/lsig_swap.teal +++ b/tests/integration/teal/stability/lsig_swap_v6.teal @@ -1,26 +1,26 @@ #pragma version 6 arg 0 -store 3 +store 0 arg 1 -store 4 -pushint 3 // 3 -pushint 4 // 4 +store 1 +pushint 0 // 0 +pushint 1 // 1 callsub swap_0 pushint 1337 // 1337 return // swap swap_0: -store 1 -store 0 -load 0 -loads store 2 -load 0 -load 1 +store 3 +load 3 +loads +store 4 +load 3 +load 2 loads stores -load 1 load 2 +load 4 stores retsub \ No newline at end of file diff --git a/tests/unit/issue199_test.py b/tests/unit/issue199_test.py new file mode 100644 index 000000000..e4dc22a85 --- /dev/null +++ b/tests/unit/issue199_test.py @@ -0,0 +1,404 @@ +# import asyncio +import time +import threading +from functools import reduce +from itertools import groupby # type: ignore +import pytest + +from pyteal import compileTeal, Compilation, Mode +from examples.signature.factorizer_game import logicsig + +# implicitly import via no_regrassions: +# from examples.application.abi.algobank import router + +from tests.compile_asserts import assert_new_v_old, assert_teal_as_expected +from tests.blackbox import PyTealDryRunExecutor +from tests.integration.graviton_test import ( + FIXTURES, + GENERATED, + ISSUE_199_CASES as ISSUE_199_CASES_BB, + wrap_compile_and_save, +) +from tests.unit.sourcemap_test import no_regressions +from tests.unit.pass_by_ref_test import ISSUE_199_CASES + +SKIPS = { + 1: False, + 2: False, + 3: False, + 4: False, + 5: False, +} +NUM_REPEATS_FOR_T3 = 5 +ASSERT_ALGOBANK_AGAINST_FIXED = False + + +@pytest.mark.skipif(SKIPS[1], reason=f"{SKIPS[1]=}") +@pytest.mark.parametrize("pt", ISSUE_199_CASES) +def test_1_pass_by_ref_teal_output_is_unchanged(pt): + assert_new_v_old(pt, 6, "unchanged") + + +def assert_only_the_slots_permuted(expected: list[str], actual: list[str]): + """ + Name a bit misleading. If expected == actual, we don't actually fail anything. + And even if only the slots are permuted, we still fail at the end. + """ + diffs = [ + (i, x[:2], y[:2]) + for i, (e, a) in enumerate(zip(expected, actual)) + if (x := e.strip().split()) != (y := a.strip().split()) + ] + + if diffs: + assert all(x == y for _, (x, _), (y, _) in diffs) + + slots_only = [(ln, int(x), int(y)) for ln, (_, x), (_, y) in diffs] + + maps = list(sorted({(x, y) for _, x, y in slots_only})) + pmaps = ", ".join(f"{x}-->{y}" for x, y in maps) + + # unique keys and values: + assert len({x for x, _ in maps}) == len(maps) + assert len({y for _, y in maps}) == len(maps) + + assert False, f"""Shoulnd't have any diffs. However, the diff is stable in the sense that only slots got reassigned: +{diffs=} +{pmaps=}""" + + +@pytest.mark.skipif(SKIPS[2], reason=f"{SKIPS[2]=}") +@pytest.mark.parametrize("pt", ISSUE_199_CASES) +def test_2_teal_output_is_unchanged(pt): + expected: list[str] + actual: list[str] + expected, actual = assert_new_v_old(pt, 6, "unchanged", skip_final_assertion=True) + + assert_only_the_slots_permuted(expected, actual) + + +# --- MULTI COMPILE WITH NO FIXTURES --- # + + +def lsig345(): + return compileTeal(logicsig(3, 4, 5), Mode.Signature, version=7) + + +is_algobank_first_time = True + + +def algobank_too_complex(): + global is_algobank_first_time + expected_lines, actual_lines = no_regressions(skip_final_assertion=True) + if ASSERT_ALGOBANK_AGAINST_FIXED: + assert expected_lines == actual_lines, "bailing out early!!!" + + return_value = expected_lines if is_algobank_first_time else actual_lines + is_algobank_first_time = False + return "\n".join(return_value) + + +def algobank(): + expected_lines, actual_lines = no_regressions(skip_final_assertion=True) + if ASSERT_ALGOBANK_AGAINST_FIXED: + assert expected_lines == actual_lines, "bailing out early!!!" + + return "\n".join(actual_lines) + + +def wrap_and_compile(subr): + def compile(): + mode = Mode.Application + return PyTealDryRunExecutor(subr, mode).compile(6, True) + + return compile + + +WRAPPED_199s = [wrap_and_compile(subr) for subr in ISSUE_199_CASES_BB] + + +def multi_compile(N, comp, sync): + async def compile(teals: list[str], idx: int): + teals[idx] = comp() + + teals = [""] * N + + async def main(): + # Use asyncio.gather to execute multiple foo() concurrently + await asyncio.gather(*(compile(teals, idx) for idx in range(N))) + + if sync: + for idx in range(N): + teals[idx] = comp() + else: + asyncio.run(main()) + + teals_gb = list(groupby(teals)) + return teals_gb + + +@pytest.mark.skipif(SKIPS[3], reason=f"{SKIPS[3]=}") +@pytest.mark.parametrize("expr", [lsig345, algobank] + WRAPPED_199s) +@pytest.mark.parametrize("sync", [True, False]) +def test_3_repeated_compilation(expr, sync): + global is_algobank_first_time + is_algobank_first_time = True + teals_gb = multi_compile(NUM_REPEATS_FOR_T3, expr, sync) + + assert teals_gb, "we should have at least one element!!!" + + if len(teals_gb) == 1: + return # COPACETI + + # WLOG - at least 2 + t1, t2 = [t.splitlines() for t, _ in teals_gb[:2]] + + pairs = list(zip(t1, t2)) + diffs = [(i, x, y) for i, (x, y) in enumerate(pairs) if x != y] + + def op(x): + return x.split()[0] + + assert all(op(d[1]) == op(d[2]) for d in diffs) + + def slot(x): + return int(x.split()[1]) + + maps = [(i, slot(x), {slot(y)}) for i, x, y in diffs] + + def dict_append(d, t): + d[k] = (d[k] | t[2]) if (k := t[1]) in d else t[2] + return d + + unified = reduce( + dict_append, + maps, + dict(), + ) + + assert all(len(v) == 1 for v in unified.values()) + + assert False, f"""At the end of the day, this was unstable with {len(teals_gb)} distinct compilations: +{diffs=} +""" + + +# Really, this was an integration test: +@pytest.mark.skipif(SKIPS[4], reason=f"{SKIPS[4]=}") +@pytest.mark.parametrize("subr", ISSUE_199_CASES_BB) +@pytest.mark.parametrize("mode", Mode) +def test_4_stable_teal_generation(subr, mode): + """ + TODO: here's an example of issue #199 at play - need to run a dynamic version of `git bisect` + to figure out what is driving this + """ + case_name = subr.name() + print(f"stable TEAL generation test for {case_name} in mode {mode}") + + # HANG NOTE: I prefer not to modify this test, for it is skipped now on thread-unsafe behavior, + # and I would suggest revisiting later after we have satisfied solution for #199. + _, _, tealfile = wrap_compile_and_save(subr, mode, 6, True, "stability", case_name) + path2actual = GENERATED / "stability" / tealfile + path2expected = FIXTURES / "stability" / tealfile + expected_lines, actual_lines = assert_teal_as_expected( + path2actual, path2expected, skip_final_assertion=True + ) + assert_only_the_slots_permuted(expected_lines, actual_lines) + + +@pytest.mark.parametrize("N", range(2, 11)) +@pytest.mark.skipif(SKIPS[5], reason=f"{SKIPS[5]=}") +def test_5_algobank_in_detail(N): + from pyteal import ( + abi, + ABIReturnSubroutine, + App, + Approve, + Assert, + BareCallActions, + Bytes, + CallConfig, + Expr, + Int, + Global, + InnerTxnBuilder, + MethodConfig, + OnCompleteAction, + OptimizeOptions, + Router, + Seq, + Subroutine, + TealType, + Txn, + TxnField, + TxnType, + ) + + @Subroutine(TealType.none) + def assert_sender_is_creator() -> Expr: + return Assert(Txn.sender() == Global.creator_address()) + + # move any balance that the user has into the "lost" amount when they close out or clear state + transfer_balance_to_lost = App.globalPut( + Bytes("lost"), + App.globalGet(Bytes("lost")) + App.localGet(Txn.sender(), Bytes("balance")), + ) + + mc = MethodConfig(no_op=CallConfig.CALL, opt_in=CallConfig.CALL) + + @ABIReturnSubroutine + def deposit(payment: abi.PaymentTransaction, sender: abi.Account) -> Expr: + """This method receives a payment from an account opted into this app and records it as a deposit. + + The caller may opt into this app during this call. + + Args: + payment: A payment transaction containing the amount of Algos the user wishes to deposit. + The receiver of this transaction must be this app's escrow account. + sender: An account that is opted into this app (or will opt in during this method call). + The deposited funds will be recorded in this account's local state. This account must + be the same as the sender of the `payment` transaction. + """ + return Seq( + Assert(payment.get().sender() == sender.address()), + Assert(payment.get().receiver() == Global.current_application_address()), + App.localPut( + sender.address(), + Bytes("balance"), + App.localGet(sender.address(), Bytes("balance")) + + payment.get().amount(), + ), + ) + + @ABIReturnSubroutine + def getBalance(user: abi.Account, *, output: abi.Uint64) -> Expr: + """Lookup the balance of a user held by this app. + + Args: + user: The user whose balance you wish to look up. This user must be opted into this app. + + Returns: + The balance corresponding to the given user, in microAlgos. + """ + return output.set(App.localGet(user.address(), Bytes("balance"))) + + @ABIReturnSubroutine + def withdraw(amount: abi.Uint64, recipient: abi.Account) -> Expr: + """Withdraw an amount of Algos held by this app. + + The sender of this method call will be the source of the Algos, and the destination will be + the `recipient` argument. + + The Algos will be transferred to the recipient using an inner transaction whose fee is set + to 0, meaning the caller's transaction must include a surplus fee to cover the inner + transaction. + + Args: + amount: The amount of Algos requested to be withdraw, in microAlgos. This method will fail + if this amount exceeds the amount of Algos held by this app for the method call sender. + recipient: An account who will receive the withdrawn Algos. This may or may not be the same + as the method call sender. + """ + return Seq( + # if amount is larger than App.localGet(Txn.sender(), Bytes("balance")), the subtraction + # will underflow and fail this method call + App.localPut( + Txn.sender(), + Bytes("balance"), + App.localGet(Txn.sender(), Bytes("balance")) - amount.get(), + ), + InnerTxnBuilder.Begin(), + InnerTxnBuilder.SetFields( + { + TxnField.type_enum: TxnType.Payment, + TxnField.receiver: recipient.address(), + TxnField.amount: amount.get(), + TxnField.fee: Int(0), + } + ), + InnerTxnBuilder.Submit(), + ) + + bcas = BareCallActions( + # approve a creation no-op call + no_op=OnCompleteAction(action=Approve(), call_config=CallConfig.CREATE), + # approve opt-in calls during normal usage, and during creation as a convenience for the creator + opt_in=OnCompleteAction(action=Approve(), call_config=CallConfig.ALL), + # move any balance that the user has into the "lost" amount when they close out or clear state + close_out=OnCompleteAction( + action=transfer_balance_to_lost, call_config=CallConfig.CALL + ), + # only the creator can update or delete the app + update_application=OnCompleteAction( + action=assert_sender_is_creator, call_config=CallConfig.CALL + ), + delete_application=OnCompleteAction( + action=assert_sender_is_creator, call_config=CallConfig.CALL + ), + ) + + def build(): + r = Router( + name="AlgoBank", + bare_calls=bcas, + clear_state=transfer_balance_to_lost, + ) + r.add_method_handler(deposit, method_config=mc) + r.add_method_handler(getBalance) + r.add_method_handler(withdraw) + return r + + def compile(router): + return router.compile_program( + version=6, optimize=OptimizeOptions(scratch_slots=True) + ) + + # routers = [build() for _ in range(N)] + # assert routers[0] is not routers[1] + router = build() + routers = [router] * N + outputs = [None] * N + + def assert_sameness(i, j): + assert (o1 := outputs[i]) + assert (o2 := outputs[j]) + a1, c1, j1 = o1 + a2, c2, j2 = o2 + assert j1.dictify() == j2.dictify() + assert c1 == c2 + assert a1 == a2 + + #NEW: + def compile_at(idx: int): + outputs[idx] = compile(routers[idx]) + + def main(): + threads = [threading.Thread(target=compile_at, args=(i,)) for i in range(N)] + for t in threads: + t.start() + for t in threads: + t.join() + + while Compilation.ready_to_unpause < N: + time.sleep(0.1) + + Compilation.paused = False + + + # ORIG: + # async def compile_at(idx: int): + # outputs[idx] = compile(routers[idx]) + + # async def main(): + # await asyncio.gather(*(compile_at(i) for i in range(N))) + # while Compilation.ready_to_unpause < N: + # asyncio.sleep(0.1) + # Compilation.paused = False + + # asyncio.run(main()) + + main() + + for i in range(1, N): + assert_sameness(i-1, i) + diff --git a/tests/unit/sourcemap_test.py b/tests/unit/sourcemap_test.py index 6e68f22a2..e8375951e 100644 --- a/tests/unit/sourcemap_test.py +++ b/tests/unit/sourcemap_test.py @@ -5,12 +5,12 @@ """ import ast +import difflib import json import time from configparser import ConfigParser from pathlib import Path from unittest import mock - import pytest from pyteal.compiler.sourcemap import R3SourceMap, R3SourceMapJSON @@ -18,7 +18,6 @@ ALGOBANK = Path.cwd() / "examples" / "application" / "abi" -@pytest.mark.serial def test_frames(): from pyteal.stack_frame import StackFrames @@ -50,7 +49,6 @@ def test_frames(): StackFrames._no_stackframes = originally -@pytest.mark.serial def test_TealMapItem_source_mapping(): from pyteal.stack_frame import StackFrames @@ -108,7 +106,7 @@ def mock_teal(ops): StackFrames._no_stackframes = originally -def no_regressions(): +def no_regressions(skip_final_assertion=False): from examples.application.abi.algobank import router from pyteal import OptimizeOptions @@ -116,24 +114,25 @@ def no_regressions(): version=6, optimize=OptimizeOptions(scratch_slots=True) ) - def compare_and_assert(file, actual): + def compare_and_assert(file, actual, skip_final_assertion): with open(ALGOBANK / file, "r") as f: expected_lines = f.read().splitlines() actual_lines = actual.splitlines() assert len(expected_lines) == len(actual_lines) - assert expected_lines == actual_lines + if not skip_final_assertion: + assert expected_lines == actual_lines - compare_and_assert("algobank.json", json.dumps(contract.dictify(), indent=4)) - compare_and_assert("algobank_clear_state.teal", clear) - compare_and_assert("algobank_approval.teal", approval) + return expected_lines, actual_lines + + compare_and_assert("algobank.json", json.dumps(contract.dictify(), indent=4), False) + compare_and_assert("algobank_clear_state.teal", clear, False) + return compare_and_assert("algobank_approval.teal", approval, skip_final_assertion) -@pytest.mark.serial def test_no_regression_with_sourcemap_as_configured(): no_regressions() -@pytest.mark.serial def test_no_regression_with_sourcemap_enabled(): from pyteal.stack_frame import StackFrames @@ -145,7 +144,6 @@ def test_no_regression_with_sourcemap_enabled(): StackFrames._no_stackframes = originally -@pytest.mark.serial def test_no_regression_with_sourcemap_disabled(): from pyteal.stack_frame import StackFrames @@ -157,7 +155,6 @@ def test_no_regression_with_sourcemap_disabled(): StackFrames._no_stackframes = originally -@pytest.mark.serial def test_sourcemap_fails_because_unconfigured(): from examples.application.abi.algobank import router from pyteal import OptimizeOptions @@ -173,7 +170,6 @@ def test_sourcemap_fails_because_unconfigured(): assert "pyteal.ini" in str(smde.value) -@pytest.mark.serial def test_config(): from pyteal.stack_frame import StackFrames @@ -209,30 +205,41 @@ def test_config(): StackFrames._no_stackframes = originally -@pytest.mark.skip( - reason="""Supressing this flaky test as -router_test::test_router_compile_program_idempotence is similar in its goals -and we expect flakiness to persist until https://github.com/algorand/pyteal/issues/199 -is finally addressed """ -) -@pytest.mark.serial def test_idempotent(): # make sure we get clean up properly and therefore get idempotent results from examples.application.abi.algobank import router from pyteal import OptimizeOptions - approval1, clear1, contract1 = ( + def assert_same_results(first_compilation, second_compilation): + approval1, clear1, contract1 = first_compilation + approval2, clear2, contract2 = second_compilation + + assert contract1.dictify() == contract2.dictify() + + assert len(clear1.splitlines()) == len(clear2.splitlines()) + assert clear1 == clear2 + + assert len(a1 := approval1.splitlines()) == len(a2 := approval2.splitlines()) + print( + '----------unified_diff(a1, a2, "approval1.teal", "approval2.teal")----------' + ) + for d in list(difflib.unified_diff(a1, a2, "approval1.teal", "approval2.teal")): + print(d) + + assert approval1 == approval2 + + compilation_1 = ( func := lambda: router.compile_program( version=6, optimize=OptimizeOptions(scratch_slots=True) ) )() - approval2, clear2, contract2 = func() + compilation_2 = func() + compilation_3 = func() + compilation_4 = func() - assert contract1.dictify() == contract2.dictify() - assert len(clear1.splitlines()) == len(clear2.splitlines()) - assert clear1 == clear2 - assert len(approval1.splitlines()) == len(approval2.splitlines()) - assert approval1 == approval2 + assert_same_results(compilation_1, compilation_4) + assert_same_results(compilation_1, compilation_3) + assert_same_results(compilation_1, compilation_2) # ---- BENCHMARKS - SKIPPED BY DEFAULT ---- # @@ -344,3 +351,85 @@ def test_time_benchmark_sourcemap_enabled(_): trial(annotated_teal) assert False + + +def multi_compile(N, comp, sync=True): + import asyncio + from itertools import groupby + + async def compile(teals: list[str], idx: int): + teals[idx] = comp() + + teals = [""] * N + + async def main(): + # Use asyncio.gather to execute multiple foo() concurrently + await asyncio.gather(*(compile(teals, idx) for idx in range(N))) + + if sync: + for idx in range(N): + teals[idx] = comp() + else: + asyncio.run(main()) + + teals_gb = list(groupby(teals)) + assert len(teals_gb) >= 2 # this is provably "bad" + + return teals_gb + + +from pyteal import compileTeal, Mode +from examples.signature.factorizer_game import logicsig + + +def lsig345(): + return compileTeal(logicsig(3, 4, 5), Mode.Signature, version=7) + + +from examples.application.abi.algobank import router + + +def algobank(): + from pyteal import OptimizeOptions + + no_regressions() + + return router.compile_program( + version=6, optimize=OptimizeOptions(scratch_slots=True) + )[0] + + +@pytest.mark.parametrize("expr", [lsig345, algobank]) +def test_multithreaded_compilation(expr): + from functools import reduce + + N = 100 + + teals_gb = multi_compile(N, expr) + + t1, t2 = [t.splitlines() for t, _ in teals_gb[:2]] + + pairs = list(zip(t1, t2)) + diffs = [(i, x, y) for i, (x, y) in enumerate(pairs) if x != y] + + def op(x): + return x.split()[0] + + assert all(op(d[1]) == op(d[2]) for d in diffs) + + def slot(x): + return int(x.split()[1]) + + maps = [(i, slot(x), {slot(y)}) for i, x, y in diffs] + + def dict_append(d, t): + d[k] = (d[k] | t[2]) if (k := t[1]) in d else t[2] + return d + + unified = reduce( + dict_append, + maps, + dict(), + ) + + assert all(len(v) == 1 for v in unified.values()) diff --git a/tests/unit/teal/unchanged/lots_o_vars.teal b/tests/unit/teal/unchanged/lots_o_vars.teal index 351b812a0..dff8e8296 100644 --- a/tests/unit/teal/unchanged/lots_o_vars.teal +++ b/tests/unit/teal/unchanged/lots_o_vars.teal @@ -120,9 +120,9 @@ txna ApplicationArgs 1 store 14 txna ApplicationArgs 2 btoi -store 6 +store 4 txna ApplicationArgs 3 -store 7 +store 5 load 0 pop load 1 @@ -160,25 +160,25 @@ byte "twelve" app_global_get pop int 13 -store 4 -load 4 +store 6 +load 6 loads itob log int 14 -store 5 -load 5 +store 7 +load 7 loads log -int 6 -store 4 -load 4 +int 4 +store 6 +load 6 loads itob log -int 7 -store 5 -load 5 +int 5 +store 7 +load 7 loads log int 1337 diff --git a/tests/unit/teal/unchanged/sub_logcat_dynamic.teal b/tests/unit/teal/unchanged/sub_logcat_dynamic.teal index 87a430bc3..1711ed206 100644 --- a/tests/unit/teal/unchanged/sub_logcat_dynamic.teal +++ b/tests/unit/teal/unchanged/sub_logcat_dynamic.teal @@ -1,28 +1,28 @@ #pragma version 6 byte "hello" -store 0 // 0: "hello" +store 0 int 0 -int 42 // >@0,42 -callsub logcatdynamic_0 // <> -byte "hello42" // >"hello42" -load 0 // >"hello42","hello42" -== // >1 -assert // <> -int 1 // >1 -return // < +int 42 +callsub logcatdynamic_0 +byte "hello42" +load 0 +== +assert +int 1 +return // logcat_dynamic -logcatdynamic_0: // >@0,42 -store 2 // 2: 42 -store 1 // 1: @0 +logcatdynamic_0: +store 1 +store 2 +load 2 +load 2 +loads load 1 -load 1 // >@0,@0 -loads // >@0,"hello" -load 2 // >@0,"hello",42 -itob // >@0,"hello","42" -concat // >@0,"hello42" -stores // 0: "hello42" -load 1 // >@0 -loads // >"hello42" -log +itob +concat +stores +load 2 +loads +log retsub \ No newline at end of file diff --git a/tests/unit/teal/unchanged/sub_mixed.teal b/tests/unit/teal/unchanged/sub_mixed.teal index 98934bb83..a6e263004 100644 --- a/tests/unit/teal/unchanged/sub_mixed.teal +++ b/tests/unit/teal/unchanged/sub_mixed.teal @@ -1,24 +1,24 @@ #pragma version 6 -int 42 // >42 -byte "x" // >42,"x" -int 0 // >42,"x",0 -callsub mixedannotations_0 // >42 -return // <> +int 42 +byte "x" +int 0 +callsub mixedannotations_0 +return // mixed_annotations -mixedannotations_0: // >42,"x",0 -store 3 // 3:0 -store 2 // 2:"x" -store 1 // 1:42 -load 3 // >0 -load 1 // >0,42 -stores // 0:42 -load 2 // >"x" -byte "=" // >"x","=" -concat // >"x=" -load 1 // >"x=",42 -itob // >"x=","42" -concat // >"x=42" -log // LOG -load 1 // >42 +mixedannotations_0: +store 1 +store 2 +store 3 +load 1 +load 3 +stores +load 2 +byte "=" +concat +load 3 +itob +concat +log +load 3 retsub \ No newline at end of file diff --git a/tests/unit/teal/unchanged/swapper.teal b/tests/unit/teal/unchanged/swapper.teal index 00a996f15..aece179b7 100644 --- a/tests/unit/teal/unchanged/swapper.teal +++ b/tests/unit/teal/unchanged/swapper.teal @@ -1,47 +1,47 @@ #pragma version 6 byte "hello" -store 5 // 5: hello // x +store 0 byte "goodbye" -store 6 // 6: goodbye // y -load 5 -load 6 -callsub cat_1 // <> -int 5 -int 6 -callsub swap_0 // >5,6 -load 5 // >goodbye +store 1 +load 0 +load 1 +callsub cat_1 +int 0 +int 1 +callsub swap_0 +load 0 byte "goodbye" == -assert // <> -load 6 // >hello +assert +load 1 byte "hello" == -assert // <> -int 1000 // >1000 -return // <> +assert +int 1000 +return // swap swap_0: -store 1 // 1: 6 // @y -store 0 // 0: 5 // @x -load 0 // >5 -loads // >hello -store 2 // 2: hello // z -load 0 // >5 -load 1 // >5,6 -loads // >5,goodbye -stores // 5: goodbye -load 1 // >6 -load 2 // >6,hello -stores // 6: hello +store 2 +store 3 +load 3 +loads +store 4 +load 3 +load 2 +loads +stores +load 2 +load 4 +stores retsub // cat cat_1: -store 4 // 4: goodbye -store 3 // 3: hello -load 3 // >hello -load 4 // >hello,goodbye -concat // >hellogoodbye -pop // > +store 5 +store 6 +load 6 +load 5 +concat +pop retsub \ No newline at end of file diff --git a/tests/unit/teal/unchanged/wilt_the_stilt.teal b/tests/unit/teal/unchanged/wilt_the_stilt.teal index 28d339350..ceb00fa70 100644 --- a/tests/unit/teal/unchanged/wilt_the_stilt.teal +++ b/tests/unit/teal/unchanged/wilt_the_stilt.teal @@ -1,38 +1,38 @@ #pragma version 6 int 129 -store 0 // 0: @129 // pointer to wilt's address +store 0 load 0 int 100 -stores // 129: 100 // set wilt's value +stores int 1 -store 0 // 0: @1 // pointer to kobe's address (compiler assigned) +store 0 load 0 int 81 -stores // 1: 81 // set kobe's value +stores int 131 -store 0 // 0: @131 // pointer to dt's address +store 0 load 0 int 73 -stores // 131: 73 // set dt's value +stores load 0 loads -int 73 // >73,73 -== // >1 -assert // <> -load 0 -int 131 // >131,131 -== // >1 -assert // <> +int 73 +== +assert +load 0 +int 131 +== +assert int 129 -store 0 // 0: @129 +store 0 load 0 loads -int 100 // >100,100 -== // >1 -assert // <> +int 100 +== +assert load 0 -int 129 // >129,129 -== // >1 -assert // <> +int 129 +== +assert int 100 return \ No newline at end of file