
    )j!                    &   d Z ddlmZ ddlZddlmc mZ ddl	Z	ddl
Z
ddlZddlmZmZ ddlmZ ej                   j#                  d e ee      j)                         j*                  j*                               ddlmZmZmZ ddlmZ ddlmZ  G d	 d
      Z G d d      Z G d d      Z  G d de
jB                        Z" G d d      Z#ddZ$d dZ%ddddd!dZ&d Z'd Z(d Z)d Z*d Z+d"dZ,d"dZ-e.dk(  r ej^                   e-              yy)#a:  
V18 anti-freeze envelope tests.

Verify that the orchestrator survives an unresponsive broker/provider:
  - `_build_tech` returns None when the market data provider hangs
    longer than `market_data_timeout_seconds`.
  - `_refresh_account_balance` keeps its cached value when the broker
    hangs longer than `broker_call_timeout_seconds`.
  - The maintenance loop emits a heartbeat line on system.log on
    every iteration (even when nothing else is logged), so watchdog.sh
    (STALE_SECONDS=600) does not interpret idle-but-alive ticks as
    freezes.

These tests pin down the regression that caused the bot to be
SIGTERMed every ~10min in live: an SDK WS init call without timeout
froze the scan loop until system.log went stale.

Run:
    cd ~/apex_v16
    python -m pytest tests/test_orchestrator_freeze_guard.py -q
    )annotationsN)datetimetimezone)Path)AccountKindRunModeRuntimeConfig)Orchestrator)SessionStatec                      e Zd ZdZddZd Zy)HangingProvideruN   Mimics an SDK whose get_bars never resolves — provokes asyncio.TimeoutError.c                    d| _         y Nr   )callsselfs    ;/home/work/apex_v16/tests/test_orchestrator_freeze_guard.py__init__zHangingProvider.__init__-   s	    
    c                t   K   | xj                   dz  c_         t        j                  d       d {    y 7 wN   g      N@)r   asynciosleep)r   symbol	timeframens       r   get_barszHangingProvider.get_bars0   s%     

a
mmD!!!   .868NreturnNone)__name__
__module____qualname____doc__r   r    r   r   r   r   *   s    X"r   r   c                      e Zd ZdZddZd Zy)HangingBrokeruE   get_account_balance never resolves — provokes asyncio.TimeoutError.c                    d| _         y r   )balance_callsr   s    r   r   zHangingBroker.__init__9   s
    r   c                t   K   | xj                   dz  c_         t        j                  d       d {    y 7 wr   )r+   r   r   r   s    r   get_account_balancez!HangingBroker.get_account_balance<   s)     ammD!!!r   Nr    )r#   r$   r%   r&   r   r-   r'   r   r   r)   r)   6   s    O"r   r)   c                      e Zd ZddZd Zy)	_MemStorec                    d | _         y N_stater   s    r   r   z_MemStore.__init__B   s	    r   c                    || _         y r1   r2   )r   states     r   savez_MemStore.saveE   s	    r   Nr    )r#   r$   r%   r   r6   r'   r   r   r/   r/   A   s    r   r/   c                  ,     e Zd ZdZd fdZddZ xZS )_ListHandleruE   Capture log records on a list — used to inspect heartbeat emission.c                0    t         |           g | _        y r1   )superr   records)r   	__class__s    r   r   z_ListHandler.__init__L   s    02r   c                :    | j                   j                  |       y r1   )r;   append)r   records     r   emitz_ListHandler.emitP   s    F#r   r    )r?   zlogging.LogRecordr!   r"   )r#   r$   r%   r&   r   r@   __classcell__)r<   s   @r   r8   r8   I   s    O3$r   r8   c                  0    e Zd ZdZddZd Zd Zd Zd Zy)	_CapturingLoggerzMimics LoggerBundle: orchestrator only needs `.system` (stdlib Logger)
    and optional log_* hooks. Tests inspect `.system_handler.records` directly.c                   t        j                  d      | _        | j                  j                  t         j                         t        | j                  j                        D ]  }| j                  j                  |        t               | _	        | j                  j                  t         j                         | j                  j                  | j                         y )Nztest.freeze_guard)logging	getLoggersystemsetLevelDEBUGlisthandlersremoveHandlerr8   system_handler
addHandler)r   hs     r   r   z_CapturingLogger.__init__X   s    ''(;<W]]+dkk**+ 	)AKK%%a(	)*n$$W]]3t223r   c                     y r1   r'   r   aks      r   log_session_eventz"_CapturingLogger.log_session_eventc       r   c                     y r1   r'   rQ   s      r   log_trade_openedz!_CapturingLogger.log_trade_openedd   rU   r   c                     y r1   r'   rQ   s      r   log_trade_closedz!_CapturingLogger.log_trade_closede   rU   r   c                     y r1   r'   rQ   s      r   	log_errorz_CapturingLogger.log_errorf   rU   r   Nr    )	r#   r$   r%   r&   r   rT   rW   rY   r[   r'   r   r   rC   rC   T   s    S4 /--&r   rC   c                     t        d|         y )Nz  ok  )print)labels    r   _okr_   m   s    	F5'
r   c                 "   t        t        j                  t        j                        }dg|_        d|_        d|_        d|_        d|_	        d|_
        d|_        d|_        d|_        | j                         D ]  \  }}t        |||        |S )N)modeaccountMESg        r   g333333?)r	   r   PAPERr   
INELIGIBLEasset_filterscan_loop_phase_offset_secondsmanage_loop_interval_seconds!maintenance_loop_interval_secondsmarket_data_timeout_secondsbroker_call_timeout_secondsh4_fetch_timeout_secondswatchdog_timeout_secondsnews_sync_timeout_secondsitemssetattr)overcfgrS   vs       r   make_cfgrt   q   s    
W]]K4J4J
KCwC),C&'(C$,-C)&)C#&)C##&C #&C $'C!

 1QJr   )providerbrokerloggerc                   t               }t        | d |xs
 t               |t               |xs
 t	               i d d d |d      S )Nr   )config	ai_clientmarket_data_providerr5   storerw   brain_dispatchopenercloserrisk_managerrv   max_iterations)r   r
   r   r/   rC   )rr   ru   rv   rw   r5   s        r   	make_orchr      sK    NE%:):k+)+Dt r   c                   K   t               } t               }t               }t        | ||      }|j	                  d       d{   }d}||u }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }t        j                  d      d	z   d
|iz  }t        t        j                  |            dx}}|j                  }d}	||	k\  }
|
st        j                  d|
fd||	f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |	      dz  }dd|iz  }t        t        j                  |            dx}x}
}	|j                  j                   D cg c]  }|j#                          }}d |D        }t%        |      }
|
st        j                  |      dz   dt        j                         v st        j                  t$              rt        j                  t$              ndt        j                  |      t        j                  |
      dz  }t        t        j                  |            dx}}
t'        d       y7 c c}w w)uC   _build_tech wraps build_tech_snapshot with wait_for; hang → None.ru   rw   rc   Nisz%(py0)s is %(py3)sresultpy0py3z.hang must be converted to None, not propagated
>assert %(py5)spy5r   >=)z-%(py2)s
{%(py2)s = %(py0)s.calls
} >= %(py5)sru   )r   py2r   zassert %(py7)spy7c              3  $   K   | ]  }d |v  
 yw)zbuild_tech_snapshot timed outNr'   .0ms     r   	<genexpr>zCtest_build_tech_returns_none_when_provider_hangs.<locals>.<genexpr>        B.!3B   .
>assert %(py4)s
{%(py4)s = %(py0)s(%(py2)s)
}anyr   r   py4z7_build_tech: provider hang -> None + system.log warning)rt   r   rC   r   _build_tech
@pytest_ar_call_reprcompare@py_builtinslocals_should_repr_global_name	_saferepr_format_assertmsgAssertionError_format_explanationr   rM   r;   
getMessager   r_   )rr   ru   rw   orchr   @py_assert2@py_assert1@py_format4@py_format6@py_assert4@py_assert3@py_format8rmsgs@py_format5s                  r   0test_build_tech_returns_none_when_provider_hangsr      s    
*C HFS8F;D##E**FK6T>KKK6TKKKKKK6KKK6KKKTKKKKKKKKKK>>Q>Q>Q88>Q$*$9$9$A$ABqALLNBDBBTBH3BBHBHHDHHHHHH3HHH3HHHBHHHBHHHHHHAB +
 Cs&   AKKFK#K:CKKc                   K   t               } t               }t               }t        | ||      }|j	                  d       d{   }d}||u }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }dd	|iz  }t        t        j                  |            dx}}|j                  j                  D 	cg c]  }	|	j                          }
}	d
 |
D        }t!        |      }|st        j"                  |
      dz   dt        j                         v st        j                  t               rt        j                  t               ndt        j                  |      t        j                  |      dz  }t        t        j                  |            dx}}t%        d       y7 c c}	w w)z4_fetch_h4 returns None when provider.get_bars hangs.r   rc   Nr   r   r   r   zassert %(py5)sr   c              3  $   K   | ]  }d |v  
 yw)zH4 fetch timed outNr'   r   s     r   r   z8test_fetch_h4_returns_none_on_timeout.<locals>.<genexpr>   s     7Q#q(7r   r   r   r   z5_fetch_h4: provider hang -> None + system.log warning)rt   r   rC   r   	_fetch_h4r   r   r   r   r   r   r   r   rM   r;   r   r   r   r_   )rr   ru   rw   r   r   r   r   r   r   r   r   r   r   s                r   %test_fetch_h4_returns_none_on_timeoutr      s4    
*C HFS8F;D>>%((F6T>6T66T$*$9$9$A$ABqALLNBDB7$7=377=7========3===3===7===7======?@	 )Bs&   AG7G/CG7G2CG72G7c                   K   t               } t               }t               }t        | ||      }|j	                          d{    |j
                  }| j                  }||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      dz  }dd	|iz  }t        t        j                  |            dx}x}}|j                  j                   D 	cg c]  }	|	j#                          }
}	d
 |
D        }t%        |      }|st        j&                  |
      dz   dt        j                         v st        j                  t$              rt        j                  t$              ndt        j                  |      t        j                  |      dz  }t        t        j                  |            dx}}t)        d       y7 c c}	w w)z?_refresh_account_balance falls back to dry_run_balance on hang.rv   rw   N==)zd%(py2)s
{%(py2)s = %(py0)s._cached_account_balance
} == %(py6)s
{%(py6)s = %(py4)s.dry_run_balance
}r   rr   )r   r   r   py6zassert %(py8)spy8c              3  $   K   | ]  }d |v  
 yw)zget_account_balance timed outNr'   r   s     r   r   z?test_refresh_balance_keeps_cached_on_timeout.<locals>.<genexpr>   r   r   r   r   r   zA_refresh_account_balance: broker hang -> dry_run_balance fallback)rt   r)   rC   r   _refresh_account_balance_cached_account_balancedry_run_balancer   r   r   r   r   r   r   r   rM   r;   r   r   r   r_   )rr   rv   rw   r   r   @py_assert5r   @py_format7@py_format9r   r   r   s               r   ,test_refresh_balance_keeps_cached_on_timeoutr      sy    
*C_FFS7D

'
'
)))''>3+>+>>'+>>>>>'+>>>>>>>4>>>4>>>'>>>>>>3>>>3>>>+>>>>>>>>$*$9$9$A$ABqALLNBDBBTBH3BBHBHHDHHHHHH3HHH3HHHBHHHBHHHHHHKL * Cs&   A I$ID/I$2I	CI$I$c            
       K   t               } t               }t        | d|      }d|_        |j	                          d{    |j
                  j                  D cg c]  }d|j                         v r| }}t        |      }d}||k\  }|s7t        j                  d|fd||f      dt        j                         v st        j                  t              rt        j                  t              nddt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      d	z  }t        j                  d
t        |       d|D cg c]  }|j                          c}       dz   d|iz  }	t!        t        j"                  |	            dx}x}}t%        dt        |       d       y7 c c}w c c}w w)z=One '[maintenance] heartbeat' line per maintenance iteration.Nr      [maintenance] heartbeatr   )z0%(py3)s
{%(py3)s = %(py0)s(%(py1)s)
} >= %(py6)slen
heartbeats)r   py1r   r   z/expected >=3 heartbeats over 3 iterations, got z: z
>assert %(py8)sr   zmaintenance heartbeat: z lines over 3 iterations)rt   rC   r   r   runrM   r;   r   r   r   r   r   r   r   r   r   r   r   r_   )
rr   rw   r   r   r   r   r   r   r   r   s
             r   ,test_maintenance_heartbeat_emitted_each_tickr      s    
*CF Sf5DD
((* ((00$6 	
J  z? a ?a   ?a                                 :#j/9J"$./qALLN/
0	2     
!#j/!22JKL  0s0   =G1G$ G1G'7DG1?G,AG1'
G1c                   K   t               } d| _        t               }t        | d|      }d|_        |j                          d{    |j                  j                  D cg c]  }d|j                         v r| }}g }||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      d	z  }t        j                  |      d
z   d|iz  }t!        t        j"                  |            dx}}t%        d       y7 c c}w w)z@Setting maintenance_heartbeat_enabled=False suppresses the line.FNr      r   r   )z%(py0)s == %(py3)sr   r   r   r   z6maintenance heartbeat: disabled flag silences the line)rt   maintenance_heartbeat_enabledrC   r   r   r   rM   r;   r   r   r   r   r   r   r   r   r   r   r_   )	rr   rw   r   r   r   r   r   r   r   s	            r   *test_maintenance_heartbeat_can_be_disabledr      s     
*C(-C%FSf5DD
((* ((00$6 	
J  ':''':'''''':''':''''''Z''''''@A s%   AEEE#E>CEEc                    K   t        d       t                d {    t                d {    t                d {    t	                d {    t                d {    t        d       y7 X7 H7 87 (7 w)Nz!test_orchestrator_freeze_guard.pyzALL TESTS PASSEDr   )r]   r   r   r   r   r   r'   r   r   _async_mainr      sl     	
-.
:
<<<
/
111
6
888
6
888
4
666	
 =1886sT   A>A4A>A6A>A8 A>A:A>#A<$A>6A>8A>:A><A>c                 <    t        j                  t                     S r1   )r   r   r   r'   r   r   mainr      s    ;;{}%%r   __main__)r^   strr!   r"   )r!   r	   )r!   r
   )r!   int)0r&   
__future__r   builtinsr   _pytest.assertion.rewrite	assertionrewriter   r   rE   sysr   r   pathlibr   pathinsertr   __file__resolveparentcore.configr   r   r	   orchestratorr
   persistence.state_storer   r   r)   r/   Handlerr8   rC   r_   rt   r   r   r   r   r   r   r   r   r#   exitr'   r   r   <module>r      s   , #     
 '  3tH~--/66==> ? ; ; % 0	" 	"" " $7?? $' '2"  $D (C"AMM,B,& zCHHTV r   