
    /jg              
         U d Z ddlmZ ddlZddlmc mZ ddl	Z	ddl
Z
ddlmZmZmZ ddlmZ ddlZe
j$                  j'                  d e ee      j-                         j.                  j.                               ddlmZmZmZmZmZmZ ddlm Z m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. dd	l/m0Z0m1Z1m2Z2 dd
l3m4Z4m5Z5 d Z6 G d d      Z7 G d d      Z8 G d d      Z9 G d d      Z: G d d      Z; G d de      Z< G d d      Z= G d d      Z> eddddddej~                         Z@d!eAd"<   ddd#dddddd$d%		 	 	 d;d&ZBd<d'ZCd( ZDd=d>d)ZEd* ZFd+ ZGd, ZHd- ZId. ZJd/ ZKd0 ZLd1 ZMd2 ZNd3 ZOd4 ZPd5 ZQd6 ZRd7 ZSeTd8k(  rs eUd9        eF         eG         eH         eI         eJ         eK         eL         eM         eN         eO         eP         eQ         eR         eS         eUd:       yy)?aa  
Phase C2b orchestrator tests: degraded mode + reconnect + post-reconnect
reconcile + safe-mode behavior.

13 tests:
  Detection (3):
    1. classify_broker_error: ConnectionError -> "transient"
    2. classify_broker_error: 401 unauthorized -> "auth"
    3. broker exception in scan/manage marks orchestrator degraded

  Reconnect (2):
    4. mid-loop reconnect succeeds -> degraded cleared, reconcile runs
    5. mid-loop reconnect fails -> stays degraded

  Safe mode (3):
    6. degraded -> _scan_entries skipped (no opener calls)
    7. degraded -> manage EXIT logged "exit_deferred_disconnected"
    8. degraded -> watchdog skipped

  Hard timeout (1):
    9. degraded > broker_degraded_max_minutes -> exit code 4

  Post-reconnect reconcile (2):
    10. reconcile_startup called with post_reconnect=True after recovery
    11. reconcile failure post-reconnect doesn't break loop

  E2E disconnect-defer-resume (1):
    12. trade open, brain says EXIT during disconnect (deferred), reconnect,
        brain re-says EXIT, executes successfully

Run:
    cd ~/apex_v16
    python tests/test_orchestrator_phase_c2b.py
    )annotationsN)datetime	timedeltatimezone)Path)
BrokerBaseCancelResultClosedTradeOrderOrderResultPosition)RuntimeConfigRunModeAccountKind)BrainContextBrainDecision	BrainName	DirectionEntryDecisionEntryEvalResultMarketStructureRegimeTradeAction
TradeEntryTradeRuntime)EXIT_BROKER_UNRECOVERABLEOrchestratorclassify_broker_error)ActiveTradeSessionStatec                     t        d|         y )Nz  ok  )print)labels    8/home/work/apex_v16/tests/test_orchestrator_phase_c2b.py_okr%   >   s    ug&'    c                      e Zd ZddZy)FakeAINc                *   K   ddl m}  |d d      S w)Nr   )
AIResponseunknown)text
error_kind)brain.ai_clientr*   )selfprompttemperature
max_tokensr*   s        r$   askz
FakeAI.askF   s     .t	::s   )g?N)__name__
__module____qualname__r3    r&   r$   r(   r(   E   s    ;r&   r(   c                      e Zd Zd Zd Zy)	JsonlSinkc                    g | _         y N)eventsr/   s    r$   __init__zJsonlSink.__init__L   s    br&   c                B    | j                   j                  d|i|       y )Nevent)r<   appendr/   r@   fieldss      r$   writezJsonlSink.writeM   s    dkk&8&8'59SF9S&Tr&   N)r4   r5   r6   r>   rD   r7   r&   r$   r9   r9   K   s    (Tr&   r9   c                  *    e Zd Zd Zd Zd Zd Zd Zy)
FakeLoggerc                    dd l }t               | _        t               | _        t               | _        |j                  d      | _        y )Nr   ztest.c2b)loggingr9   	brain_logsession_log	error_log	getLoggersystem)r/   rH   s     r$   r>   zFakeLogger.__init__Q   s3    "$;"''
3r&   c                >     | j                   j                  |fi | y r;   )rJ   rD   rB   s      r$   log_session_eventzFakeLogger.log_session_eventW   s    u//r&   c                B     | j                   j                  d||d| y )N)whereerror)rR   )rK   rD   )r/   rQ   rR   extras       r$   	log_errorzFakeLogger.log_errorY   s     HEH%Hr&   c                <     | j                   j                  di | y )N)trade_openedrI   rD   r/   rC   s     r$   log_trade_openedzFakeLogger.log_trade_opened[       *>$..*>*>*XQW*Xr&   c                <     | j                   j                  di | y )N)trade_closedrW   rX   s     r$   log_trade_closedzFakeLogger.log_trade_closed\   rZ   r&   N)r4   r5   r6   r>   rO   rT   rY   r]   r7   r&   r$   rF   rF   P   s    40IXXr&   rF   c                      e Zd Zd Zy)FakeProviderc                :   K   t        j                  g d      S wN)openhighlowclosevolume)columnspd	DataFramer/   symbol	timeframens       r$   get_barszFakeProvider.get_bars`        ||$NOO   N)r4   r5   r6   ro   r7   r&   r$   r_   r_   _   s    Pr&   r_   c                      e Zd Zd Zy)_MemoryStorec                     y r;   r7   )r/   states     r$   savez_MemoryStore.savee   s    r&   N)r4   r5   r6   rv   r7   r&   r$   rs   rs   d   s    r&   rs   c                      e Zd ZdZd Zd Zd Zd Zd Zd Z	ddZ
dd	Zdd
Zd Zd Zd Zd Zd ZddZd Zd Zd Zd Zy)
FakeBrokerz*Programmable BrokerBase for C2b scenarios.c                X    g | _         d| _        d| _        d| _        d | _        d | _        y )NTr   F)calls	connectedconnect_attemptsfail_connectraise_on_balanceraise_on_closer=   s    r$   r>   zFakeBroker.__init__l   s0     "
 !!6:48r&   c                ^   K   | xj                   dz  c_         | j                  ryd| _        yw)N   FT)r|   r}   r{   r=   s    r$   connectzFakeBroker.connectu   s-     "s   +-c                P   K   | j                   j                  d       d| _        y w)N
disconnectF)rz   rA   r{   r=   s    r$   r   zFakeBroker.disconnect{   s     

,'s   $&c                "   K   | j                   S wr;   )r{   r=   s    r$   is_connectedzFakeBroker.is_connected~   s     "7   c                   K   yw)N     @r7   r/   rl   s     r$   get_last_pricezFakeBroker.get_last_price   s     6   Nc                D   K   | j                   j                  d       g S w)Npositions_getrz   rA   r   s     r$   r   zFakeBroker.positions_get   s     

/*	s    c                   K   g S wr;   r7   r   s     r$   pending_orderszFakeBroker.pending_orders   s
        c                   K   g S wr;   r7   )r/   rl   sincelimits       r$   recent_tradeszFakeBroker.recent_trades   s
     Rir   c           	     d   K   | j                   j                  d       t        ddddddd	      S w)
Nplace_market_bracketTr        @     @EST)successentry_pricesl_pricetp_priceentry_idstop_id	target_id)rz   rA   r   )r/   kws     r$   r   zFakeBroker.place_market_bracket   s8     

014Vf$*S#QTV 	Vs   .0c                (   K   t        d|dd      S w)NTr   )r   r   r   r   r   )r/   rl   side	contracts
stop_prices        r$   place_stop_orderzFakeBroker.place_stop_order   s     4*cTWXX   c                (   K   t        d|dd      S w)NTr   )r   r   r   r   r   )r/   rl   r   r   limit_prices        r$   place_limit_orderzFakeBroker.place_limit_order   s     4+WZ[[r   c                "   K   t        d      S w)NTr   )r	   )r/   rl   order_ids      r$   cancel_orderzFakeBroker.cancel_order   s     <PT;U4Ur   c                   K   ywNr   r7   r   s     r$   cancel_all_for_symbolz FakeBroker.cancel_all_for_symbol   s     !r   c                   K   | j                   j                  d       | j                  | j                  }d | _        |t        d      S w)Nclose_positionTr   )rz   rA   r   r   )r/   rl   r   excs       r$   r   zFakeBroker.close_position   sH     

*+*%%C"&DI4((s   A	Ac	                ,   K   t        dddd||      S w)NTzC-FAKEzS2-FAKEzT2-FAKE)r   r   r   r   r   r   r   )	r/   rl   	directioncontracts_to_closeresidual_contractsnew_sl_pricenew_tp_priceold_stop_order_idold_target_order_ids	            r$    partial_close_via_opposite_orderz+FakeBroker.partial_close_via_opposite_order   s#      8Y)!L
 	
   c                $   K   t        d|      S w)NT)r   r   r   r/   rl   r   r   s       r$   modify_stopzFakeBroker.modify_stop   s     4,??s   c                   K   | j                   j                  d       | j                  | j                  }d | _        |yw)Nget_account_balanceg     j@)rz   rA   r~   )r/   r   s     r$   r   zFakeBroker.get_account_balance   s@     

/0  ,''C$(D!Is   >A c                :   K   t        j                  g d      S wra   rh   rk   s       r$   
fetch_barszFakeBroker.fetch_bars   rp   rq   r;   )NN2   )r4   r5   r6   __doc__namer>   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r7   r&   r$   rx   rx   h   sa    4D9 89 ;OVY\U;)
@Pr&   rx   c                  *    e Zd ZdddZddd	dZd Zy)
FakeReconcilerFc                .    g | _         d| _        || _        y r   )startup_callswatchdog_calls_raise)r/   raise_in_reconciles     r$   r>   zFakeReconciler.__init__   s    )+(r&   )post_reconnectc               p   K   | j                   rt        d      | j                  j                  |       y w)Nzreconcile boom)r   RuntimeErrorr   rA   )r/   r   s     r$   reconcile_startupz FakeReconciler.reconcile_startup   s0     ;;/00!!.1s   46c                6   K   | xj                   dz  c_         y w)Nr   )r   r=   s    r$   watchdog_naked_positionsz'FakeReconciler.watchdog_naked_positions   s     q s   N)F)r   bool)r   r   )r4   r5   r6   r>   r   r   r7   r&   r$   r   r      s    )
 AF r&   r   c                  X    e Zd ZdZej
                  j                  dfdZddddZd Z	y)		FakeBrainz2Driver that returns canned BrainDecision per call.cannedc                .    || _         || _        d| _        y r   )actionreasonrz   )r/   r   r   s      r$   r>   zFakeBrain.__init__   s    
r&   N        )	bias_datalast_entry_eval_timec               "   K   t        d       S w)N)decision)r   )r/   rl   techr   r   s        r$   evaluate_entryzFakeBrain.evaluate_entry   s     --r   c                z   K   | xj                   dz  c_         t        | j                  | j                  d i       S w)Nr   r   r   
move_sl_tometadata)rz   r   r   r   )r/   ctxs     r$   manage_exitzFakeBrain.manage_exit   s2     

a
;;t{{b
 	
s   9;)
r4   r5   r6   r   r   HOLDvaluer>   r   r   r7   r&   r$   r   r      s.    <)..44X  ?CY\ .
r&   r   i           )tzinfor   FIXED_DAY_UTC      )	broker
reconcilermax_iterationsbrain_dispatchopenercloserrisk_managerru   degraded_max_minutesc        	        \   t        t        j                  t        j                        }	dg|	_        d|	_        d|	_        d|	_        d|	_	        d|	_
        ||	_        |xs
 t               }t               }
t        |	t               t!               |t#               |
|xs i |||| |d |      }||
fS )NmodeaccountMESr   r   r   c                     t         S r;   r   r7   r&   r$   <lambda>zmake_orch.<locals>.<lambda>        r&   config	ai_clientmarket_data_providerru   storeloggerr   r   r   r   r   r   now_utc_providerr   )r   r   LIVEr   
INELIGIBLEasset_filterloop_sleep_secondsscan_loop_phase_offset_secondsmanage_loop_interval_seconds!maintenance_loop_interval_secondsreconcile_interval_iterationsbroker_degraded_max_minutesr    rF   r   r(   r_   rs   )r   r   r   r   r   r   r   ru   r   cfgr  orchs               r$   	make_orchr     s     W\\;3I3I
JCwCC),C&'(C$,-C)()C%&:C##\^E\Ffh)^<>&%+f<*.%	D <r&   c                    ddl m}   | dDi ddddddd	dd
ddddddddddddddddddddt        j                  j                  ddd dd!d"d#t
        j                  j                  d$d%d&g d'dd(d)d*dd+dd,d-d.dd/dd0dd1dd2dd3d4d5dd6dd7dd8dd9dd:dd;dd<dd=dd>d?d@dAdBddCd%S )Eu   Build a TechSnapshot with safe defaults — used when manage_exit
    is the system under test, not the indicator computation. We only
    need build_tech to NOT return None so the action dispatch is reached.r   TechSnapshotrl   r  pricer   rb   g     @candle_timeis_candle_closedFcandle_age_secondsr   rsi     K@rsi_prevg      K@rsi_h1      J@rsi_h4      I@atr_m5_pointsg       @	atr_ratio      ?
vol_regimeNORMAL	vol_spikemarket_structureh1_struct_bullTh1_struct_beartrend_maturityr   regimeregime_reasontestregime_near_trendingdeviation_pct
divergenceNONEmacd_deceleratingmacd_hist_lastcandle_strengthg      ?hammershooting_starbull_engulfingbear_engulfingdoji	doji_typeNpiercing
dark_cloudmorning_starevening_starvolume_weakbuy_absorptionsell_absorptionvwapvwap_deviation_pctbiasNEUTROallowed_directionBOTHh1_compatibility	h1_reasonr7   )analysis.tech_snapshotr  r   BULLISH_EXPANSIONr   r   TRENDINGr  s    r$   _minimal_techrV     s    4 ")/=>36    )- 6: 	 &)	 6>	 JO	
 )::@@  -2 CD $$ 5; RT  '- AF   %* :?  .3 :? KO  $) 8= LQ  +0 BG  ),  *0  !  )/! r&   c                2    t               fd}|| _        y )Nc                   K   S wr;   r7   )_symbolsnaps    r$   
_fake_techzstub_tech.<locals>._fake_tech	  s     $;s   )rV  _build_tech)r  r[  rZ  s     @r$   	stub_techr]    s    ?D.!Dr&   c           	        t        d"i d| dt        j                  j                  dt        j
                  j                  d|ddddd	d
dt        j                  t        j                        t        d      z
  dddddddddt        j                  j                  dt        j                  j                  dddddddddd }t        |t!               !      S )#Nrl   
brain_namer   r   r   r   r   r   r   r   	opened_at   minutesrsi_m5_at_entryr$  rsi_h1_at_entryr'  rsi_h4_at_entryr)  atr_ratio_at_entryr,  market_structure_at_entryregime_at_entryh1_compat_at_entryconfidence_at_entryF   entry_order_idE1stop_order_idS1target_order_idT1)entryruntimer7   )r   r   TFr   r   BUYr   nowr   utcr   r   rT  r   rU  r   r   )rl   r   rs  s      r$   make_active_tradery    s    "+,,"4"4--%%1:  &, 7= ,,x||,y/CC	
 
 /3
 EI  #2"C"C"I"I --  57  ,0 BFE ULN;;r&   c                    t        t        j                  t        j                        } dg| _        d| _        d| _        d| _        d| _	        t        | t               t               t               t               t               i dd	      }|j!                         }t#        |t$              }|s-t'        j(                  dt+        |             d	z   d
t-        j.                         v st'        j0                  t"              rt'        j2                  t"              nd
dt-        j.                         v st'        j0                  |      rt'        j2                  |      nddt-        j.                         v st'        j0                  t$              rt'        j2                  t$              ndt'        j2                  |      dz  }t5        t'        j6                  |            d}|j8                  }d}||u}|st'        j:                  d|fd||f      dt-        j.                         v st'        j0                  |      rt'        j2                  |      ndt'        j2                  |      t'        j2                  |      dz  }t'        j(                  d      dz   d|iz  }t5        t'        j6                  |            dx}x}}t=        t%        j>                  t@        jB                        |z
  jE                               }	d}
|	|
k  }|st'        j:                  d|fd|	|
f      dt-        j.                         v st'        j0                  |	      rt'        j2                  |	      ndt'        j2                  |
      dz  }t'        j(                  d|	 d      dz   d|iz  }t5        t'        j6                  |            dx}}
tG        d       y)a^  
    Regression: a global replace_all of datetime.now(timezone.utc) in
    orchestrator.py once turned the default fallback for now_utc_provider
    into a self-referential lambda (`lambda: self._now_utc()`), causing
    RecursionError on first call in production. Verify the default returns
    a real tz-aware datetime, not infinite recursion.
    r  r  r   r   Nr   )	r
  r  r  ru   r  r  r   r   r   z#_now_utc must return datetime, got z7
>assert %(py4)s
{%(py4)s = %(py0)s(%(py1)s, %(py2)s)
}
isinstancerw  r   )py0py1py2py4is not)z2%(py2)s
{%(py2)s = %(py0)s.tzinfo
} is not %(py5)sr|  r~  py5z!_now_utc default must be tz-aware
>assert %(py7)spy7g      @)<)z%(py0)s < %(py3)sdeltar|  py3z*_now_utc default drifted from wall clock: s
>assert %(py5)sr  zAclock injection default: real UTC tz-aware datetime, no recursion)$r   r   PAPERr   r  r  r  r  r  r  r   r(   r_   r    rs   rF   _now_utcr{  r   
@pytest_ar_format_assertmsgtype@py_builtinslocals_should_repr_global_name	_safereprAssertionError_format_explanationr   _call_reprcompareabsrw  r   rx  total_secondsr%   )r  r  rw  @py_assert3@py_format5@py_assert1@py_assert4@py_format6@py_format8r  @py_assert2@py_format4s               r$   5test_now_utc_default_is_real_clock_not_self_referencer  !  s-    W]]K4J4J
KCwCC),C&'(C$,-C)fh)^nLN:<$D --/Cc8$W$WW(KDQTI;&WWWWWWW:WWW:WWWWWWcWWWcWWWWWW8WWW8WWW$WWWWWW::FTF:T!FFF:TFFFFFF3FFF3FFF:FFFTFFF#FFFFFFFFhll+c1@@BCEM53;MMM53MMMMMM5MMM5MMM3MMMDUG1MMMMMMMKLr&   c                 P   t        d      } t        |       }d}||k(  }|st        j                  d|fd||f      dt	        j
                         v st        j                  t              rt        j                  t              nddt	        j
                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd	|iz  }t        t        j                  |            d x}x}}t        d
       y )Nz"WebSocket closed: connection reset	transient==)z0%(py3)s
{%(py3)s = %(py0)s(%(py1)s)
} == %(py6)sr   e)r|  r}  r  py6zassert %(py8)spy8z;classify: ConnectionError + 'connection reset' -> transient)ConnectionErrorr   r  r  r  r  r  r  r  r  r%   )r  r  @py_assert5r  @py_format7@py_format9s         r$   %test_classify_broker_error_connectionr  D  s    <=A #2{2#{2222#{222222 222 222222222222#222{2222222EFr&   c            	         G d dt               } d} | |      }t        |      }d}||k(  }|st        j                  d|fd||f      dt	        j
                         v st        j                  t              rt        j                  t              nddt	        j
                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      t        j                  |      t        j                  |      dz  }d	d
|iz  }t        t        j                  |            d x}x}x}x}}t        d       y )Nc                      e Zd Zy)0test_classify_broker_error_auth.<locals>.Auth401N)r4   r5   r6   r7   r&   r$   Auth401r  K  s    r&   r  zHTTP 401 unauthorizedauthr  )zO%(py7)s
{%(py7)s = %(py0)s(%(py5)s
{%(py5)s = %(py1)s(%(py3)s)
})
} == %(py10)sr   )r|  r}  r  r  r  py10zassert %(py12)spy12z$classify: '401 unauthorized' -> auth)	Exceptionr   r  r  r  r  r  r  r  r  r%   )r  r  r  @py_assert6@py_assert9@py_assert8@py_format11@py_format13s           r$   test_classify_broker_error_authr  J  s    ")")@L)@!AL !ABLfLBfLLLLBfLLLLLL LLL LLLLLLLLLLLL)@LLL!ALLLBLLLfLLLLLLL./r&   c                    t               } t        d      | _        t        | d      \  }}|j	                  dt        d             |j
                  }d}||u }|st        j                  d|fd||f      d	t        j                         v st        j                  |      rt        j                  |      nd	t        j                  |      t        j                  |      d
z  }dd|iz  }t        t        j                  |            dx}x}}|j                  }d}||u}|st        j                  d|fd||f      d	t        j                         v st        j                  |      rt        j                  |      nd	t        j                  |      t        j                  |      d
z  }dd|iz  }t        t        j                  |            dx}x}}t        d       y)zA broker raise during _refresh_account_balance routes through
    _maybe_handle_broker_exception, but balance fetch swallows exception
    in the existing C1 path. Verify direct exception path: scan with a
    raising opener marks degraded.zws disconnectr   r   r   r6  ws dropTisz8%(py2)s
{%(py2)s = %(py0)s._broker_degraded
} is %(py5)sr  r  assert %(py7)sr  Nr  )z;%(py2)s
{%(py2)s = %(py0)s._degraded_since
} is not %(py5)szH_maybe_handle_broker_exception: ConnectionError -> degraded mode entered)rx   r  r~   r  _maybe_handle_broker_exception_broker_degradedr  r  r  r  r  r  r  r  _degraded_sincer%   )r   r  _r  r  r  r  r  s           r$   $test_broker_exception_marks_degradedr  P  s,   
 \F-o>Fva8GD! 	''	0JK  (D( D(((( D((((((4(((4((( (((D(((((((+t+t++++t++++++4+++4++++++t+++++++RSr&   c                   
 t               } t               }t        | |d      \  
}
fd}t        j                   |              
j
                  }d}||u }|st        j                  d|fd||f      dt        j                         v st        j                  
      rt        j                  
      ndt        j                  |      t        j                  |      dz  }t        j                  d	      d
z   d|iz  }t        t        j                  |            dx}x}}|j                  }dg}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      dz  }t        j                  d|j                         d
z   d|iz  }t        t        j                  |            dx}x}}d |j                   j"                  D        }t%        |      }|sddt        j                         v st        j                  t$              rt        j                  t$              ndt        j                  |      t        j                  |      dz  }	t        t        j                  |	            dx}}t'        d       y)zSReconnect succeeds -> degraded cleared, reconciler called with post_reconnect=True.r   r   r   r   c                 x   K    j                  dt        d              j                          d {    y 7 wNr6  r  _mark_degradedr  runr  s   r$   runnerz0test_mid_loop_reconnect_succeeds.<locals>.runnerj  s+     FOI$>?hhj   /:8:Fr  r  r  r  z'must recover after successful reconnectr  r  NTr  z5%(py2)s
{%(py2)s = %(py0)s.startup_calls
} == %(py5)srecz5reconciler called once with post_reconnect=True, got c              3  ,   K   | ]  }|d    dk(    yw)r@   broker_degraded_exitedNr7   .0r  s     r$   	<genexpr>z3test_mid_loop_reconnect_succeeds.<locals>.<genexpr>s  s     W!qz55Wr   ,assert %(py4)s
{%(py4)s = %(py0)s(%(py2)s)
}anyr|  r~  r  zKmid-loop reconnect succeeds: degraded cleared, post-reconnect reconcile run)rx   r   r  asyncior  r  r  r  r  r  r  r  r  r  r  r   rI   r<   r  r%   )r   r  r  r  r  r  r  r  r  r  r  s             @r$    test_mid_loop_reconnect_succeedsr  d  s   \F

CFs1MLD&
 KK  TET E)TTT ETTTTTT4TTT4TTT TTTETTT+TTTTTTTT T T& TBSBST T TMSVT T;S;S  T TJS)  T TJS)  T TJS) !' T TBSBS
?@Q@Q?RST T T@S@ST TWv?O?O?V?VWW3WWWWWWWWW3WWW3WWWWWWWWWWWWWWUVr&   c                 (  	 t               } d| _        t               }t        | |d      \  	}	fd}t	        j
                   |              	j                  }d}||u }|st        j                  d|fd||f      dt        j                         v st        j                  	      rt        j                  	      ndt        j                  |      t        j                  |      dz  }t        j                  d	      d
z   d|iz  }t        t        j                  |            dx}x}}|j                   }g }||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      dz  }t        j                  d      d
z   d|iz  }t        t        j                  |            dx}x}}t#        d       y)z5Reconnect fails -> stays degraded, no reconcile call.Tr   r  c                 x   K    j                  dt        d              j                          d {    y 7 wr  r  r  s   r$   r  z<test_mid_loop_reconnect_fails_stays_degraded.<locals>.runner~  s+     FOI$>?hhjr  r  r  r  r  z+must remain degraded after failed reconnectr  r  Nr  r  r  z'no reconcile call when reconnect failedz>mid-loop reconnect fails: stays degraded, no reconcile attempt)rx   r}   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r   r%   )
r   r  r  r  r  r  r  r  r  r  s
            @r$   ,test_mid_loop_reconnect_fails_stays_degradedr  w  sF   \FF

Cv#aHGD! KK  WDW D(WWW DWWWWWW4WWW4WWW WWWDWWW*WWWWWWWWMM"MMMMMMMMM3MMM3MMMMMMMMM$MMMMMMMMHIr&   c                 &  	 t               } t        | d      \  }}|j                  dt        d             d| _        ddi		fd}||_        t        j                  |j                                	d   }d}||k(  }|st        j                  d	|fd
||f      t        j                  |      t        j                  |      dz  }t        j                  d      dz   d|iz  }t        t        j                  |            dx}x}}t        d       y)z(_scan_entries not called while degraded.r   r  r6  r  Trn   r   c                 (   K    dxx   dz  cc<   y w)Nrn   r   r7   )scan_calleds   r$   _patched_scanz/test_degraded_skips_scan.<locals>._patched_scan  s     CAs   r  )z%(py1)s == %(py4)s)r}  r  z)_scan_entries must NOT run while degradedz
>assert %(py6)sr  Nz/safe mode: _scan_entries skipped while degraded)rx   r  r  r  r}   _scan_entriesr  r  r  r  r  r  r  r  r%   )
r   r  r  r  @py_assert0r  r  r  r  r  s
            @r$   test_degraded_skips_scanr    s    \Fva8GD!	 :;F(K&DKK
sMqMq MMMqMMMMMMqMMM"MMMMMMMM9:r&   c                    t               } t               }t               |j                  d<   t	        t
        j                  j                  d      }d| _        t        | t        j                  j                  |i|d      \  }}t        |       |j                  dt        d             t        j                   |j!                                d	}| j"                  }||v}|st%        j&                  d
|fd||f      t%        j(                  |      dt+        j,                         v st%        j.                  |       rt%        j(                  |       ndt%        j(                  |      dz  }t%        j0                  d      dz   d|iz  }	t3        t%        j4                  |	            dx}x}}d}|j                  }||v }|st%        j&                  d|fd||f      t%        j(                  |      dt+        j,                         v st%        j.                  |      rt%        j(                  |      ndt%        j(                  |      dz  }t%        j0                  d      dz   d|iz  }	t3        t%        j4                  |	            dx}x}}d |j6                  j8                  D        }
t;        |
      }|sddt+        j,                         v st%        j.                  t:              rt%        j(                  t:              ndt%        j(                  |
      t%        j(                  |      dz  }t3        t%        j4                  |            dx}
}t=        d       y)zSWhen manage_exit returns EXIT during degraded, broker.close_position is NOT called.r  zexit-while-downr   r   Tr   r   r   ru   r   r6  r  r   not inz1%(py1)s not in %(py5)s
{%(py5)s = %(py3)s.calls
}r   r}  r  r  u;   EXIT must be deferred — close_position MUST NOT be calledr  r  Ninz5%(py1)s in %(py5)s
{%(py5)s = %(py3)s.active_trades
}ru   z%trade preserved through deferred EXITc              3  ,   K   | ]  }|d    dk(    ywr@   exit_deferred_disconnectedNr7   r  s     r$   r  z3test_degraded_defers_exit_action.<locals>.<genexpr>  "      1 z99 1r   r  r  r  zAsafe mode: EXIT deferred (close_position NOT called, log emitted))rx   r    ry  active_tradesr   r   EXITr   r}   r  r   ru  r]  r  r  r  r  rz   r  r  r  r  r  r  r  r  r  rI   r<   r  r%   )r   ru   brainr  r  r  r  r  r  r  r  r  r  s                r$    test_degraded_defers_exit_actionr    s   \FNE!2!4E[--33<MNEFy||'9'95&AALD& dO	 :;KK
 F6<< F</ F4E4EF< F F<EI  F F?EvF F-E-E $* F F<EI $* F F<EI $0 F F4E4EEF F F2E2EF FPE''P5''PPP5'PPP5PPPPPPEPPPEPPP'PPP)PPPPPPPP1((//1 13 1 1 1 1 1*0&1 100  1 1'0y  1 1'0y1 1 1'0y1 1 1 1001 1KLr&   c                    t               } d| _        t               }t        | |d      \  }}|j	                  dt        d             t        j                  |j                                |j                  }d}||k(  }|st        j                  d|fd||f      d	t        j                         v st        j                  |      rt        j                  |      nd	t        j                  |      t        j                  |      d
z  }t        j                  d      dz   d|iz  }t!        t        j"                  |            dx}x}}t%        d       y)z#Watchdog not called while degraded.Tr   r  r6  r  r   r  )z6%(py2)s
{%(py2)s = %(py0)s.watchdog_calls
} == %(py5)sr  r  z$watchdog must NOT run while degradedr  r  Nz*safe mode: watchdog skipped while degraded)rx   r}   r   r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r%   	r   r  r  r  r  r  r  r  r  s	            r$   test_degraded_skips_watchdogr    s    \FF

Cv#aHGD!	 :;KK
JJ"JJJJJJJJJ3JJJ3JJJJJJJJJ$JJJJJJJJ45r&   c                    t               } d| _        t        | dd      \  }}|j                  dt	        d             t
        t        d      z
  |_        t        j                  |j                               }|t        k(  }|st        j                  d	|fd
|t        f      dt        j                         v st        j                  |      rt        j                   |      nddt        j                         v st        j                  t              rt        j                   t              nddz  }t        j"                  dt         d|       dz   d|iz  }t%        t        j&                  |            d}t)        dt                y)zNdegraded > broker_degraded_max_minutes -> exit code EXIT_BROKER_UNRECOVERABLE.T
   r   )r   r   r   r6  r     rb  r  )z%(py0)s == %(py2)srcr   )r|  r~  zexpected exit code z, got z
>assert %(py4)sr  Nz,hard timeout: degraded > 15min -> exit code )rx   r}   r  r  r  r   r   r  r  r  r   r  r  r  r  r  r  r  r  r  r%   )r   r  r  r  r  @py_format3r  s          r$   #test_hard_timeout_exits_with_code_4r    sV   \FFvbrRGD!	 :; )9R+@@D	TXXZ	 B** D2C2CD2* D D=CVD D+C+C  D D:C)  D D=CVD D+C+C + D D:C) + D D2C2C
78rdCD D D0C0CD D
67P6QRSr&   c                 r   t               } t               }t        | |d      \  }}|j                  dt	        d             t        j                  |j                                |j                  }dg}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      d	z  }d
d|iz  }t        t        j                  |            dx}x}}t!        d       y)z;After recovery, reconcile_startup gets post_reconnect=True.r   r  startupr6  Tr  r  r  r  r  r  Nz>post-reconnect: reconcile_startup(post_reconnect=True) invoked)rx   r   r  r  r  r  r  r   r  r  r  r  r  r  r  r  r%   r  s	            r$   7test_reconcile_startup_called_with_post_reconnect_kwargr    s    \F

Cv#aHGD!	?6#:;KK
&&&&&&&&&&&&3&&&3&&&&&&&&&&&&&HIr&   c                    t               } t        d      }t        | |d      \  }}|j                  dt	        d             t        j                  |j                               }d}||k(  }|st        j                  d|fd	||f      d
t        j                         v st        j                  |      rt        j                  |      nd
t        j                  |      dz  }t        j                  d|       dz   d|iz  }t        t        j                  |            dx}}|j                   }d}	||	u }
|
st        j                  d|
fd||	f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |	      dz  }t        j                  d      dz   d|iz  }t        t        j                  |            dx}x}
}	t#        d       y)zGIf reconciler.reconcile_startup raises after reconnect, loop continues.T)r   r   r  r6  r  r   r  )z%(py0)s == %(py3)sr  r  z1loop must continue after reconcile error, got rc=r  r  NFr  r  r  r  z(still recovered (reconcile is non-fatal)r  r  z5post-reconnect reconcile error: loop continues (rc=0))rx   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r%   )r   r  r  r  r  r  r  r  r  r  r  r  s               r$   7test_post_reconnect_reconcile_failure_doesnt_break_loopr    s=   \F
D
1Cv#aHGD!	 :;	TXXZ	 BL27LLL2LLLLLL2LLL2LLLLLLGtLLLLLLL  UEU E)UUU EUUUUUU4UUU4UUU UUUEUUU+UUUUUUUU?@r&   c                    t               } t               }d|j                  _        || j                  d<    G d dt
              } |       }t        t        j                  j                  d      }d }||_
        t        |t        j                  j                  |i| d	      \  }}t        |       t        j                   |j#                                | j                  d   }|j                  }	|	j                  }
d}|
|k(  }|st%        j&                  d
|fd|
|f      t%        j(                  |      t%        j(                  |	      t%        j(                  |
      t%        j(                  |      dz  }t%        j*                  d      dz   d|iz  }t-        t%        j.                  |            dx}x}	x}
x}}t1        d       y)a  
    broker.modify_stop raises -> runtime.current_sl_price NOT updated
    (broker is authoritative; state lying is worse than missed update).

    3-loop arch: drive _manage_open_trades() directly instead of orch.run()
    so we don't depend on scan/manage/maintenance interleaving. The
    custom broker overrides positions_get to return a non-empty list so
    _check_external_close (now run before manage_exit) doesn't short-
    circuit the flow with state cleanup.
    r   r  c                      e Zd ZddZd Zy)Etest_modify_stop_failure_runtime_unchanged.<locals>._BrokerModifyFailNc                L   K   | j                   j                  d       dddgS w)Nr   r  r   rl   sizer   r   s     r$   r   zStest_modify_stop_failure_runtime_unchanged.<locals>._BrokerModifyFail.positions_get  s&     JJo.$a011s   "$c                    K   t        d      w)Nzws drop during modify_stop)r  r   s       r$   r   zQtest_modify_stop_failure_runtime_unchanged.<locals>._BrokerModifyFail.modify_stop  s     !">??s   r;   )r4   r5   r6   r   r   r7   r&   r$   _BrokerModifyFailr    s    	2	@r&   r  trailr  c                X   K   t        t        j                  j                  ddi       S w)Nr  g     @r   )r   r   MOVE_SLr   )r   s    r$   _manage_with_movezEtest_modify_stop_failure_runtime_unchanged.<locals>._manage_with_move  s+     &&,,	
 	
s   (*r   r  r  )zU%(py5)s
{%(py5)s = %(py3)s
{%(py3)s = %(py1)s.runtime
}.current_sl_price
} == %(py8)s)r}  r  r  r  zCmodify_stop failure: runtime.current_sl_price MUST stay at originalz
>assert %(py10)sr  NzLmodify_stop fails: runtime.current_sl_price unchanged (broker authoritative))r    ry  rt  current_sl_pricer  rx   r   r   r  r   r   r  r   ru  r]  r  r  _manage_open_tradesr  r  r  r  r  r  r%   )ru   atr  r   r  r  r  _loggerr  r  r  @py_assert7r  r  r  s                  r$   *test_modify_stop_failure_runtime_unchangedr    s    NE		B"(BJJ!#E@J @  F[0066wGE
 *Ey||'9'95&AAMD' dOKK((*+u% N%-- N->> N& N>&H N<M<MN>& N NDMI & N NDMI . N NDMI ? N NDMI CI N N<M<MMN N N:M:MN N NVWr&   c                 J   ddl m}  ddlm}  G d dt              } |       }t               }t        d      |j                  d<   t               }t        t        j                  j                  d	
      }t        t        j                  t         j"                        }dg|_        d|_        d|_        d|_        d|_        d|_         | |dt1                     } |||t1                     }	t1               }
t3        |t5               t7               |t9               |
t:        j<                  j                  |id||	||d d      }t?        |       |jA                  dtC        d             d|_"        tG        jH                  |jK                                d}|j                  }||v }|stM        jN                  d|fd||f      tM        jP                  |      dtS        jT                         v stM        jV                  |      rtM        jP                  |      ndtM        jP                  |      dz  }tM        jX                  d      dz   d|iz  }t[        tM        j\                  |            dx}x}}d}|j^                  }||v}|stM        jN                  d|fd ||f      tM        jP                  |      d!tS        jT                         v stM        jV                  |      rtM        jP                  |      nd!tM        jP                  |      dz  }tM        jX                  d"      dz   d|iz  }t[        tM        j\                  |            dx}x}}d# |
j`                  jb                  D        }te        |      }|stM        jX                  d$      d%z   d&tS        jT                         v stM        jV                  td              rtM        jP                  td              nd&tM        jP                  |      tM        jP                  |      d'z  }t[        tM        j\                  |            dx}}d|_"        tG        jH                  |jg                                |jh                  }d}||u }|stM        jN                  d(|fd)||f      d*tS        jT                         v stM        jV                  |      rtM        jP                  |      nd*tM        jP                  |      tM        jP                  |      d+z  }tM        jX                  d,      dz   d|iz  }t[        tM        j\                  |            dx}x}}tG        jH                  |jK                                d}|j^                  }||v }|stM        jN                  d|fd-||f      tM        jP                  |      d!tS        jT                         v stM        jV                  |      rtM        jP                  |      nd!tM        jP                  |      dz  }tM        jX                  d.|j^                         dz   d|iz  }t[        tM        j\                  |            dx}x}}d}|j                  }||v}|stM        jN                  d|fd/||f      tM        jP                  |      dtS        jT                         v stM        jV                  |      rtM        jP                  |      ndtM        jP                  |      dz  }tM        jX                  d0      dz   d|iz  }t[        tM        j\                  |            dx}x}}tk        d1       y)2a  
    Trade open + brain says EXIT during disconnect (deferred), reconnect,
    brain re-says EXIT, executes successfully -> trade closed.

    3-loop arch: drive each phase directly. orch.run() races
    manage_loop and maintenance_loop; here we sequence the same effects
    by calling _manage_open_trades() (phase 1), then _maybe_reconnect()
    + _manage_open_trades() (phase 2). Same invariants, no interleaving.
    r   )TradeCloser)RiskManagerc                      e Zd ZddZy)=test_e2e_disconnect_defer_resume.<locals>._BrokerWithPositionNc                l   K   | j                   j                  d       d| j                   v rg S dddgS w)Nr   r   r  r   r  r   r   s     r$   r   zKtest_e2e_disconnect_defer_resume.<locals>._BrokerWithPosition.positions_get8  s7     JJo.4::-	$a011s   24r;   )r4   r5   r6   r   r7   r&   r$   _BrokerWithPositionr  3  s    
	2r&   r!  r   )r   r  zexit-nowr  r  r   F)r   is_paperr  )r  Nc                     t         S r;   r  r7   r&   r$   r  z2test_e2e_disconnect_defer_resume.<locals>.<lambda>W  r  r&   r   r	  r6  r  Tr  r  ru   r  zphase 1: trade preservedr  r  r   r  r  r   zphase 1: close NOT attemptedc              3  ,   K   | ]  }|d    dk(    ywr  r7   r  s     r$   r  z3test_e2e_disconnect_defer_resume.<locals>.<genexpr>d  r  r   z8phase 1: exit_deferred_disconnected event must be loggedz.
>assert %(py4)s
{%(py4)s = %(py0)s(%(py2)s)
}r  r  r  r  r  r  zphase 2: reconnect must recover)z-%(py1)s in %(py5)s
{%(py5)s = %(py3)s.calls
}z.phase 2: close_position must be called, calls=)z9%(py1)s not in %(py5)s
{%(py5)s = %(py3)s.active_trades
}z"phase 2: trade removed after closezFE2E: trade preserved through disconnect, closed cleanly post-reconnect)6trading.trade_closerr  trading.risk_managerr  rx   r    ry  r  r   r   r   r  r   r   r   r  r   r  r  r  r  r  r  r  rF   r   r(   r_   rs   r   ru  r]  r  r  r}   r  r  r  r  r  r  r  r  r  r  r  r  rz   rI   r<   r  _maybe_reconnectr  r%   )r  r  r!  r   ru   r  r  r  r   r   r  r  r  r  r  r  r  r  r  r  s                       r$    test_e2e_disconnect_defer_resumer(  &  s3    10	2j 	2 !"FNE!2Q!?E

C[--33JGE
W\\;3I3I
JCwCC),C&'(C$,-C)()C%z|LFsE*,?L\Ffh)^<>&!**E2F#.	D dO
 		 :;FKK((*+CE''C5''CCC5'CCC5CCCCCCECCCECCC'CCC)CCCCCCCCO6<<O</OOO<OOOOOOOOO6OOO6OOO<OOO1OOOOOOOO1((//1 C3 1 1 C 1 C1B1BBC C<BFC C*B*B  C C9B  C C9B1 C C9B1 C C C/B/BC C
  FKK%%'(  LEL E)LLL ELLLLLL4LLL4LLL LLLELLL+LLLLLLLLKK((*+ Hv|| H|+ H6G6GH| H H>Gi  H HAGH H/G/G  & H H>Gi  & H H>Gi  , H H6G6G
8GH H H4G4GH HQ++Q5++QQQ5+QQQ5QQQQQQQQQQQQ+QQQ-QQQQQQQQPQr&   __main__ztest_orchestrator_phase_c2b.pyzALL 14 TESTS PASSED)r   intreturntuple)r+  z'TechSnapshot')r  r   )r+  r   )Vr   
__future__r   builtinsr  _pytest.assertion.rewrite	assertionrewriter  r  sysr   r   r   pathlibr   pandasri   pathinsertstr__file__resolveparentbroker.broker_baser   r	   r
   r   r   r   core.configr   r   r   core.contractsr   r   r   r   r   r   r   r   r   r   r   orchestratorr   r   r   persistence.state_storer   r    r%   r(   r9   rF   r_   rs   rx   r   r   rx  r   __annotations__r  rV  r]  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  r(  r4   r"   r7   r&   r$   <module>rA     s  !F #    
 2 2   3tH~--/66==> ?  < ;   
  > (; ;U U
Y YP P
   @P @PF "
 
( #4BAqNx N a!$tt*, %( 2762"<(MFG0T(W&J*;"M.	6 T(J	A ,XfJRb z	
*+9;)+#%(*$&02$& "');=;=.0$&	
 ! r&   