431 lines
17 KiB
Python
431 lines
17 KiB
Python
import json
|
|
import logging
|
|
from datetime import datetime
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.config import settings
|
|
from app.models.models import (ArbitrageEvent, Block, Instruction, Swap,
|
|
TokenTransfer, Transaction)
|
|
from app.schemas.schemas import (ArbitrageEventCreate, BlockCreate,
|
|
TokenTransferCreate, TransactionCreate)
|
|
from app.services.solana_client import SolanaClient
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class BlockScanner:
|
|
"""
|
|
Scanner service for processing Solana blocks and detecting arbitrage
|
|
"""
|
|
|
|
def __init__(self, db: Session, solana_client: Optional[SolanaClient] = None):
|
|
"""
|
|
Initialize scanner with database session and Solana client
|
|
"""
|
|
self.db = db
|
|
self.solana_client = solana_client or SolanaClient()
|
|
self._scan_in_progress = False
|
|
|
|
@property
|
|
def scan_in_progress(self) -> bool:
|
|
"""
|
|
Check if a scan is currently in progress
|
|
"""
|
|
return self._scan_in_progress
|
|
|
|
def get_last_scanned_block(self) -> Optional[int]:
|
|
"""
|
|
Get the latest block that was scanned
|
|
"""
|
|
latest_block = self.db.query(Block).order_by(Block.slot.desc()).first()
|
|
if latest_block:
|
|
return latest_block.slot
|
|
return None
|
|
|
|
def save_block(self, block_data: Dict) -> Block:
|
|
"""
|
|
Save block data to database
|
|
"""
|
|
block_create = BlockCreate(
|
|
slot=block_data["parentSlot"] + 1 if "parentSlot" in block_data else 0,
|
|
blockhash=block_data.get("blockhash", ""),
|
|
parent_blockhash=block_data.get("previousBlockhash", ""),
|
|
processed=False,
|
|
)
|
|
|
|
block = Block(**block_create.model_dump())
|
|
self.db.add(block)
|
|
self.db.commit()
|
|
self.db.refresh(block)
|
|
|
|
return block
|
|
|
|
def save_transaction(self, tx_data: Dict, block_id: int) -> Transaction:
|
|
"""
|
|
Save transaction data to database
|
|
"""
|
|
signature = tx_data.get("transaction", {}).get("signatures", [""])[0]
|
|
|
|
tx_create = TransactionCreate(
|
|
signature=signature,
|
|
block_id=block_id,
|
|
fee=tx_data.get("meta", {}).get("fee", 0),
|
|
status="success" if tx_data.get("meta", {}).get("err") is None else "error",
|
|
raw_data=json.dumps(tx_data),
|
|
)
|
|
|
|
transaction = Transaction(**tx_create.model_dump())
|
|
self.db.add(transaction)
|
|
self.db.commit()
|
|
self.db.refresh(transaction)
|
|
|
|
return transaction
|
|
|
|
def save_token_transfers(self, tx_data: Dict, transaction_id: int) -> List[TokenTransfer]:
|
|
"""
|
|
Extract and save token transfers from transaction data using instruction-level parsing
|
|
"""
|
|
from app.parsers.parser_registry import parser_registry
|
|
|
|
transfers = []
|
|
|
|
# Use the parser registry to parse all instructions in the transaction
|
|
parsed_instructions = parser_registry.parse_transaction_instructions(tx_data)
|
|
|
|
# Extract token transfers from parsed instructions using each parser
|
|
for instruction in parsed_instructions:
|
|
program_id = instruction.get("program_id", "")
|
|
parser = parser_registry.get_parser_for_program(program_id)
|
|
|
|
if not parser:
|
|
continue
|
|
|
|
# Extract transfer info if the parser is a TransferParser
|
|
if hasattr(parser, "extract_transfer_info"):
|
|
try:
|
|
token_address, from_address, to_address, amount = parser.extract_transfer_info(instruction)
|
|
|
|
if token_address and from_address and to_address and amount is not None and amount > 0:
|
|
# Create token transfer record
|
|
transfer_create = TokenTransferCreate(
|
|
transaction_id=transaction_id,
|
|
token_address=token_address,
|
|
from_address=from_address,
|
|
to_address=to_address,
|
|
amount=amount,
|
|
program_id=program_id,
|
|
)
|
|
|
|
transfer = TokenTransfer(**transfer_create.model_dump())
|
|
self.db.add(transfer)
|
|
transfers.append(transfer)
|
|
except Exception as e:
|
|
logger.error(f"Error extracting transfer info: {str(e)}")
|
|
|
|
# Fallback to pre/post balance comparison if no transfers found with instruction parsing
|
|
if not transfers and "meta" in tx_data and "postTokenBalances" in tx_data["meta"] and "preTokenBalances" in tx_data["meta"]:
|
|
logger.info("No transfers found with instruction parsing, falling back to balance comparison")
|
|
post_balances = tx_data["meta"]["postTokenBalances"]
|
|
pre_balances = tx_data["meta"]["preTokenBalances"]
|
|
|
|
# Compare pre and post balances to identify transfers
|
|
for post in post_balances:
|
|
mint = post.get("mint", "")
|
|
owner = post.get("owner", "")
|
|
|
|
# Find matching pre-balance
|
|
pre = next((p for p in pre_balances if p.get("mint") == mint and p.get("owner") == owner), None)
|
|
|
|
if pre:
|
|
# Calculate amount change
|
|
pre_amount = float(pre.get("uiTokenAmount", {}).get("uiAmount", 0) or 0)
|
|
post_amount = float(post.get("uiTokenAmount", {}).get("uiAmount", 0) or 0)
|
|
amount_change = post_amount - pre_amount
|
|
|
|
if amount_change != 0:
|
|
# Create token transfer record
|
|
transfer_create = TokenTransferCreate(
|
|
transaction_id=transaction_id,
|
|
token_address=mint,
|
|
from_address="" if amount_change > 0 else owner,
|
|
to_address=owner if amount_change > 0 else "",
|
|
amount=abs(amount_change),
|
|
program_id="", # We don't know the program from balance comparison
|
|
)
|
|
|
|
transfer = TokenTransfer(**transfer_create.model_dump())
|
|
self.db.add(transfer)
|
|
transfers.append(transfer)
|
|
|
|
if transfers:
|
|
self.db.commit()
|
|
for transfer in transfers:
|
|
self.db.refresh(transfer)
|
|
|
|
return transfers
|
|
|
|
def save_arbitrage_event(self, arbitrage_data: Dict, transaction_id: int) -> ArbitrageEvent:
|
|
"""
|
|
Save detected arbitrage event to database
|
|
"""
|
|
event_create = ArbitrageEventCreate(
|
|
transaction_id=transaction_id,
|
|
profit_token_address=arbitrage_data["profit_token_address"],
|
|
profit_amount=arbitrage_data["profit_amount"],
|
|
profit_usd=arbitrage_data.get("profit_usd"),
|
|
path=arbitrage_data["path"],
|
|
confidence_score=arbitrage_data["confidence_score"],
|
|
)
|
|
|
|
event = ArbitrageEvent(**event_create.model_dump())
|
|
self.db.add(event)
|
|
self.db.commit()
|
|
self.db.refresh(event)
|
|
|
|
return event
|
|
|
|
def save_instructions(self, tx_data: Dict, transaction_id: int) -> List[Instruction]:
|
|
"""
|
|
Extract and save instruction data from a transaction
|
|
"""
|
|
from app.parsers.parser_registry import parser_registry
|
|
|
|
instructions = []
|
|
|
|
# Extract message and accounts from transaction
|
|
if not tx_data or "transaction" not in tx_data or "message" not in tx_data["transaction"]:
|
|
return instructions
|
|
|
|
tx_message = tx_data["transaction"]["message"]
|
|
accounts = tx_message.get("accountKeys", [])
|
|
|
|
# Process each instruction
|
|
for idx, instruction in enumerate(tx_message.get("instructions", [])):
|
|
program_id = instruction.get("programId", "")
|
|
parser = parser_registry.get_parser_for_program(program_id)
|
|
|
|
# Parse instruction data
|
|
instruction_data = instruction.get("data", "")
|
|
accounts_list = [accounts[idx] for idx in instruction.get("accounts", []) if idx < len(accounts)]
|
|
|
|
# Create instruction record
|
|
instruction_type = "unknown"
|
|
parsed_data = None
|
|
|
|
if parser:
|
|
try:
|
|
# Decode instruction data from base64
|
|
try:
|
|
import base64
|
|
raw_data = base64.b64decode(instruction_data)
|
|
except Exception:
|
|
raw_data = b""
|
|
|
|
# Parse instruction with appropriate parser
|
|
parsed_instruction = parser.parse_instruction(instruction, accounts, raw_data)
|
|
instruction_type = parsed_instruction.get("type", "unknown")
|
|
parsed_data = parsed_instruction
|
|
except Exception as e:
|
|
logger.error(f"Error parsing instruction: {str(e)}")
|
|
|
|
# Create instruction record
|
|
from app.schemas.schemas import InstructionCreate
|
|
instruction_create = InstructionCreate(
|
|
transaction_id=transaction_id,
|
|
program_id=program_id,
|
|
instruction_type=instruction_type,
|
|
instruction_index=idx,
|
|
accounts=json.dumps(accounts_list),
|
|
data=instruction_data,
|
|
parsed_data=parsed_data,
|
|
)
|
|
|
|
instruction_obj = Instruction(**instruction_create.model_dump())
|
|
self.db.add(instruction_obj)
|
|
instructions.append(instruction_obj)
|
|
|
|
if instructions:
|
|
self.db.commit()
|
|
for instruction in instructions:
|
|
self.db.refresh(instruction)
|
|
|
|
return instructions
|
|
|
|
def save_swaps(self, tx_data: Dict, transaction_id: int, instructions: List[Instruction]) -> List[Swap]:
|
|
"""
|
|
Extract and save swap data from a transaction
|
|
"""
|
|
from app.parsers.parser_registry import parser_registry
|
|
|
|
swaps = []
|
|
|
|
# Get parsed instructions
|
|
parsed_instructions = parser_registry.parse_transaction_instructions(tx_data)
|
|
|
|
# Map instruction indices to IDs
|
|
instruction_id_map = {inst.instruction_index: inst.id for inst in instructions if inst.instruction_index is not None}
|
|
|
|
# Extract swap information from parsed instructions
|
|
for idx, parsed_instruction in enumerate(parsed_instructions):
|
|
program_id = parsed_instruction.get("program_id", "")
|
|
parser = parser_registry.get_parser_for_program(program_id)
|
|
|
|
if not parser or not hasattr(parser, "extract_swap_info"):
|
|
continue
|
|
|
|
try:
|
|
token_in, token_out, amount_in, amount_out = parser.extract_swap_info(parsed_instruction)
|
|
|
|
if token_in and token_out and amount_in is not None and amount_out is not None:
|
|
instruction_id = instruction_id_map.get(idx)
|
|
if not instruction_id:
|
|
continue
|
|
|
|
# Create swap record
|
|
from app.schemas.schemas import SwapCreate
|
|
swap_create = SwapCreate(
|
|
transaction_id=transaction_id,
|
|
instruction_id=instruction_id,
|
|
program_id=program_id,
|
|
token_in_address=token_in,
|
|
token_out_address=token_out,
|
|
amount_in=amount_in,
|
|
amount_out=amount_out,
|
|
user_account=parsed_instruction.get("user_wallet", ""),
|
|
)
|
|
|
|
swap = Swap(**swap_create.model_dump())
|
|
self.db.add(swap)
|
|
swaps.append(swap)
|
|
except Exception as e:
|
|
logger.error(f"Error extracting swap info: {str(e)}")
|
|
|
|
if swaps:
|
|
self.db.commit()
|
|
for swap in swaps:
|
|
self.db.refresh(swap)
|
|
|
|
return swaps
|
|
|
|
def process_transaction(self, tx_data: Dict, block_id: int) -> Tuple[Transaction, Optional[ArbitrageEvent]]:
|
|
"""
|
|
Process a single transaction, looking for arbitrage
|
|
"""
|
|
# Save transaction record
|
|
transaction = self.save_transaction(tx_data, block_id)
|
|
|
|
# Extract and save instructions
|
|
instructions = self.save_instructions(tx_data, transaction.id)
|
|
|
|
# Extract and save swaps
|
|
swaps = self.save_swaps(tx_data, transaction.id, instructions)
|
|
|
|
# Extract and save token transfers
|
|
self.save_token_transfers(tx_data, transaction.id)
|
|
|
|
# Analyze for arbitrage
|
|
is_arbitrage, arbitrage_data = self.solana_client.analyze_transaction_for_arbitrage(tx_data)
|
|
|
|
arbitrage_event = None
|
|
if is_arbitrage and arbitrage_data:
|
|
# Add swap sequence if available
|
|
if swaps:
|
|
swap_ids = [swap.id for swap in swaps]
|
|
arbitrage_data["swap_sequence"] = json.dumps(swap_ids)
|
|
|
|
arbitrage_event = self.save_arbitrage_event(arbitrage_data, transaction.id)
|
|
logger.info(f"Detected arbitrage in transaction {transaction.signature}")
|
|
|
|
return transaction, arbitrage_event
|
|
|
|
def process_block(self, block_data: Dict) -> Tuple[Block, List[ArbitrageEvent]]:
|
|
"""
|
|
Process a single block, looking for arbitrage transactions
|
|
"""
|
|
# Save block record
|
|
block = self.save_block(block_data)
|
|
|
|
arbitrage_events = []
|
|
|
|
# Process each transaction in the block
|
|
if "transactions" in block_data:
|
|
for tx_data in block_data["transactions"]:
|
|
_, arbitrage_event = self.process_transaction(tx_data, block.slot)
|
|
if arbitrage_event:
|
|
arbitrage_events.append(arbitrage_event)
|
|
|
|
# Mark block as processed
|
|
block.processed = True
|
|
self.db.commit()
|
|
self.db.refresh(block)
|
|
|
|
return block, arbitrage_events
|
|
|
|
def scan_blocks(self, num_blocks: int = None, start_slot: int = None) -> List[ArbitrageEvent]:
|
|
"""
|
|
Scan the specified number of latest blocks for arbitrage
|
|
"""
|
|
if self._scan_in_progress:
|
|
logger.warning("Scan already in progress, ignoring request")
|
|
return []
|
|
|
|
self._scan_in_progress = True
|
|
all_events = []
|
|
|
|
try:
|
|
num_blocks = num_blocks or settings.SOLANA_BLOCKS_TO_SCAN
|
|
|
|
# Get latest block if start_slot not provided
|
|
if start_slot is None:
|
|
latest_block = self.solana_client.get_latest_block()
|
|
# Get block at that slot
|
|
block_data = self.solana_client.get_block(latest_block["lastValidBlockHeight"])
|
|
start_slot = block_data["parentSlot"]
|
|
|
|
# Get list of blocks to scan
|
|
end_slot = start_slot - num_blocks
|
|
if end_slot < 0:
|
|
end_slot = 0
|
|
|
|
blocks_to_scan = self.solana_client.get_blocks(end_slot, start_slot)
|
|
|
|
# Scan each block
|
|
for slot in blocks_to_scan:
|
|
try:
|
|
block_data = self.solana_client.get_block(slot)
|
|
_, events = self.process_block(block_data)
|
|
all_events.extend(events)
|
|
except Exception as e:
|
|
logger.error(f"Error processing block {slot}: {str(e)}")
|
|
continue
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error scanning blocks: {str(e)}")
|
|
|
|
finally:
|
|
self._scan_in_progress = False
|
|
|
|
return all_events
|
|
|
|
def get_scan_status(self) -> Dict:
|
|
"""
|
|
Get the current status of the block scanners
|
|
"""
|
|
last_block = self.db.query(Block).order_by(Block.slot.desc()).first()
|
|
last_scan_time = None
|
|
last_scanned_block = 0
|
|
|
|
if last_block:
|
|
last_scan_time = last_block.timestamp
|
|
last_scanned_block = last_block.slot
|
|
|
|
arbitrage_count = self.db.query(ArbitrageEvent).count()
|
|
|
|
return {
|
|
"last_scanned_block": last_scanned_block,
|
|
"last_scan_time": last_scan_time or datetime.utcnow(),
|
|
"arbitrage_events_count": arbitrage_count,
|
|
"scan_in_progress": self._scan_in_progress,
|
|
} |