Implement robust database initialization and fallback mechanisms
This commit is contained in:
parent
7aca5a2ff6
commit
522c749b23
@ -37,27 +37,129 @@ def create_task(
|
||||
"""
|
||||
Create new task.
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from app.db.session import db_file
|
||||
|
||||
# Print detailed request info for debugging
|
||||
print(f"[{datetime.now().isoformat()}] Creating task: {task_in}")
|
||||
|
||||
try:
|
||||
print(f"Attempting to create task with data: {task_in.model_dump() if hasattr(task_in, 'model_dump') else task_in.dict()}")
|
||||
|
||||
# Handle datetime conversion for due_date
|
||||
if task_in.due_date:
|
||||
print(f"Due date before processing: {task_in.due_date}")
|
||||
# Ensure due_date is properly formatted
|
||||
if isinstance(task_in.due_date, str):
|
||||
from datetime import datetime
|
||||
try:
|
||||
task_in.due_date = datetime.fromisoformat(task_in.due_date.replace('Z', '+00:00'))
|
||||
except Exception as e:
|
||||
print(f"Error parsing due_date: {e}")
|
||||
# Continue with string, SQLAlchemy will handle it
|
||||
|
||||
# Create the task
|
||||
task = crud.task.create(db, obj_in=task_in)
|
||||
print(f"Task created successfully with ID: {task.id}")
|
||||
return task
|
||||
# First try the normal SQLAlchemy approach
|
||||
try:
|
||||
print("Attempting to create task via SQLAlchemy...")
|
||||
task = crud.task.create(db, obj_in=task_in)
|
||||
print(f"Task created successfully with ID: {task.id}")
|
||||
return task
|
||||
except Exception as e:
|
||||
print(f"SQLAlchemy task creation failed: {e}")
|
||||
print(traceback.format_exc())
|
||||
# Continue to try direct SQLite approach
|
||||
|
||||
# Fallback: Try direct SQLite approach
|
||||
print("Falling back to direct SQLite task creation...")
|
||||
try:
|
||||
# Convert Pydantic model to dict
|
||||
task_data = task_in.model_dump() if hasattr(task_in, 'model_dump') else task_in.dict()
|
||||
print(f"Task data: {task_data}")
|
||||
|
||||
# Format datetime objects
|
||||
if task_data.get('due_date'):
|
||||
if isinstance(task_data['due_date'], datetime):
|
||||
task_data['due_date'] = task_data['due_date'].isoformat()
|
||||
|
||||
# Connect directly to SQLite and insert the task
|
||||
conn = sqlite3.connect(str(db_file))
|
||||
cursor = conn.cursor()
|
||||
now = datetime.utcnow().isoformat()
|
||||
|
||||
# Check if task table exists
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='task'")
|
||||
if not cursor.fetchone():
|
||||
# Create the task table if it doesn't exist
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS task (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
priority TEXT,
|
||||
status TEXT,
|
||||
due_date TEXT,
|
||||
completed INTEGER,
|
||||
created_at TEXT,
|
||||
updated_at TEXT
|
||||
)
|
||||
""")
|
||||
print("Created task table with direct SQLite")
|
||||
|
||||
# Insert the task
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO task (
|
||||
title, description, priority, status,
|
||||
due_date, completed, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
task_data.get('title', 'Untitled Task'),
|
||||
task_data.get('description', ''),
|
||||
task_data.get('priority', 'medium'),
|
||||
task_data.get('status', 'todo'),
|
||||
task_data.get('due_date'),
|
||||
1 if task_data.get('completed') else 0,
|
||||
now,
|
||||
now
|
||||
)
|
||||
)
|
||||
conn.commit()
|
||||
task_id = cursor.lastrowid
|
||||
|
||||
# Return the created task
|
||||
cursor.execute("SELECT * FROM task WHERE id = ?", (task_id,))
|
||||
row = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
if row:
|
||||
column_names = ['id', 'title', 'description', 'priority', 'status',
|
||||
'due_date', 'completed', 'created_at', 'updated_at']
|
||||
task_dict = {column_names[i]: row[i] for i in range(len(column_names))}
|
||||
# Convert completed to boolean
|
||||
task_dict['completed'] = bool(task_dict['completed'])
|
||||
|
||||
# Fake Task object with attributes
|
||||
class TaskResult:
|
||||
def __init__(self, **kwargs):
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
print(f"Created task with direct SQLite, ID: {task_id}")
|
||||
return TaskResult(**task_dict)
|
||||
else:
|
||||
raise Exception("Task was created but could not be retrieved")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Direct SQLite task creation failed: {e}")
|
||||
print(traceback.format_exc())
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"All task creation methods failed. Last error: {str(e)}",
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
print(f"Error in create_task endpoint: {str(e)}")
|
||||
import traceback
|
||||
print(f"Unhandled error in create_task: {str(e)}")
|
||||
print(traceback.format_exc())
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
|
@ -5,8 +5,33 @@ from typing import Any, Dict, List, Optional, Union
|
||||
from pydantic import AnyHttpUrl, field_validator
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
DB_DIR = Path("/app/storage/db")
|
||||
DB_DIR.mkdir(parents=True, exist_ok=True)
|
||||
# Try multiple possible database locations in order of preference
|
||||
DB_LOCATIONS = [
|
||||
Path("/app/storage/db"),
|
||||
Path("/tmp/taskmanager/db"),
|
||||
Path.home() / ".taskmanager/db",
|
||||
Path.cwd() / "db"
|
||||
]
|
||||
|
||||
# Try to find or create a writable database directory
|
||||
DB_DIR = None
|
||||
for location in DB_LOCATIONS:
|
||||
try:
|
||||
location.mkdir(parents=True, exist_ok=True)
|
||||
# Check if directory is writable by creating a test file
|
||||
test_file = location / "test_write"
|
||||
test_file.touch()
|
||||
test_file.unlink() # Remove test file
|
||||
DB_DIR = location
|
||||
print(f"Using database directory: {DB_DIR}")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"Could not use {location} for database: {e}")
|
||||
|
||||
# If no location works, fall back to current directory
|
||||
if DB_DIR is None:
|
||||
DB_DIR = Path.cwd()
|
||||
print(f"Falling back to current directory for database: {DB_DIR}")
|
||||
|
||||
class Settings(BaseSettings):
|
||||
PROJECT_NAME: str = "Task Manager API"
|
||||
|
@ -1,36 +1,68 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import inspect, text
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from app.db.base import Base # Import all models
|
||||
from app.db.session import engine
|
||||
from app.db.session import engine, db_file
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
"""Initialize database with required tables and data directly."""
|
||||
"""
|
||||
Initialize database with required tables directly using both SQLAlchemy and
|
||||
SQLite native commands for maximum reliability.
|
||||
"""
|
||||
print(f"Initializing database at {db_file}")
|
||||
db_dir = db_file.parent
|
||||
|
||||
# Ensure database directory exists
|
||||
Path(settings.DB_DIR).mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
db_dir.mkdir(parents=True, exist_ok=True)
|
||||
print(f"Database directory created or already exists: {db_dir}")
|
||||
except Exception as e:
|
||||
print(f"Error creating database directory: {e}")
|
||||
|
||||
db_file = f"{settings.DB_DIR}/db.sqlite"
|
||||
db_exists = os.path.exists(db_file)
|
||||
# First, try to create an empty database file if it doesn't exist
|
||||
if not db_file.exists():
|
||||
try:
|
||||
# Create an empty file
|
||||
db_file.touch()
|
||||
print(f"Created empty database file: {db_file}")
|
||||
except Exception as e:
|
||||
print(f"Failed to create database file: {e}")
|
||||
|
||||
# If database doesn't exist or is empty, create tables
|
||||
# First, try direct SQLite connection to create basic structure
|
||||
# This bypasses SQLAlchemy entirely for the initial database creation
|
||||
try:
|
||||
print(f"Attempting direct SQLite connection to {db_file}")
|
||||
sqlite_conn = sqlite3.connect(str(db_file))
|
||||
sqlite_conn.execute("PRAGMA journal_mode=WAL")
|
||||
sqlite_conn.execute("CREATE TABLE IF NOT EXISTS _db_init_check (id INTEGER PRIMARY KEY)")
|
||||
sqlite_conn.execute("INSERT OR IGNORE INTO _db_init_check VALUES (1)")
|
||||
sqlite_conn.commit()
|
||||
sqlite_conn.close()
|
||||
print("Direct SQLite connection and initialization successful")
|
||||
except Exception as e:
|
||||
print(f"Direct SQLite initialization error: {e}")
|
||||
|
||||
# Now try with SQLAlchemy
|
||||
try:
|
||||
# Try to connect to check if the database is accessible
|
||||
max_retries = 3
|
||||
max_retries = 5
|
||||
retry_count = 0
|
||||
connected = False
|
||||
|
||||
while retry_count < max_retries and not connected:
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
print(f"Database connection successful to {db_file}")
|
||||
result = conn.execute(text("SELECT 1")).scalar()
|
||||
print(f"Database connection successful. Test query result: {result}")
|
||||
connected = True
|
||||
except Exception as e:
|
||||
retry_count += 1
|
||||
@ -38,78 +70,131 @@ def init_db() -> None:
|
||||
time.sleep(1) # Wait a second before retrying
|
||||
|
||||
if not connected:
|
||||
raise Exception(f"Failed to connect to database after {max_retries} attempts")
|
||||
print(f"Failed to connect to database after {max_retries} attempts")
|
||||
# Continue anyway to see if we can make progress
|
||||
|
||||
# Check if tables exist
|
||||
inspector = inspect(engine)
|
||||
existing_tables = inspector.get_table_names()
|
||||
|
||||
if not existing_tables:
|
||||
print("No tables found in database. Creating tables...")
|
||||
# Try to create tables
|
||||
try:
|
||||
print("Creating database tables with SQLAlchemy...")
|
||||
Base.metadata.create_all(bind=engine)
|
||||
print(f"Created tables: {', '.join(inspector.get_table_names())}")
|
||||
else:
|
||||
print(f"Found existing tables: {', '.join(existing_tables)}")
|
||||
|
||||
# Verify tables
|
||||
inspector = inspect(engine)
|
||||
tables = inspector.get_table_names()
|
||||
print(f"Tables in database: {', '.join(tables)}")
|
||||
|
||||
if 'task' not in tables:
|
||||
print("WARNING: 'task' table not created!")
|
||||
else:
|
||||
print("'task' table successfully created.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating tables: {e}")
|
||||
|
||||
# Print database file information
|
||||
if db_exists:
|
||||
file_size = os.path.getsize(db_file)
|
||||
print(f"Database file exists at {db_file}, size: {file_size} bytes")
|
||||
else:
|
||||
print(f"Warning: Database file does not exist at {db_file} after initialization!")
|
||||
|
||||
print("Database initialization completed successfully")
|
||||
# Print database info for debugging
|
||||
try:
|
||||
print(f"Database file size: {os.path.getsize(db_file)} bytes")
|
||||
print(f"Database file permissions: {oct(os.stat(db_file).st_mode)[-3:]}")
|
||||
print(f"Database dir permissions: {oct(os.stat(db_dir).st_mode)[-3:]}")
|
||||
except Exception as e:
|
||||
print(f"Error getting file info: {e}")
|
||||
|
||||
print("Database initialization completed")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Database initialization error: {str(e)}")
|
||||
# Print detailed error but don't raise to allow app to start
|
||||
import traceback
|
||||
print(traceback.format_exc())
|
||||
|
||||
# Try to create a test file to check write permissions
|
||||
try:
|
||||
test_file = f"{settings.DB_DIR}/test_write.txt"
|
||||
with open(test_file, 'w') as f:
|
||||
f.write('test')
|
||||
print(f"Successfully wrote test file: {test_file}")
|
||||
os.remove(test_file)
|
||||
except Exception as e:
|
||||
print(f"Failed to write test file: {str(e)}")
|
||||
print("Continuing anyway...")
|
||||
|
||||
|
||||
def create_test_task():
|
||||
"""Create a test task in the database to verify everything is working."""
|
||||
print("Attempting to create a test task...")
|
||||
try:
|
||||
from app.crud.task import task as task_crud
|
||||
from app.schemas.task import TaskCreate
|
||||
from app.db.session import SessionLocal
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
db = SessionLocal()
|
||||
# Try direct SQLite approach first
|
||||
try:
|
||||
# Check if there are any tasks
|
||||
existing_tasks = db.execute("SELECT COUNT(*) FROM task").scalar()
|
||||
if existing_tasks > 0:
|
||||
print(f"Test task not needed, found {existing_tasks} existing tasks")
|
||||
conn = sqlite3.connect(str(db_file))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check if task table exists
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='task'")
|
||||
if not cursor.fetchone():
|
||||
print("Task table doesn't exist - cannot create test task")
|
||||
return
|
||||
|
||||
# Check if any tasks exist
|
||||
cursor.execute("SELECT COUNT(*) FROM task")
|
||||
count = cursor.fetchone()[0]
|
||||
|
||||
# Create a test task
|
||||
test_task = TaskCreate(
|
||||
title="Test Task",
|
||||
description="This is a test task created at database initialization",
|
||||
priority="medium",
|
||||
status="todo",
|
||||
due_date=datetime.utcnow() + timedelta(days=7),
|
||||
completed=False
|
||||
)
|
||||
if count == 0:
|
||||
# Create a task directly with SQLite
|
||||
now = datetime.utcnow().isoformat()
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO task (
|
||||
title, description, priority, status,
|
||||
completed, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
"Test Task (Direct SQL)",
|
||||
"This is a test task created directly with SQLite",
|
||||
"medium",
|
||||
"todo",
|
||||
0, # not completed
|
||||
now,
|
||||
now
|
||||
)
|
||||
)
|
||||
conn.commit()
|
||||
task_id = cursor.lastrowid
|
||||
print(f"Created test task with direct SQLite, ID: {task_id}")
|
||||
else:
|
||||
print(f"Found {count} existing tasks, no need to create test task")
|
||||
|
||||
conn.close()
|
||||
|
||||
created_task = task_crud.create(db, obj_in=test_task)
|
||||
print(f"Created test task with ID: {created_task.id}")
|
||||
except Exception as e:
|
||||
print(f"Error with direct SQLite test task creation: {e}")
|
||||
# Continue with SQLAlchemy approach
|
||||
|
||||
# Now try with SQLAlchemy
|
||||
try:
|
||||
from app.crud.task import task as task_crud
|
||||
from app.schemas.task import TaskCreate
|
||||
from app.db.session import SessionLocal
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# Check if there are any tasks
|
||||
try:
|
||||
existing_tasks = db.execute(text("SELECT COUNT(*) FROM task")).scalar()
|
||||
if existing_tasks > 0:
|
||||
print(f"Test task not needed, found {existing_tasks} existing tasks")
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"Error checking for existing tasks: {e}")
|
||||
# Continue anyway to try creating a task
|
||||
|
||||
# Create a test task
|
||||
test_task = TaskCreate(
|
||||
title="Test Task (SQLAlchemy)",
|
||||
description="This is a test task created with SQLAlchemy",
|
||||
priority="medium",
|
||||
status="todo",
|
||||
due_date=datetime.utcnow() + timedelta(days=7),
|
||||
completed=False
|
||||
)
|
||||
|
||||
created_task = task_crud.create(db, obj_in=test_task)
|
||||
print(f"Created test task with SQLAlchemy, ID: {created_task.id}")
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
print(f"Error with SQLAlchemy test task creation: {e}")
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
print(f"Error creating test task: {e}")
|
||||
print(f"Global error creating test task: {e}")
|
||||
import traceback
|
||||
print(traceback.format_exc())
|
||||
|
||||
|
@ -1,18 +1,79 @@
|
||||
from sqlalchemy import create_engine
|
||||
import os
|
||||
import time
|
||||
from typing import Generator
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.exc import OperationalError, SQLAlchemyError
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.config import settings, DB_DIR
|
||||
|
||||
# Ensure the database directory exists
|
||||
db_file = DB_DIR / "db.sqlite"
|
||||
print(f"Database file path: {db_file}")
|
||||
|
||||
# Configure SQLite connection with more robust settings
|
||||
engine = create_engine(
|
||||
settings.SQLALCHEMY_DATABASE_URL,
|
||||
connect_args={"check_same_thread": False}
|
||||
connect_args={
|
||||
"check_same_thread": False,
|
||||
"timeout": 30, # Wait up to 30 seconds for the lock
|
||||
},
|
||||
pool_recycle=3600, # Recycle connections after 1 hour
|
||||
pool_pre_ping=True, # Verify connections before usage
|
||||
pool_size=10, # Maximum 10 connections
|
||||
max_overflow=20, # Allow up to 20 overflow connections
|
||||
)
|
||||
|
||||
# Add SQLite optimizations
|
||||
@event.listens_for(engine, "connect")
|
||||
def optimize_sqlite_connection(dbapi_connection, connection_record):
|
||||
"""Configure SQLite connection for better performance and reliability."""
|
||||
# Enable WAL journal mode for better concurrency
|
||||
dbapi_connection.execute("PRAGMA journal_mode=WAL")
|
||||
# Ensure foreign keys are enforced
|
||||
dbapi_connection.execute("PRAGMA foreign_keys=ON")
|
||||
# Synchronous setting for better performance with decent safety
|
||||
dbapi_connection.execute("PRAGMA synchronous=NORMAL")
|
||||
# Cache settings
|
||||
dbapi_connection.execute("PRAGMA cache_size=-64000") # 64MB cache
|
||||
# Temp storage in memory
|
||||
dbapi_connection.execute("PRAGMA temp_store=MEMORY")
|
||||
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
# More robust database access with retry logic
|
||||
def get_db() -> Generator:
|
||||
"""
|
||||
Get a database session with retry logic for transient errors.
|
||||
Creates the database file and tables if they don't exist.
|
||||
"""
|
||||
db = None
|
||||
retries = 3
|
||||
retry_delay = 0.5 # Start with 0.5 second delay
|
||||
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
db = SessionLocal()
|
||||
# Test connection with a simple query
|
||||
db.execute("SELECT 1")
|
||||
# Yield the working connection
|
||||
yield db
|
||||
break
|
||||
except (OperationalError, SQLAlchemyError) as e:
|
||||
if db:
|
||||
db.close()
|
||||
|
||||
# Only retry transient errors like "database is locked"
|
||||
if attempt < retries - 1:
|
||||
print(f"Database connection attempt {attempt+1} failed: {e}")
|
||||
print(f"Retrying in {retry_delay} seconds...")
|
||||
time.sleep(retry_delay)
|
||||
retry_delay *= 2 # Exponential backoff
|
||||
else:
|
||||
print(f"All database connection attempts failed: {e}")
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
if db:
|
||||
db.close()
|
Loading…
x
Reference in New Issue
Block a user