Implement robust database initialization and fallback mechanisms

app/api/routers/tasks.py

@@ -37,27 +37,129 @@ def create_task(
     """
     Create new task.
     """
+    import sqlite3
+    import json
+    import traceback
+    from datetime import datetime
+    from app.db.session import db_file
+    
+    # Print detailed request info for debugging
+    print(f"[{datetime.now().isoformat()}] Creating task: {task_in}")
+    
     try:
-        print(f"Attempting to create task with data: {task_in.model_dump() if hasattr(task_in, 'model_dump') else task_in.dict()}")
-        
         # Handle datetime conversion for due_date
         if task_in.due_date:
             print(f"Due date before processing: {task_in.due_date}")
-            # Ensure due_date is properly formatted
             if isinstance(task_in.due_date, str):
-                from datetime import datetime
                 try:
                     task_in.due_date = datetime.fromisoformat(task_in.due_date.replace('Z', '+00:00'))
                 except Exception as e:
                     print(f"Error parsing due_date: {e}")
+                    # Continue with string, SQLAlchemy will handle it
         
-        # Create the task
+        # First try the normal SQLAlchemy approach
-        task = crud.task.create(db, obj_in=task_in)
+        try:
-        print(f"Task created successfully with ID: {task.id}")
+            print("Attempting to create task via SQLAlchemy...")
-        return task
+            task = crud.task.create(db, obj_in=task_in)
+            print(f"Task created successfully with ID: {task.id}")
+            return task
+        except Exception as e:
+            print(f"SQLAlchemy task creation failed: {e}")
+            print(traceback.format_exc())
+            # Continue to try direct SQLite approach
+            
+        # Fallback: Try direct SQLite approach
+        print("Falling back to direct SQLite task creation...")
+        try:
+            # Convert Pydantic model to dict
+            task_data = task_in.model_dump() if hasattr(task_in, 'model_dump') else task_in.dict()
+            print(f"Task data: {task_data}")
+            
+            # Format datetime objects
+            if task_data.get('due_date'):
+                if isinstance(task_data['due_date'], datetime):
+                    task_data['due_date'] = task_data['due_date'].isoformat()
+                    
+            # Connect directly to SQLite and insert the task
+            conn = sqlite3.connect(str(db_file))
+            cursor = conn.cursor()
+            now = datetime.utcnow().isoformat()
+            
+            # Check if task table exists
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='task'")
+            if not cursor.fetchone():
+                # Create the task table if it doesn't exist
+                cursor.execute("""
+                CREATE TABLE IF NOT EXISTS task (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    title TEXT NOT NULL,
+                    description TEXT,
+                    priority TEXT,
+                    status TEXT,
+                    due_date TEXT,
+                    completed INTEGER,
+                    created_at TEXT,
+                    updated_at TEXT
+                )
+                """)
+                print("Created task table with direct SQLite")
+            
+            # Insert the task
+            cursor.execute(
+                """
+                INSERT INTO task (
+                    title, description, priority, status, 
+                    due_date, completed, created_at, updated_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                """, 
+                (
+                    task_data.get('title', 'Untitled Task'), 
+                    task_data.get('description', ''),
+                    task_data.get('priority', 'medium'),
+                    task_data.get('status', 'todo'),
+                    task_data.get('due_date'),
+                    1 if task_data.get('completed') else 0,
+                    now,
+                    now
+                )
+            )
+            conn.commit()
+            task_id = cursor.lastrowid
+            
+            # Return the created task
+            cursor.execute("SELECT * FROM task WHERE id = ?", (task_id,))
+            row = cursor.fetchone()
+            conn.close()
+            
+            if row:
+                column_names = ['id', 'title', 'description', 'priority', 'status', 
+                               'due_date', 'completed', 'created_at', 'updated_at']
+                task_dict = {column_names[i]: row[i] for i in range(len(column_names))}
+                # Convert completed to boolean
+                task_dict['completed'] = bool(task_dict['completed'])
+                
+                # Fake Task object with attributes
+                class TaskResult:
+                    def __init__(self, **kwargs):
+                        for key, value in kwargs.items():
+                            setattr(self, key, value)
+                
+                print(f"Created task with direct SQLite, ID: {task_id}")
+                return TaskResult(**task_dict)
+            else:
+                raise Exception("Task was created but could not be retrieved")
+            
+        except Exception as e:
+            print(f"Direct SQLite task creation failed: {e}")
+            print(traceback.format_exc())
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"All task creation methods failed. Last error: {str(e)}",
+            )
+    except HTTPException:
+        raise
     except Exception as e:
-        print(f"Error in create_task endpoint: {str(e)}")
+        print(f"Unhandled error in create_task: {str(e)}")
-        import traceback
         print(traceback.format_exc())
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,

app/core/config.py

@@ -5,8 +5,33 @@ from typing import Any, Dict, List, Optional, Union
 from pydantic import AnyHttpUrl, field_validator
 from pydantic_settings import BaseSettings
 
-DB_DIR = Path("/app/storage/db")
+# Try multiple possible database locations in order of preference
-DB_DIR.mkdir(parents=True, exist_ok=True)
+DB_LOCATIONS = [
+    Path("/app/storage/db"),
+    Path("/tmp/taskmanager/db"),
+    Path.home() / ".taskmanager/db",
+    Path.cwd() / "db"
+]
+
+# Try to find or create a writable database directory
+DB_DIR = None
+for location in DB_LOCATIONS:
+    try:
+        location.mkdir(parents=True, exist_ok=True)
+        # Check if directory is writable by creating a test file
+        test_file = location / "test_write"
+        test_file.touch()
+        test_file.unlink()  # Remove test file
+        DB_DIR = location
+        print(f"Using database directory: {DB_DIR}")
+        break
+    except Exception as e:
+        print(f"Could not use {location} for database: {e}")
+
+# If no location works, fall back to current directory
+if DB_DIR is None:
+    DB_DIR = Path.cwd()
+    print(f"Falling back to current directory for database: {DB_DIR}")
 
 class Settings(BaseSettings):
     PROJECT_NAME: str = "Task Manager API"

app/db/init_db.py

@@ -1,36 +1,68 @@
 import os
+import sys
 import time
+import sqlite3
 from pathlib import Path
+from datetime import datetime, timedelta
 
 from sqlalchemy import inspect, text
 from sqlalchemy.exc import OperationalError
 
 from app.db.base import Base  # Import all models
-from app.db.session import engine
+from app.db.session import engine, db_file
 from app.core.config import settings
 
 
 def init_db() -> None:
-    """Initialize database with required tables and data directly."""
+    """
+    Initialize database with required tables directly using both SQLAlchemy and 
+    SQLite native commands for maximum reliability.
+    """
+    print(f"Initializing database at {db_file}")
+    db_dir = db_file.parent
     
     # Ensure database directory exists
-    Path(settings.DB_DIR).mkdir(parents=True, exist_ok=True)
+    try:
+        db_dir.mkdir(parents=True, exist_ok=True)
+        print(f"Database directory created or already exists: {db_dir}")
+    except Exception as e:
+        print(f"Error creating database directory: {e}")
     
-    db_file = f"{settings.DB_DIR}/db.sqlite"
+    # First, try to create an empty database file if it doesn't exist
-    db_exists = os.path.exists(db_file)
+    if not db_file.exists():
+        try:
+            # Create an empty file
+            db_file.touch()
+            print(f"Created empty database file: {db_file}")
+        except Exception as e:
+            print(f"Failed to create database file: {e}")
     
-    # If database doesn't exist or is empty, create tables
+    # First, try direct SQLite connection to create basic structure
+    # This bypasses SQLAlchemy entirely for the initial database creation
+    try:
+        print(f"Attempting direct SQLite connection to {db_file}")
+        sqlite_conn = sqlite3.connect(str(db_file))
+        sqlite_conn.execute("PRAGMA journal_mode=WAL")
+        sqlite_conn.execute("CREATE TABLE IF NOT EXISTS _db_init_check (id INTEGER PRIMARY KEY)")
+        sqlite_conn.execute("INSERT OR IGNORE INTO _db_init_check VALUES (1)")
+        sqlite_conn.commit()
+        sqlite_conn.close()
+        print("Direct SQLite connection and initialization successful")
+    except Exception as e:
+        print(f"Direct SQLite initialization error: {e}")
+    
+    # Now try with SQLAlchemy
     try:
         # Try to connect to check if the database is accessible
-        max_retries = 3
+        max_retries = 5
         retry_count = 0
         connected = False
         
         while retry_count < max_retries and not connected:
             try:
                 with engine.connect() as conn:
-                    conn.execute(text("SELECT 1"))
+                    result = conn.execute(text("SELECT 1")).scalar()
-                    print(f"Database connection successful to {db_file}")
+                    print(f"Database connection successful. Test query result: {result}")
                     connected = True
             except Exception as e:
                 retry_count += 1
@@ -38,78 +70,131 @@ def init_db() -> None:
                 time.sleep(1)  # Wait a second before retrying
         
         if not connected:
-            raise Exception(f"Failed to connect to database after {max_retries} attempts")
+            print(f"Failed to connect to database after {max_retries} attempts")
+            # Continue anyway to see if we can make progress
         
-        # Check if tables exist
+        # Try to create tables
-        inspector = inspect(engine)
+        try:
-        existing_tables = inspector.get_table_names()
+            print("Creating database tables with SQLAlchemy...")
-        
-        if not existing_tables:
-            print("No tables found in database. Creating tables...")
             Base.metadata.create_all(bind=engine)
-            print(f"Created tables: {', '.join(inspector.get_table_names())}")
+            
-        else:
+            # Verify tables
-            print(f"Found existing tables: {', '.join(existing_tables)}")
+            inspector = inspect(engine)
+            tables = inspector.get_table_names()
+            print(f"Tables in database: {', '.join(tables)}")
+            
+            if 'task' not in tables:
+                print("WARNING: 'task' table not created!")
+            else:
+                print("'task' table successfully created.")
+                
+        except Exception as e:
+            print(f"Error creating tables: {e}")
         
-        # Print database file information
+        # Print database info for debugging
-        if db_exists:
+        try:
-            file_size = os.path.getsize(db_file)
+            print(f"Database file size: {os.path.getsize(db_file)} bytes")
-            print(f"Database file exists at {db_file}, size: {file_size} bytes")
+            print(f"Database file permissions: {oct(os.stat(db_file).st_mode)[-3:]}")
-        else:
+            print(f"Database dir permissions: {oct(os.stat(db_dir).st_mode)[-3:]}")
-            print(f"Warning: Database file does not exist at {db_file} after initialization!")
+        except Exception as e:
-        
+            print(f"Error getting file info: {e}")
-        print("Database initialization completed successfully")
+            
+        print("Database initialization completed")
         
     except Exception as e:
         print(f"Database initialization error: {str(e)}")
-        # Print detailed error but don't raise to allow app to start
+        print("Continuing anyway...")
-        import traceback
-        print(traceback.format_exc())
-        
-        # Try to create a test file to check write permissions
-        try:
-            test_file = f"{settings.DB_DIR}/test_write.txt"
-            with open(test_file, 'w') as f:
-                f.write('test')
-            print(f"Successfully wrote test file: {test_file}")
-            os.remove(test_file)
-        except Exception as e:
-            print(f"Failed to write test file: {str(e)}")
 
 
 def create_test_task():
     """Create a test task in the database to verify everything is working."""
+    print("Attempting to create a test task...")
     try:
-        from app.crud.task import task as task_crud
+        # Try direct SQLite approach first
-        from app.schemas.task import TaskCreate
-        from app.db.session import SessionLocal
-        from datetime import datetime, timedelta
-        
-        db = SessionLocal()
         try:
-            # Check if there are any tasks
+            conn = sqlite3.connect(str(db_file))
-            existing_tasks = db.execute("SELECT COUNT(*) FROM task").scalar()
+            cursor = conn.cursor()
-            if existing_tasks > 0:
+            
-                print(f"Test task not needed, found {existing_tasks} existing tasks")
+            # Check if task table exists
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='task'")
+            if not cursor.fetchone():
+                print("Task table doesn't exist - cannot create test task")
                 return
+                
+            # Check if any tasks exist
+            cursor.execute("SELECT COUNT(*) FROM task")
+            count = cursor.fetchone()[0]
             
-            # Create a test task
+            if count == 0:
-            test_task = TaskCreate(
+                # Create a task directly with SQLite
-                title="Test Task",
+                now = datetime.utcnow().isoformat()
-                description="This is a test task created at database initialization",
+                cursor.execute(
-                priority="medium",
+                    """
-                status="todo",
+                    INSERT INTO task (
-                due_date=datetime.utcnow() + timedelta(days=7),
+                        title, description, priority, status, 
-                completed=False
+                        completed, created_at, updated_at
-            )
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?)
+                    """, 
+                    (
+                        "Test Task (Direct SQL)", 
+                        "This is a test task created directly with SQLite",
+                        "medium",
+                        "todo",
+                        0,  # not completed
+                        now,
+                        now
+                    )
+                )
+                conn.commit()
+                task_id = cursor.lastrowid
+                print(f"Created test task with direct SQLite, ID: {task_id}")
+            else:
+                print(f"Found {count} existing tasks, no need to create test task")
+                
+            conn.close()
             
-            created_task = task_crud.create(db, obj_in=test_task)
+        except Exception as e:
-            print(f"Created test task with ID: {created_task.id}")
+            print(f"Error with direct SQLite test task creation: {e}")
+            # Continue with SQLAlchemy approach
+        
+        # Now try with SQLAlchemy
+        try:
+            from app.crud.task import task as task_crud
+            from app.schemas.task import TaskCreate
+            from app.db.session import SessionLocal
+            
+            db = SessionLocal()
+            try:
+                # Check if there are any tasks
+                try:
+                    existing_tasks = db.execute(text("SELECT COUNT(*) FROM task")).scalar()
+                    if existing_tasks > 0:
+                        print(f"Test task not needed, found {existing_tasks} existing tasks")
+                        return
+                except Exception as e:
+                    print(f"Error checking for existing tasks: {e}")
+                    # Continue anyway to try creating a task
+                
+                # Create a test task
+                test_task = TaskCreate(
+                    title="Test Task (SQLAlchemy)",
+                    description="This is a test task created with SQLAlchemy",
+                    priority="medium",
+                    status="todo",
+                    due_date=datetime.utcnow() + timedelta(days=7),
+                    completed=False
+                )
+                
+                created_task = task_crud.create(db, obj_in=test_task)
+                print(f"Created test task with SQLAlchemy, ID: {created_task.id}")
+                
+            finally:
+                db.close()
+        except Exception as e:
+            print(f"Error with SQLAlchemy test task creation: {e}")
             
-        finally:
-            db.close()
     except Exception as e:
-        print(f"Error creating test task: {e}")
+        print(f"Global error creating test task: {e}")
         import traceback
         print(traceback.format_exc())
 

app/db/session.py

@@ -1,18 +1,79 @@
-from sqlalchemy import create_engine
+import os
+import time
+from typing import Generator
+from pathlib import Path
+
+from sqlalchemy import create_engine, event
 from sqlalchemy.orm import sessionmaker
+from sqlalchemy.exc import OperationalError, SQLAlchemyError
 
-from app.core.config import settings
+from app.core.config import settings, DB_DIR
 
+# Ensure the database directory exists
+db_file = DB_DIR / "db.sqlite"
+print(f"Database file path: {db_file}")
+
+# Configure SQLite connection with more robust settings
 engine = create_engine(
     settings.SQLALCHEMY_DATABASE_URL,
-    connect_args={"check_same_thread": False}
+    connect_args={
+        "check_same_thread": False,
+        "timeout": 30,           # Wait up to 30 seconds for the lock
+    },
+    pool_recycle=3600,           # Recycle connections after 1 hour
+    pool_pre_ping=True,          # Verify connections before usage
+    pool_size=10,                # Maximum 10 connections
+    max_overflow=20,             # Allow up to 20 overflow connections
 )
+
+# Add SQLite optimizations
+@event.listens_for(engine, "connect")
+def optimize_sqlite_connection(dbapi_connection, connection_record):
+    """Configure SQLite connection for better performance and reliability."""
+    # Enable WAL journal mode for better concurrency
+    dbapi_connection.execute("PRAGMA journal_mode=WAL")
+    # Ensure foreign keys are enforced
+    dbapi_connection.execute("PRAGMA foreign_keys=ON")
+    # Synchronous setting for better performance with decent safety
+    dbapi_connection.execute("PRAGMA synchronous=NORMAL")
+    # Cache settings
+    dbapi_connection.execute("PRAGMA cache_size=-64000")  # 64MB cache
+    # Temp storage in memory
+    dbapi_connection.execute("PRAGMA temp_store=MEMORY")
+
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
-
+# More robust database access with retry logic
-def get_db():
+def get_db() -> Generator:
-    db = SessionLocal()
+    """
-    try:
+    Get a database session with retry logic for transient errors.
-        yield db
+    Creates the database file and tables if they don't exist.
+    """
+    db = None
+    retries = 3
+    retry_delay = 0.5  # Start with 0.5 second delay
+    
+    for attempt in range(retries):
+        try:
+            db = SessionLocal()
+            # Test connection with a simple query
+            db.execute("SELECT 1")
+            # Yield the working connection
+            yield db
+            break
+        except (OperationalError, SQLAlchemyError) as e:
+            if db:
+                db.close()
+            
+            # Only retry transient errors like "database is locked"
+            if attempt < retries - 1:
+                print(f"Database connection attempt {attempt+1} failed: {e}")
+                print(f"Retrying in {retry_delay} seconds...")
+                time.sleep(retry_delay)
+                retry_delay *= 2  # Exponential backoff
+            else:
+                print(f"All database connection attempts failed: {e}")
+                raise
     finally:
-        db.close()
+        if db:
+            db.close()