76 lines
2.6 KiB
Python
76 lines
2.6 KiB
Python
"""Initial migration
|
|
|
|
Revision ID: 0001
|
|
Revises:
|
|
Create Date: 2023-06-25
|
|
|
|
"""
|
|
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
from sqlalchemy.dialects import sqlite
|
|
|
|
# revision identifiers, used by Alembic.
|
|
revision = "0001"
|
|
down_revision = None
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def upgrade():
|
|
# Create scrape_jobs table
|
|
op.create_table(
|
|
"scrape_jobs",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column("url", sa.String(length=2048), nullable=False),
|
|
sa.Column(
|
|
"status",
|
|
sa.Enum("pending", "in_progress", "completed", "failed", name="jobstatus"),
|
|
nullable=False,
|
|
default="pending",
|
|
),
|
|
sa.Column(
|
|
"created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()
|
|
),
|
|
sa.Column(
|
|
"updated_at", sa.DateTime(), nullable=False, server_default=sa.func.now()
|
|
),
|
|
sa.Column("started_at", sa.DateTime(), nullable=True),
|
|
sa.Column("completed_at", sa.DateTime(), nullable=True),
|
|
sa.Column("selector", sa.String(length=255), nullable=True),
|
|
sa.Column("error", sa.Text(), nullable=True),
|
|
sa.Column("result", sqlite.JSON(), nullable=True),
|
|
sa.Column("user_agent", sa.String(length=255), nullable=True),
|
|
sa.Column("timeout", sa.Integer(), nullable=True),
|
|
sa.PrimaryKeyConstraint("id"),
|
|
)
|
|
op.create_index(op.f("ix_scrape_jobs_id"), "scrape_jobs", ["id"], unique=False)
|
|
op.create_index(op.f("ix_scrape_jobs_url"), "scrape_jobs", ["url"], unique=False)
|
|
|
|
# Create scrape_results table
|
|
op.create_table(
|
|
"scrape_results",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column("job_id", sa.Integer(), nullable=False),
|
|
sa.Column(
|
|
"created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()
|
|
),
|
|
sa.Column("content_type", sa.String(length=100), nullable=True),
|
|
sa.Column("headers", sqlite.JSON(), nullable=True),
|
|
sa.Column("html_content", sa.Text(), nullable=True),
|
|
sa.Column("extracted_data", sqlite.JSON(), nullable=True),
|
|
sa.ForeignKeyConstraint(["job_id"], ["scrape_jobs.id"], ondelete="CASCADE"),
|
|
sa.PrimaryKeyConstraint("id"),
|
|
)
|
|
op.create_index(
|
|
op.f("ix_scrape_results_id"), "scrape_results", ["id"], unique=False
|
|
)
|
|
|
|
|
|
def downgrade():
|
|
op.drop_index(op.f("ix_scrape_results_id"), table_name="scrape_results")
|
|
op.drop_table("scrape_results")
|
|
op.drop_index(op.f("ix_scrape_jobs_url"), table_name="scrape_jobs")
|
|
op.drop_index(op.f("ix_scrape_jobs_id"), table_name="scrape_jobs")
|
|
op.drop_table("scrape_jobs")
|