From 09b92eceb83f96ea793695b94ffdaf0826ce8b64 Mon Sep 17 00:00:00 2001 From: Samir Boulahtit Date: Sun, 21 Sep 2025 16:03:44 +0200 Subject: [PATCH] Alembic configuration --- Makefile | 202 ++++++++++++---- alembic-env.py | 59 ----- alembic/env.py | 50 ++++ .../dbe48f596a44_initial_complete_schema.py | 220 ++++++++++++++++++ models/__init__.py | 25 ++ models/api/__init__.py | 24 ++ models/database/__init__.py | 19 ++ requirements.txt | 2 - scripts/setup_dev.py | 175 -------------- scripts/verify_setup.py | 200 ++++++++++++++++ 10 files changed, 695 insertions(+), 281 deletions(-) delete mode 100644 alembic-env.py create mode 100644 alembic/versions/dbe48f596a44_initial_complete_schema.py create mode 100644 models/__init__.py delete mode 100644 scripts/setup_dev.py create mode 100644 scripts/verify_setup.py diff --git a/Makefile b/Makefile index 3729c371..c17a13da 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,6 @@ install-test: install-dev: pip install -r requirements.txt - pip install -r tests/requirements-test.txt pip install -r requirements-dev.txt install-docs: @@ -33,9 +32,9 @@ dev: uvicorn main:app --reload --host 0.0.0.0 --port 8000 # Safe development startup (checks migrations first) -dev-safe: migrate-check - @echo ๐Ÿ” Migration check passed, starting development server... - uvicorn main:app --reload --host 0.0.0.0 --port 8000 +# Development workflow with migration check +dev-safe: migrate-check dev + @echo Development server started with migration check dev-with-docs: @echo Starting API server and documentation server... @@ -148,7 +147,7 @@ ci: format lint test-coverage # Database-aware CI pipeline ci-db: format lint migrate-check test-coverage - @echo โœ… CI pipeline with database checks completed + @echo CI pipeline with database checks completed # Database migrations migrate-create: @@ -158,69 +157,69 @@ migrate-create-manual: @if "$(message)"=="" (echo Error: Please provide a message. Usage: make migrate-create-manual message="your_description") else (alembic revision -m "$(message)") migrate-up: - @echo Running database migrations... + @echo [MIGRATE] Running database migrations... alembic upgrade head - @echo โœ… Migrations completed successfully + @echo [MIGRATE] Migrations completed successfully migrate-down: - @echo Rolling back last migration... + @echo [MIGRATE] Rolling back last migration... alembic downgrade -1 - @echo โœ… Rollback completed + @echo [MIGRATE] Rollback completed migrate-down-to: @if "$(revision)"=="" (echo Error: Please provide revision. Usage: make migrate-down-to revision="revision_id") else (alembic downgrade $(revision)) migrate-reset: - @echo Resetting database... + @echo [MIGRATE] Resetting database... alembic downgrade base alembic upgrade head - @echo โœ… Database reset completed + @echo [MIGRATE] Database reset completed migrate-status: - @echo ๐Ÿ“Š Current migration status: + @echo [STATUS] Current migration status: alembic current @echo. - @echo ๐Ÿ“‹ Migration history: + @echo [HISTORY] Migration history: alembic history --verbose migrate-show: @if "$(revision)"=="" (echo Error: Please provide revision. Usage: make migrate-show revision="revision_id") else (alembic show $(revision)) migrate-heads: - @echo ๐Ÿ“ Current migration heads: + @echo [INFO] Current migration heads: alembic heads migrate-check: - @echo ๐Ÿ” Checking for pending migrations... - @python -c "from alembic import command, config; cfg = config.Config('alembic.ini'); command.check(cfg)" && echo "โœ… No pending migrations" || echo "โš ๏ธ Pending migrations found" + @echo [CHECK] Checking for pending migrations... + @python -c "from alembic import command, config; cfg = config.Config('alembic.ini'); command.check(cfg)" && echo "[CHECK] No pending migrations" || echo "[WARNING] Pending migrations found" + +# Create the index migration we discussed +migrate-create-indexes: + @echo [CREATE] Creating index migration... + alembic revision -m "add_database_indexes" + @echo [CREATE] Index migration created. Please edit the file to add your indexes. # Database initialization (enhanced) db-init: migrate-up - @echo ๐Ÿš€ Database initialization completed + @echo [INIT] Database initialization completed db-fresh: migrate-reset - @echo ๐Ÿ”„ Fresh database setup completed - -# Database backup before risky operations (if using PostgreSQL/MySQL) -backup-db: - @echo ๐Ÿ’พ Creating database backup... - @python scripts/backup_database.py - @echo โœ… Database backup created + @echo [INIT] Fresh database setup completed # === FRESH START COMMANDS (Development) === fresh-backup: - @echo ๐Ÿ’พ Creating backup of current state... + @echo [BACKUP] Creating backup of current state... @if not exist scripts mkdir scripts @python scripts/backup_database.py fresh-clean: - @echo ๐Ÿงน Cleaning up for fresh start... + @echo [CLEAN] Cleaning up for fresh start... @if exist ecommerce.db del ecommerce.db @if exist alembic\versions\*.py (for %%f in (alembic\versions\*.py) do if not "%%~nf"=="__init__" del "%%f") - @echo โœ… Cleanup completed + @echo [CLEAN] Cleanup completed fresh-setup: fresh-backup fresh-clean - @echo ๐Ÿš€ Setting up fresh database with proper migrations... + @echo [SETUP] Setting up fresh database with proper migrations... @echo. @echo Step 1: Creating initial migration from models... alembic revision --autogenerate -m "initial_schema_and_indexes" @@ -228,12 +227,12 @@ fresh-setup: fresh-backup fresh-clean @echo Step 2: Running the migration to create database... alembic upgrade head @echo. - @echo โœ… Fresh setup completed! + @echo [SETUP] Fresh setup completed! @echo Database is now managed entirely by Alembic migrations. # Check what the fresh migration would contain fresh-preview: - @echo ๐Ÿ” Previewing what the fresh migration would contain... + @echo [PREVIEW] Showing what the fresh migration would contain... @echo This will show what tables/indexes would be created. @echo. @if exist ecommerce.db (echo Current database detected - showing diff) else (echo No database - showing full schema) @@ -241,12 +240,12 @@ fresh-preview: # Complete development environment setup with fresh database dev-fresh-setup: install-all fresh-setup - @echo ๐ŸŽ‰ Complete fresh development setup completed! + @echo [SUCCESS] Complete fresh development setup completed! @echo. @echo What was done: - @echo โœ… All dependencies installed - @echo โœ… Database created with migrations - @echo โœ… Migration tracking initialized + @echo [OK] All dependencies installed + @echo [OK] Database created with migrations + @echo [OK] Migration tracking initialized @echo. @echo Next steps: @echo 1. Review the migration file in alembic/versions/ @@ -256,15 +255,8 @@ dev-fresh-setup: install-all fresh-setup # Verify the fresh setup worked verify-fresh: - @echo ๐Ÿ” Verifying fresh setup... - @echo. - @echo Migration status: - @alembic current - @echo. - @echo Database tables: - @python -c "from sqlalchemy import create_engine, text; engine = create_engine('sqlite:///./ecommerce.db'); print('Tables:', [r[0] for r in engine.execute(text('SELECT name FROM sqlite_master WHERE type=\"table\"')).fetchall()])" - @echo. - @echo โœ… Verification completed + @echo [VERIFY] Running comprehensive verification... + @python scripts/verify_setup.py # Docker commands docker-build: @@ -283,10 +275,10 @@ docker-restart: docker-down docker-up # Pre-deployment checks pre-deploy: qa migrate-status - @echo ๐Ÿš€ Pre-deployment checks completed! + @echo [DEPLOY] Pre-deployment checks completed! @echo Ready for deployment. -# Production deployment +# Production deployment with migrations deploy-staging: migrate-up docker-compose -f docker-compose.staging.yml up -d @@ -315,6 +307,7 @@ clean-all: clean docs-clean @echo All build artifacts cleaned! # Development workflow shortcuts +# Enhanced setup commands setup: install-all db-init @echo Development environment setup complete! @echo Run 'make dev-full' to start both API and documentation servers @@ -323,6 +316,7 @@ setup-fresh: install-all db-fresh @echo Fresh development environment setup complete! @echo Run 'make dev-full' to start both API and documentation servers + setup-test: install-test @echo Test environment setup complete! @@ -360,6 +354,106 @@ workflow-deploy: @echo 2. Ready for deployment! @echo Run 'make deploy-staging' or 'make deploy-prod' to deploy. +# Create missing __init__.py files +fix-init-files: + @echo [FIX] Creating missing __init__.py files... + @if not exist models\__init__.py (echo # models/__init__.py > models\__init__.py && echo Created models\__init__.py) + @if not exist models\database\__init__.py (echo # models/database/__init__.py > models\database\__init__.py && echo Created models\database\__init__.py) + @if not exist models\api\__init__.py (echo # models/api/__init__.py > models\api\__init__.py && echo Created models\api\__init__.py) + @echo [FIX] Init files created + +# Test model imports separately +test-database-imports: + @echo [TEST] Testing database model imports... + @python -c "from models.database.base import Base; print('โœ“ Base imported:', type(Base)); print('โœ“ Base metadata tables:', list(Base.metadata.tables.keys()))" + +test-api-imports: + @echo [TEST] Testing API model imports... + @python -c "import models.api; print('โœ“ API models package imported')" + +test-all-imports: + @echo [TEST] Testing all model imports... + @python -c "from models import Base, User, Product, Stock; print('โœ“ Database models imported'); print('โœ“ Found tables:', list(Base.metadata.tables.keys())); import models.api; print('โœ“ API models imported')" + +# Enhanced migration fix with better error handling +fix-migration: fix-init-files + @echo [FIX] Fixing migration with proper model imports... + @echo Step 1: Testing imports first... + @$(MAKE) test-database-imports + @echo Step 2: Removing current migration... + @alembic downgrade base || echo "No migrations to downgrade" + @echo Step 3: Deleting migration files... + @for %%f in (alembic\versions\*.py) do if not "%%~nf"=="__init__" del "%%f" + @echo Step 4: Creating new migration with all models... + @alembic revision --autogenerate -m "initial_complete_schema" + @echo Step 5: Applying new migration... + @alembic upgrade head + @echo [FIX] Migration fix completed! + +# Comprehensive project verification +verify-project: + @echo [VERIFY] Comprehensive project verification... + @echo. + @echo Testing database models: + @$(MAKE) test-database-imports + @echo. + @echo Testing API models: + @$(MAKE) test-api-imports + @echo. + @echo Database verification: + @python scripts/verify_setup.py + @echo. + @echo [VERIFY] Project verification completed! + +# Complete fresh restart for your project structure +fresh-restart: test-all-imports fix-migration verify-project + @echo [SUCCESS] Complete fresh restart with full project structure completed! + @echo. + @echo Your project now has: + @echo โœ“ Database models (SQLAlchemy) for data storage + @echo โœ“ API models (Pydantic) for request/response validation + @echo โœ“ Proper Alembic migrations + @echo โœ“ All imports working correctly + @echo. + @echo Next steps: + @echo 1. Run 'make dev' to start your API server + @echo 2. Visit http://localhost:8000/docs for API documentation + @echo 3. Use 'make migrate-create message="description"' for future changes + +# Simple fix commands - add to your Makefile + +# Simple fix for import issues +simple-fix: + @echo [SIMPLE-FIX] Running simple import fix... + @python scripts/simple_fix.py + +# Test imports one by one +test-step-by-step: + @echo [TEST] Testing imports step by step... + @python -c "from models.database.base import Base; print('โœ“ Base OK:', len(Base.metadata.tables), 'tables')" + @python -c "from models.database.user import User; print('โœ“ User OK')" + @python -c "from models.database.product import Product; print('โœ“ Product OK')" + @python -c "from models.database.stock import Stock; print('โœ“ Stock OK')" + @python -c "from models.database.shop import Shop; print('โœ“ Shop OK')" + @python -c "from models.database.marketplace import MarketplaceImportJob; print('โœ“ Marketplace OK')" + +# Simple migration fix +simple-migration-fix: + @echo [MIGRATION] Simple migration fix... + @echo Step 1: Remove old migration... + @alembic downgrade base || echo "No migration to downgrade" + @echo Step 2: Delete migration files... + @for %%f in (alembic\versions\*.py) do if not "%%~nf"=="__init__" del "%%f" + @echo Step 3: Create new migration... + @alembic revision --autogenerate -m "complete_schema" + @echo Step 4: Apply migration... + @alembic upgrade head + @echo [MIGRATION] Done! + +# Complete simple workflow +simple-restart: simple-fix test-step-by-step simple-migration-fix verify-fresh + @echo [SUCCESS] Simple restart completed! + # Help command help: @echo Available commands: @@ -440,6 +534,24 @@ help: @echo make docs-serve # Start documentation server @echo make qa # Run quality checks +# Help for migration +help-migrations: + @echo === DATABASE MIGRATIONS === + @echo migrate-create message="msg" - Create auto-generated migration + @echo migrate-create-manual message="msg" - Create empty migration template + @echo migrate-up - Run all pending migrations + @echo migrate-down - Rollback last migration + @echo migrate-down-to revision="id" - Rollback to specific revision + @echo migrate-reset - Reset database to base and rerun all + @echo migrate-status - Show current migration status and history + @echo migrate-show revision="id" - Show specific migration details + @echo migrate-heads - Show current migration heads + @echo migrate-check - Check for pending migrations + @echo migrate-create-indexes - Create the database indexes migration + @echo db-init - Initialize database with migrations + @echo db-fresh - Fresh database setup + @echo. + # Help for fresh start help-fresh: @echo === FRESH START COMMANDS === diff --git a/alembic-env.py b/alembic-env.py deleted file mode 100644 index e9959486..00000000 --- a/alembic-env.py +++ /dev/null @@ -1,59 +0,0 @@ -import os -import sys -from logging.config import fileConfig - -from sqlalchemy import engine_from_config, pool - -from alembic import context - -# Add your project directory to the Python path -sys.path.append(os.path.dirname(os.path.dirname(__file__))) - -from app.core.config import settings -from models.database.base import Base - -# Alembic Config object -config = context.config - -# Override sqlalchemy.url with our settings -config.set_main_option("sqlalchemy.url", settings.database_url) - -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -target_metadata = Base.metadata - - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode.""" - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - """Run migrations in 'online' mode.""" - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/alembic/env.py b/alembic/env.py index e9959486..b6db6883 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -12,6 +12,56 @@ sys.path.append(os.path.dirname(os.path.dirname(__file__))) from app.core.config import settings from models.database.base import Base +# === IMPORTANT: Import all your DATABASE models here === +# Only import SQLAlchemy models, not Pydantic API models +print("[ALEMBIC] Importing database models...") + +try: + from models.database.user import User + print(" โœ“ User model imported") +except ImportError as e: + print(f" โœ— User model failed: {e}") + +try: + from models.database.product import Product + print(" โœ“ Product model imported") +except ImportError as e: + print(f" โœ— Product model failed: {e}") + +try: + from models.database.stock import Stock + print(" โœ“ Stock model imported") +except ImportError as e: + print(f" โœ— Stock model failed: {e}") + +try: + from models.database.shop import Shop, ShopProduct + print(" โœ“ Shop models imported") +except ImportError as e: + print(f" โœ— Shop models failed: {e}") + +try: + from models.database.marketplace import MarketplaceImportJob + print(" โœ“ Marketplace model imported") +except ImportError as e: + print(f" โœ— Marketplace model failed: {e}") + +# Check if there are any additional models in the database directory +try: + from models.database import auth as auth_models + print(" โœ“ Auth models imported") +except ImportError: + print(" - Auth models not found (optional)") + +try: + from models.database import admin as admin_models + print(" โœ“ Admin models imported") +except ImportError: + print(" - Admin models not found (optional)") + +print(f"[ALEMBIC] Model import completed. Tables found: {list(Base.metadata.tables.keys())}") +print(f"[ALEMBIC] Total tables to create: {len(Base.metadata.tables)}") + # Alembic Config object config = context.config diff --git a/alembic/versions/dbe48f596a44_initial_complete_schema.py b/alembic/versions/dbe48f596a44_initial_complete_schema.py new file mode 100644 index 00000000..716b381d --- /dev/null +++ b/alembic/versions/dbe48f596a44_initial_complete_schema.py @@ -0,0 +1,220 @@ +"""initial_complete_schema + +Revision ID: dbe48f596a44 +Revises: +Create Date: 2025-09-21 15:45:40.290712 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'dbe48f596a44' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('products', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('product_id', sa.String(), nullable=False), + sa.Column('title', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('link', sa.String(), nullable=True), + sa.Column('image_link', sa.String(), nullable=True), + sa.Column('availability', sa.String(), nullable=True), + sa.Column('price', sa.String(), nullable=True), + sa.Column('brand', sa.String(), nullable=True), + sa.Column('gtin', sa.String(), nullable=True), + sa.Column('mpn', sa.String(), nullable=True), + sa.Column('condition', sa.String(), nullable=True), + sa.Column('adult', sa.String(), nullable=True), + sa.Column('multipack', sa.Integer(), nullable=True), + sa.Column('is_bundle', sa.String(), nullable=True), + sa.Column('age_group', sa.String(), nullable=True), + sa.Column('color', sa.String(), nullable=True), + sa.Column('gender', sa.String(), nullable=True), + sa.Column('material', sa.String(), nullable=True), + sa.Column('pattern', sa.String(), nullable=True), + sa.Column('size', sa.String(), nullable=True), + sa.Column('size_type', sa.String(), nullable=True), + sa.Column('size_system', sa.String(), nullable=True), + sa.Column('item_group_id', sa.String(), nullable=True), + sa.Column('google_product_category', sa.String(), nullable=True), + sa.Column('product_type', sa.String(), nullable=True), + sa.Column('custom_label_0', sa.String(), nullable=True), + sa.Column('custom_label_1', sa.String(), nullable=True), + sa.Column('custom_label_2', sa.String(), nullable=True), + sa.Column('custom_label_3', sa.String(), nullable=True), + sa.Column('custom_label_4', sa.String(), nullable=True), + sa.Column('additional_image_link', sa.String(), nullable=True), + sa.Column('sale_price', sa.String(), nullable=True), + sa.Column('unit_pricing_measure', sa.String(), nullable=True), + sa.Column('unit_pricing_base_measure', sa.String(), nullable=True), + sa.Column('identifier_exists', sa.String(), nullable=True), + sa.Column('shipping', sa.String(), nullable=True), + sa.Column('currency', sa.String(), nullable=True), + sa.Column('marketplace', sa.String(), nullable=True), + sa.Column('shop_name', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('idx_marketplace_brand', 'products', ['marketplace', 'brand'], unique=False) + op.create_index('idx_marketplace_shop', 'products', ['marketplace', 'shop_name'], unique=False) + op.create_index(op.f('ix_products_availability'), 'products', ['availability'], unique=False) + op.create_index(op.f('ix_products_brand'), 'products', ['brand'], unique=False) + op.create_index(op.f('ix_products_google_product_category'), 'products', ['google_product_category'], unique=False) + op.create_index(op.f('ix_products_gtin'), 'products', ['gtin'], unique=False) + op.create_index(op.f('ix_products_id'), 'products', ['id'], unique=False) + op.create_index(op.f('ix_products_marketplace'), 'products', ['marketplace'], unique=False) + op.create_index(op.f('ix_products_product_id'), 'products', ['product_id'], unique=True) + op.create_index(op.f('ix_products_shop_name'), 'products', ['shop_name'], unique=False) + op.create_table('users', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('email', sa.String(), nullable=False), + sa.Column('username', sa.String(), nullable=False), + sa.Column('hashed_password', sa.String(), nullable=False), + sa.Column('role', sa.String(), nullable=False), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('last_login', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=True) + op.create_index(op.f('ix_users_id'), 'users', ['id'], unique=False) + op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True) + op.create_table('shops', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('shop_code', sa.String(), nullable=False), + sa.Column('shop_name', sa.String(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('owner_id', sa.Integer(), nullable=False), + sa.Column('contact_email', sa.String(), nullable=True), + sa.Column('contact_phone', sa.String(), nullable=True), + sa.Column('website', sa.String(), nullable=True), + sa.Column('business_address', sa.Text(), nullable=True), + sa.Column('tax_number', sa.String(), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=True), + sa.Column('is_verified', sa.Boolean(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['owner_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_shops_id'), 'shops', ['id'], unique=False) + op.create_index(op.f('ix_shops_shop_code'), 'shops', ['shop_code'], unique=True) + op.create_table('marketplace_import_jobs', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('source_url', sa.String(), nullable=False), + sa.Column('marketplace', sa.String(), nullable=False), + sa.Column('shop_name', sa.String(), nullable=False), + sa.Column('shop_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('imported_count', sa.Integer(), nullable=True), + sa.Column('updated_count', sa.Integer(), nullable=True), + sa.Column('error_count', sa.Integer(), nullable=True), + sa.Column('total_processed', sa.Integer(), nullable=True), + sa.Column('error_message', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['shop_id'], ['shops.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('idx_marketplace_import_shop_id', 'marketplace_import_jobs', ['shop_id'], unique=False) + op.create_index('idx_marketplace_import_shop_status', 'marketplace_import_jobs', ['status'], unique=False) + op.create_index('idx_marketplace_import_user_marketplace', 'marketplace_import_jobs', ['user_id', 'marketplace'], unique=False) + op.create_index(op.f('ix_marketplace_import_jobs_id'), 'marketplace_import_jobs', ['id'], unique=False) + op.create_index(op.f('ix_marketplace_import_jobs_marketplace'), 'marketplace_import_jobs', ['marketplace'], unique=False) + op.create_index(op.f('ix_marketplace_import_jobs_shop_name'), 'marketplace_import_jobs', ['shop_name'], unique=False) + op.create_table('shop_products', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('shop_id', sa.Integer(), nullable=False), + sa.Column('product_id', sa.Integer(), nullable=False), + sa.Column('shop_product_id', sa.String(), nullable=True), + sa.Column('shop_price', sa.Float(), nullable=True), + sa.Column('shop_sale_price', sa.Float(), nullable=True), + sa.Column('shop_currency', sa.String(), nullable=True), + sa.Column('shop_availability', sa.String(), nullable=True), + sa.Column('shop_condition', sa.String(), nullable=True), + sa.Column('is_featured', sa.Boolean(), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=True), + sa.Column('display_order', sa.Integer(), nullable=True), + sa.Column('min_quantity', sa.Integer(), nullable=True), + sa.Column('max_quantity', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['product_id'], ['products.id'], ), + sa.ForeignKeyConstraint(['shop_id'], ['shops.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('shop_id', 'product_id', name='uq_shop_product') + ) + op.create_index('idx_shop_product_active', 'shop_products', ['shop_id', 'is_active'], unique=False) + op.create_index('idx_shop_product_featured', 'shop_products', ['shop_id', 'is_featured'], unique=False) + op.create_index(op.f('ix_shop_products_id'), 'shop_products', ['id'], unique=False) + op.create_table('stock', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('gtin', sa.String(), nullable=False), + sa.Column('location', sa.String(), nullable=False), + sa.Column('quantity', sa.Integer(), nullable=False), + sa.Column('reserved_quantity', sa.Integer(), nullable=True), + sa.Column('shop_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['shop_id'], ['shops.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('gtin', 'location', name='uq_stock_gtin_location') + ) + op.create_index('idx_stock_gtin_location', 'stock', ['gtin', 'location'], unique=False) + op.create_index(op.f('ix_stock_gtin'), 'stock', ['gtin'], unique=False) + op.create_index(op.f('ix_stock_id'), 'stock', ['id'], unique=False) + op.create_index(op.f('ix_stock_location'), 'stock', ['location'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_stock_location'), table_name='stock') + op.drop_index(op.f('ix_stock_id'), table_name='stock') + op.drop_index(op.f('ix_stock_gtin'), table_name='stock') + op.drop_index('idx_stock_gtin_location', table_name='stock') + op.drop_table('stock') + op.drop_index(op.f('ix_shop_products_id'), table_name='shop_products') + op.drop_index('idx_shop_product_featured', table_name='shop_products') + op.drop_index('idx_shop_product_active', table_name='shop_products') + op.drop_table('shop_products') + op.drop_index(op.f('ix_marketplace_import_jobs_shop_name'), table_name='marketplace_import_jobs') + op.drop_index(op.f('ix_marketplace_import_jobs_marketplace'), table_name='marketplace_import_jobs') + op.drop_index(op.f('ix_marketplace_import_jobs_id'), table_name='marketplace_import_jobs') + op.drop_index('idx_marketplace_import_user_marketplace', table_name='marketplace_import_jobs') + op.drop_index('idx_marketplace_import_shop_status', table_name='marketplace_import_jobs') + op.drop_index('idx_marketplace_import_shop_id', table_name='marketplace_import_jobs') + op.drop_table('marketplace_import_jobs') + op.drop_index(op.f('ix_shops_shop_code'), table_name='shops') + op.drop_index(op.f('ix_shops_id'), table_name='shops') + op.drop_table('shops') + op.drop_index(op.f('ix_users_username'), table_name='users') + op.drop_index(op.f('ix_users_id'), table_name='users') + op.drop_index(op.f('ix_users_email'), table_name='users') + op.drop_table('users') + op.drop_index(op.f('ix_products_shop_name'), table_name='products') + op.drop_index(op.f('ix_products_product_id'), table_name='products') + op.drop_index(op.f('ix_products_marketplace'), table_name='products') + op.drop_index(op.f('ix_products_id'), table_name='products') + op.drop_index(op.f('ix_products_gtin'), table_name='products') + op.drop_index(op.f('ix_products_google_product_category'), table_name='products') + op.drop_index(op.f('ix_products_brand'), table_name='products') + op.drop_index(op.f('ix_products_availability'), table_name='products') + op.drop_index('idx_marketplace_shop', table_name='products') + op.drop_index('idx_marketplace_brand', table_name='products') + op.drop_table('products') + # ### end Alembic commands ### diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 00000000..5f904bd3 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1,25 @@ +# models/__init__.py +"""Models package - Database and API models.""" + +# Database models (SQLAlchemy) +from .database.base import Base +from .database.user import User +from .database.product import Product +from .database.stock import Stock +from .database.shop import Shop, ShopProduct +from .database.marketplace import MarketplaceImportJob + +# API models (Pydantic) - import the modules, not all classes +from . import api + +# Export database models for Alembic +__all__ = [ + "Base", + "User", + "Product", + "Stock", + "Shop", + "ShopProduct", + "MarketplaceImportJob", + "api", # API models namespace +] diff --git a/models/api/__init__.py b/models/api/__init__.py index e69de29b..7ac8a6c3 100644 --- a/models/api/__init__.py +++ b/models/api/__init__.py @@ -0,0 +1,24 @@ +# models/api/__init__.py +"""API models package - Pydantic models for request/response validation.""" + +# Import API model modules +from . import base +from . import auth +from . import product +from . import stock +from . import shop +from . import marketplace +from . import stats + +# Common imports for convenience +from .base import * # Base Pydantic models + +__all__ = [ + "base", + "auth", + "product", + "stock", + "shop", + "marketplace", + "stats", +] diff --git a/models/database/__init__.py b/models/database/__init__.py index e69de29b..ae9173a4 100644 --- a/models/database/__init__.py +++ b/models/database/__init__.py @@ -0,0 +1,19 @@ +# models/database/__init__.py +"""Database models package.""" + +from .base import Base +from .user import User +from .product import Product +from .stock import Stock +from .shop import Shop, ShopProduct +from .marketplace import MarketplaceImportJob + +__all__ = [ + "Base", + "User", + "Product", + "Stock", + "Shop", + "ShopProduct", + "MarketplaceImportJob", +] diff --git a/requirements.txt b/requirements.txt index 83d9a59d..3f5141c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -# Database migrations -alembic>=1.13.0 starlette==0.27.0 # requirements.txt # Core FastAPI and web framework diff --git a/scripts/setup_dev.py b/scripts/setup_dev.py deleted file mode 100644 index e913e234..00000000 --- a/scripts/setup_dev.py +++ /dev/null @@ -1,175 +0,0 @@ -# scripts/setup_dev.py -# !/usr/bin/env python3 -"""Development environment setup script.""" - -import subprocess -import sys -from pathlib import Path - - -def run_command(command, description): - """Run a shell command and handle errors.""" - print(f"Running: {description}") - try: - subprocess.run(command, shell=True, check=True) - print(f"โœ… {description} completed successfully") - except subprocess.CalledProcessError as e: - print(f"โŒ {description} failed: {e}") - return False - return True - - -def setup_alembic(): - """Set up Alembic for database migrations.""" - alembic_dir = Path("alembic") - - # Check if alembic directory exists and has necessary files - if not alembic_dir.exists() or not (alembic_dir / "script.py.mako").exists(): - print("๐Ÿ“ Initializing Alembic...") - if alembic_dir.exists(): - # Remove incomplete alembic directory - import shutil - - shutil.rmtree(alembic_dir) - - if not run_command("alembic init alembic", "Initializing Alembic"): - return False - - # Update alembic/env.py with proper configuration - env_py_content = '''from logging.config import fileConfig -from sqlalchemy import engine_from_config, pool -from alembic import context -import os -import sys - -# Add your project directory to the Python path -sys.path.append(os.path.dirname(os.path.dirname(__file__))) - -from models.database import Base -from config.settings import settings - -# Alembic Config object -config = context.config - -# Override sqlalchemy.url with our settings -config.set_main_option("sqlalchemy.url", settings.database_url) - -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -target_metadata = Base.metadata - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode.""" - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - -def run_migrations_online() -> None: - """Run migrations in 'online' mode.""" - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() -''' - - env_py_path = alembic_dir / "env.py" - env_py_path.write_text(env_py_content) - print("โœ… Updated alembic/env.py with project configuration") - - return True - - -def setup_environment(): - """Set up the development environment.""" - print("๐Ÿš€ Setting up ecommerce API development environment...") - - # Check Python version - if sys.version_info < (3, 8): - print("โŒ Python 3.8+ is required") - return False - - # Create .env file if it doesn't exist - env_file = Path(".env") - env_example = Path(".env.example") - - if not env_file.exists() and env_example.exists(): - print("๐Ÿ“ Creating .env file from .env.example...") - env_file.write_text(env_example.read_text()) - elif not env_file.exists(): - print("๐Ÿ“ Creating default .env file...") - default_env = """DATABASE_URL=sqlite:///./ecommerce.db -JWT_SECRET_KEY=development-secret-key-change-in-production -JWT_EXPIRE_HOURS=24 -API_HOST=0.0.0.0 -API_PORT=8000 -DEBUG=True -RATE_LIMIT_ENABLED=True -DEFAULT_RATE_LIMIT=100 -DEFAULT_WINDOW_SECONDS=3600 -LOG_LEVEL=INFO -""" - env_file.write_text(default_env) - - # Install dependencies - if not run_command("pip install -r requirements.txt", "Installing dependencies"): - return False - - # Set up Alembic - if not setup_alembic(): - print( - "โš ๏ธ Alembic setup failed. You'll need to set up database migrations manually." - ) - return False - - # Create initial migration - if not run_command( - 'alembic revision --autogenerate -m "Initial migration"', - "Creating initial migration", - ): - print("โš ๏ธ Initial migration creation failed. Check your database models.") - - # Apply migrations - if not run_command("alembic upgrade head", "Setting up database"): - print( - "โš ๏ธ Database setup failed. Make sure your DATABASE_URL is correct in .env" - ) - - # Run tests - if not run_command("pytest", "Running tests"): - print("โš ๏ธ Some tests failed. Check the output above.") - - print("\n๐ŸŽ‰ Development environment setup complete!") - print("To start the development server, run:") - print(" uvicorn main:app --reload") - print("\nDatabase commands:") - print(' alembic revision --autogenerate -m "Description" # Create migration') - print(" alembic upgrade head # Apply migrations") - print(" alembic current # Check status") - - return True - - -if __name__ == "__main__": - setup_environment() diff --git a/scripts/verify_setup.py b/scripts/verify_setup.py new file mode 100644 index 00000000..a973ea37 --- /dev/null +++ b/scripts/verify_setup.py @@ -0,0 +1,200 @@ +# scripts/verify_setup.py +"""Verify database setup and migration status for complete project structure.""" + +import os +import sys +from sqlalchemy import create_engine, text +from alembic import command +from alembic.config import Config + +# Add project root to Python path (same as alembic does) +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + + +def verify_database_setup(): + """Verify database and migration setup.""" + + print("[VERIFY] Verifying database setup...") + print() + + # Check if database file exists + db_path = "ecommerce.db" + if not os.path.exists(db_path): + print("[ERROR] Database file not found!") + return False + + print(f"[OK] Database file exists: {db_path}") + + try: + # Create engine and connect + engine = create_engine('sqlite:///./ecommerce.db') + + with engine.connect() as conn: + # Get table list + result = conn.execute(text("SELECT name FROM sqlite_master WHERE type='table'")) + tables = [row[0] for row in result.fetchall()] + + print(f"[OK] Found {len(tables)} tables:") + + # Expected tables from your models + expected_tables = [ + 'users', 'products', 'stock', 'shops', 'shop_products', + 'marketplace_import_jobs', 'alembic_version' + ] + + for table in sorted(tables): + if table == 'alembic_version': + print(f" โœ“ {table} (migration tracking)") + elif table in expected_tables: + print(f" โœ“ {table}") + else: + print(f" ? {table} (unexpected)") + + # Check for missing expected tables + missing_tables = set(expected_tables) - set(tables) - {'alembic_version'} + if missing_tables: + print(f"[WARNING] Missing expected tables: {missing_tables}") + + # Check if alembic_version table exists + if 'alembic_version' in tables: + result = conn.execute(text("SELECT version_num FROM alembic_version")) + version = result.fetchone() + if version: + print(f"[OK] Migration version: {version[0]}") + else: + print("[WARNING] No migration version recorded") + else: + print("[ERROR] No alembic_version table - migrations not initialized") + return False + + except Exception as e: + print(f"[ERROR] Database connection failed: {e}") + return False + + # Check Alembic configuration + try: + alembic_cfg = Config("alembic.ini") + print("[OK] Alembic configuration found") + + # Try to get current migration + print() + print("[MIGRATE] Current migration status:") + command.current(alembic_cfg, verbose=True) + + except Exception as e: + print(f"[ERROR] Alembic configuration issue: {e}") + return False + + print() + print("[SUCCESS] Database setup verification completed!") + return True + + +def verify_model_structure(): + """Verify both database and API model structure.""" + + print("[MODELS] Verifying model structure...") + + # Test database models + try: + from models.database.base import Base + print(f"[OK] Database Base imported") + print(f"[INFO] Found {len(Base.metadata.tables)} database tables: {list(Base.metadata.tables.keys())}") + + # Import specific models + from models.database.user import User + from models.database.product import Product + from models.database.stock import Stock + from models.database.shop import Shop, ShopProduct + from models.database.marketplace import MarketplaceImportJob + + print("[OK] All database models imported successfully") + + except ImportError as e: + print(f"[ERROR] Database model import failed: {e}") + return False + + # Test API models + try: + import models.api + print("[OK] API models package imported") + + # Test specific API model imports + api_modules = ['base', 'auth', 'product', 'stock', 'shop', 'marketplace', 'admin', 'stats'] + for module in api_modules: + try: + __import__(f'models.api.{module}') + print(f" โœ“ models.api.{module}") + except ImportError: + print(f" ? models.api.{module} (not found, optional)") + + except ImportError as e: + print(f"[WARNING] API models import issue: {e}") + # This is not critical for database operations + + return True + + +def check_project_structure(): + """Check overall project structure.""" + + print(f"\n[STRUCTURE] Checking project structure...") + + critical_paths = [ + "models/database/base.py", + "models/database/user.py", + "models/database/product.py", + "models/database/stock.py", + "app/core/config.py", + "alembic/env.py", + "alembic.ini" + ] + + for path in critical_paths: + if os.path.exists(path): + print(f" โœ“ {path}") + else: + print(f" โœ— {path} (missing)") + + # Check __init__.py files + init_files = [ + "models/__init__.py", + "models/database/__init__.py", + "models/api/__init__.py" + ] + + print(f"\n[INIT] Checking __init__.py files...") + for init_file in init_files: + if os.path.exists(init_file): + print(f" โœ“ {init_file}") + else: + print(f" โœ— {init_file} (missing - will cause import issues)") + + +if __name__ == "__main__": + success = True + + check_project_structure() + + if verify_model_structure(): + success = verify_database_setup() + else: + success = False + + if success: + print() + print("[READY] ๐ŸŽ‰ Your complete project structure is ready!") + print() + print("Database models: โœ“ SQLAlchemy models for data storage") + print("API models: โœ“ Pydantic models for request/response validation") + print("Migrations: โœ“ Alembic managing database schema") + print() + print("Next steps:") + print(" 1. Run 'make dev' to start your FastAPI server") + print(" 2. Visit http://localhost:8000/docs for interactive API docs") + print(" 3. Use your API endpoints for authentication, products, stock, etc.") + sys.exit(0) + else: + print() + print("[FAILED] Project setup has issues that need to be resolved!") + sys.exit(1)