From 2c3dde2d75e821070a56d37f275dcfeeacb88b4f Mon Sep 17 00:00:00 2001 From: Dionisio Date: Mon, 2 Mar 2026 23:52:37 +0100 Subject: [PATCH] chore: remove redundant migrations directory and migrate binary All content from 003_add_device_codes.sql and 004_add_trigram_indexes.sql was already absorbed into db/schema.sql (the single source of truth). The migrations had additional problems: - Broken numbering (started at 003, missing 001/002) - 004 used CREATE INDEX CONCURRENTLY which fails inside sqlx transactions - No production flow ever invoked the migrate binary Removed: db/migrations/, src/bin/migrate.rs, migrations Cargo feature, [[bin]] migrate target, and doc/database-migrations.md. --- Cargo.toml | 6 - db/migrations/003_add_device_codes.sql | 99 ------------ db/migrations/004_add_trigram_indexes.sql | 43 ------ doc/database-migrations.md | 177 ---------------------- src/bin/migrate.rs | 49 ------ 5 files changed, 374 deletions(-) delete mode 100644 db/migrations/003_add_device_codes.sql delete mode 100644 db/migrations/004_add_trigram_indexes.sql delete mode 100644 doc/database-migrations.md delete mode 100644 src/bin/migrate.rs diff --git a/Cargo.toml b/Cargo.toml index 2f52bb44..f058bebf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,12 +59,6 @@ socket2 = { version = "0.6.2", features = ["all"] } [features] default = [] test_utils = ["mockall"] -migrations = ["sqlx/migrate"] - -[[bin]] -name = "migrate" -path = "src/bin/migrate.rs" -required-features = ["migrations"] [profile.release] lto = "fat" diff --git a/db/migrations/003_add_device_codes.sql b/db/migrations/003_add_device_codes.sql deleted file mode 100644 index fdb3fe95..00000000 --- a/db/migrations/003_add_device_codes.sql +++ /dev/null @@ -1,99 +0,0 @@ --- ============================================================ --- Migration 003: OAuth 2.0 Device Authorization Grant (RFC 8628) --- ============================================================ --- Adds the device_codes table to support the Device Authorization --- Grant flow for WebDAV/CalDAV/CardDAV client authentication. --- --- Flow: --- 1. Client POSTs to /api/auth/device/authorize → receives device_code + user_code --- 2. User opens verification_uri in browser, authenticates, enters user_code --- 3. Client polls /api/auth/device/token with device_code --- 4. Once approved, client receives access_token + refresh_token --- ============================================================ - --- Device code status enum -DO $BODY$ -BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_type t - JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace - WHERE t.typname = 'device_code_status' AND n.nspname = 'auth' - ) THEN - CREATE TYPE auth.device_code_status AS ENUM ( - 'pending', -- Waiting for user to authorize - 'authorized', -- User approved, tokens ready for polling client - 'denied', -- User denied the request - 'expired' -- TTL exceeded without user action - ); - END IF; -END $BODY$; - --- Device authorization codes table -CREATE TABLE IF NOT EXISTS auth.device_codes ( - -- Unique row ID - id VARCHAR(36) PRIMARY KEY, - - -- RFC 8628 §3.2: device_code — long opaque token sent to the client for polling - device_code VARCHAR(128) UNIQUE NOT NULL, - - -- RFC 8628 §3.2: user_code — short human-readable code shown on the client - -- and entered by the user on the verification page (e.g. "ABCD-1234") - user_code VARCHAR(16) UNIQUE NOT NULL, - - -- Name/description of the client requesting access (shown to user) - client_name VARCHAR(255) NOT NULL DEFAULT 'Unknown Client', - - -- Comma-separated scopes requested (e.g. "webdav,caldav,carddav") - scopes VARCHAR(512) NOT NULL DEFAULT 'webdav,caldav,carddav', - - -- Current status of the device flow - status auth.device_code_status NOT NULL DEFAULT 'pending', - - -- User who authorized the request (NULL until status = 'authorized') - user_id VARCHAR(36) REFERENCES auth.users(id) ON DELETE CASCADE, - - -- Tokens generated after authorization (NULL until status = 'authorized') - -- Stored encrypted/hashed depending on sensitivity - access_token TEXT, - refresh_token TEXT, - - -- RFC 8628 §3.2: verification_uri — full URL the user must visit - verification_uri TEXT NOT NULL, - - -- RFC 8628 §3.2: verification_uri_complete — URL with user_code pre-filled - verification_uri_complete TEXT, - - -- RFC 8628 §3.2: expires_in — encoded as an absolute timestamp - expires_at TIMESTAMP WITH TIME ZONE NOT NULL, - - -- RFC 8628 §3.2: interval — minimum polling interval in seconds - poll_interval_secs INTEGER NOT NULL DEFAULT 5, - - -- Last time the client polled (for slow_down enforcement) - last_poll_at TIMESTAMP WITH TIME ZONE, - - -- Timestamps - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - authorized_at TIMESTAMP WITH TIME ZONE -); - --- Index for client polling by device_code (hot path) -CREATE INDEX IF NOT EXISTS idx_device_codes_device_code - ON auth.device_codes(device_code); - --- Index for user verification page lookup by user_code -CREATE INDEX IF NOT EXISTS idx_device_codes_user_code - ON auth.device_codes(user_code) - WHERE status = 'pending'; - --- Index for cleanup of expired entries -CREATE INDEX IF NOT EXISTS idx_device_codes_expires_at - ON auth.device_codes(expires_at) - WHERE status = 'pending'; - --- Index for user's authorized devices -CREATE INDEX IF NOT EXISTS idx_device_codes_user_id - ON auth.device_codes(user_id) - WHERE status = 'authorized'; - -COMMENT ON TABLE auth.device_codes IS 'OAuth 2.0 Device Authorization Grant (RFC 8628) codes for DAV client authentication'; diff --git a/db/migrations/004_add_trigram_indexes.sql b/db/migrations/004_add_trigram_indexes.sql deleted file mode 100644 index fb88c4a3..00000000 --- a/db/migrations/004_add_trigram_indexes.sql +++ /dev/null @@ -1,43 +0,0 @@ --- Migration 004: Add GIN trigram indexes for ILIKE/LIKE substring search --- --- Eliminates full table scans on text search queries by enabling --- PostgreSQL's pg_trgm extension and creating GIN indexes with --- gin_trgm_ops on all columns used in ILIKE / LIKE '%text%' patterns. --- --- CONCURRENTLY is used so that no table locks are held during index --- creation — zero downtime for existing installations. --- --- NOTE: CREATE INDEX CONCURRENTLY cannot run inside a transaction block. --- If using sqlx migrate, run this file manually: --- psql -f db/migrations/004_add_trigram_indexes.sql - --- 0. Enable the pg_trgm extension (idempotent) -CREATE EXTENSION IF NOT EXISTS pg_trgm; - --- 1. Contacts — search_contacts(), get_contacts_by_email() -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_full_name_trgm - ON carddav.contacts USING gin (full_name gin_trgm_ops); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_first_name_trgm - ON carddav.contacts USING gin (first_name gin_trgm_ops); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_last_name_trgm - ON carddav.contacts USING gin (last_name gin_trgm_ops); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_nickname_trgm - ON carddav.contacts USING gin (nickname gin_trgm_ops); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_organization_trgm - ON carddav.contacts USING gin (organization gin_trgm_ops); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_email_text_trgm - ON carddav.contacts USING gin ((email::text) gin_trgm_ops); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_phone_text_trgm - ON carddav.contacts USING gin ((phone::text) gin_trgm_ops); - --- 2. Calendar events — find_events_by_summary() -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_calendar_events_summary_trgm - ON caldav.calendar_events USING gin (summary gin_trgm_ops); - --- 3. Files — search_files_paginated(), search_files_in_subtree(), suggest_files_by_name() -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_files_name_trgm - ON storage.files USING gin (name gin_trgm_ops); - --- 4. Folders — search_folders(), list_descendant_folders(), suggest_folders_by_name() -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_folders_name_trgm - ON storage.folders USING gin (name gin_trgm_ops); diff --git a/doc/database-migrations.md b/doc/database-migrations.md deleted file mode 100644 index de7adfc6..00000000 --- a/doc/database-migrations.md +++ /dev/null @@ -1,177 +0,0 @@ -# 18 - Database Migrations - -OxiCloud uses versioned SQL files to manage database schema changes. The migration system ensures changes are versioned, trackable, consistently applied across environments, reproducible, and independent of application code. - -## Directory Structure - -``` -OxiCloud/ -├── db/ -│ └── schema.sql # Main database schema -├── src/ - ├── bin/ - │ └── migrate.rs # CLI tool for running migrations - ├── common/ - │ └── db.rs # Database connection with schema verification -``` - -> The schema is currently applied from `db/schema.sql` at application startup (when it detects the `auth` tables don't exist). The `migrations/` directory doesn't exist yet, but `src/bin/migrate.rs` is ready to use sqlx migrations once the `migrations` feature is enabled. - -## Naming Conventions - -Migration files follow this format: `YYYYMMDDHHMMSS_brief_description.sql` - -- `YYYYMMDDHHMMSS` -- timestamp that guarantees correct ordering (year, month, day, hour, minute, second) -- `brief_description` -- short description of the migration purpose -- `.sql` -- SQL file extension - -## Running Migrations - -Migrations run via a dedicated CLI tool: - -```bash -cargo run --bin migrate --features migrations -``` - -This command: -1. Connects to the database configured in the environment -2. Looks for migrations in the `/migrations/` directory -3. Compares applied migrations against available ones -4. Sequentially executes pending migrations -5. Records applied migrations in a control table - -## Creating New Migrations - -To create a new migration: - -1. Create a new file in `migrations/` following the naming convention -2. Define the SQL changes in the file -3. Make sure the changes are compatible with the current schema version -4. Run the migrations - -Example migration structure: - -```sql --- Migración: Añadir tabla de etiquetas --- Descripción: Crea la tabla para almacenar etiquetas de archivos y sus relaciones - --- Crear tabla de etiquetas -CREATE TABLE IF NOT EXISTS auth.tags ( - id SERIAL PRIMARY KEY, - user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, - name TEXT NOT NULL, - color TEXT NOT NULL DEFAULT '#3498db', - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE(user_id, name) -); - --- Crear índices -CREATE INDEX IF NOT EXISTS idx_tags_user_id ON auth.tags(user_id); - --- Tabla de relación entre archivos y etiquetas -CREATE TABLE IF NOT EXISTS auth.file_tags ( - id SERIAL PRIMARY KEY, - tag_id INTEGER NOT NULL REFERENCES auth.tags(id) ON DELETE CASCADE, - file_id TEXT NOT NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE(tag_id, file_id) -); - --- Comentarios de documentación -COMMENT ON TABLE auth.tags IS 'Almacena etiquetas definidas por usuarios'; -COMMENT ON TABLE auth.file_tags IS 'Relación muchos-a-muchos entre archivos y etiquetas'; -``` - -## Best Practices - -1. **Incremental migrations** -- each migration should represent one atomic, coherent change. - -2. **Idempotent migrations** -- use commands that can run multiple times without errors (e.g., `CREATE TABLE IF NOT EXISTS`). - -3. **Forward-only migrations** -- design migrations to move forward, not roll back. If you need to undo a change, create a new migration. - -4. **Forward compatibility** -- migrations must be compatible with both the existing code and the code about to be deployed. - -5. **Test before deploying** -- test migrations in a production-like environment before applying them. - -6. **Documentation** -- document the purpose and key changes of each migration with comments inside the SQL file. - -## Troubleshooting - -### Checking Migration State - -OxiCloud includes startup-time detection to verify which migrations have been applied: - -```rust -// Desde src/common/db.rs -let migration_check = sqlx::query("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'auth' AND tablename = 'users')") - .fetch_one(&pool) - .await; - -match migration_check { - Ok(row) => { - let tables_exist: bool = row.get(0); - if !tables_exist { - tracing::warn!("Las tablas de la base de datos no existen. Por favor, ejecuta las migraciones con: cargo run --bin migrate --features migrations"); - } - }, - Err(_) => { - tracing::warn!("No se pudo verificar el estado de las migraciones. Por favor, ejecuta las migraciones con: cargo run --bin migrate --features migrations"); - } -} -``` - -### Common Issues - -1. **Database connection error** -- verify the connection URL in the **DATABASE_URL** environment variable. - -2. **Migration conflicts** -- if a migration fails, check the error messages for conflicts with the existing schema. - -3. **Insufficient permissions** -- make sure the database user has permissions to create schemas, tables, and indexes. - -4. **"Admin already exists" error** -- if you get this error when trying to register an admin user, follow these steps: - - a. Connect to the PostgreSQL container: - ```bash - # Find the container - docker ps - # Example: oxicloud-postgres-1 - docker exec -it oxicloud-postgres-1 bash - ``` - - b. Connect to the database: - ```bash - psql -U postgres -d oxicloud - ``` - - c. Set the schema and delete the existing admin user: - ```sql - SET search_path TO auth; - DELETE FROM auth.users WHERE username = 'admin'; - ``` - - d. Verify the deletion: - ```sql - SELECT username, email, role FROM auth.users; - ``` - - e. Exit PostgreSQL: - ```sql - \q - exit - ``` - - f. You can now register a new admin user through the OxiCloud interface. - - Alternatively, use the provided script: - ```bash - cat scripts/reset_admin.sql | docker exec -i oxicloud-postgres-1 psql -U postgres -d oxicloud - ``` - -## Benefits of Migration-Based Approach - -- **Separation of concerns** -- migrations live separately from application code. -- **Automation** -- simplifies deployment automation and CI/CD. -- **Change history** -- provides a clear history of schema evolution. -- **Collaboration** -- lets multiple developers contribute schema changes in an orderly way. -- **Multiple environments** -- guarantees identical database structures across dev, test, and production. diff --git a/src/bin/migrate.rs b/src/bin/migrate.rs deleted file mode 100644 index 6f10b1ec..00000000 --- a/src/bin/migrate.rs +++ /dev/null @@ -1,49 +0,0 @@ -use sqlx::postgres::PgPoolOptions; -use std::env; -use std::path::Path; -use std::time::Duration; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Configure logging - tracing_subscriber::fmt::init(); - - // Load environment variables (.env.local first, then .env) - if let Ok(path) = env::var("DOTENV_PATH") { - dotenvy::from_path(Path::new(&path)).ok(); - } else { - dotenvy::from_filename(".env.local").ok(); - dotenvy::dotenv().ok(); - } - - // Get DATABASE_URL from environment variables - let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be configured"); - - println!("Connecting to the database..."); - - // Create connection pool - let pool = PgPoolOptions::new() - .max_connections(5) - .acquire_timeout(Duration::from_secs(10)) - .connect(&database_url) - .await?; - - // Run migrations - println!("Running migrations..."); - - // Get the directory from an environment variable or use a default value - let migrations_dir = env::var("MIGRATIONS_DIR").unwrap_or_else(|_| "./migrations".to_string()); - println!("Migrations directory: {}", migrations_dir); - - // Create a migrator - let migrator = sqlx::migrate::Migrator::new(Path::new(&migrations_dir)) - .await - .expect("Could not create the migrator"); - - // Run all pending migrations - migrator.run(&pool).await?; - - println!("Migrations applied successfully"); - - Ok(()) -}