chore: remove redundant migrations directory and migrate binary
All content from 003_add_device_codes.sql and 004_add_trigram_indexes.sql was already absorbed into db/schema.sql (the single source of truth). The migrations had additional problems: - Broken numbering (started at 003, missing 001/002) - 004 used CREATE INDEX CONCURRENTLY which fails inside sqlx transactions - No production flow ever invoked the migrate binary Removed: db/migrations/, src/bin/migrate.rs, migrations Cargo feature, [[bin]] migrate target, and doc/database-migrations.md.
This commit is contained in:
Dionisio
@@ -59,12 +59,6 @@ socket2 = { version = "0.6.2", features = ["all"] }
|
||||
[features]
|
||||
default = []
|
||||
test_utils = ["mockall"]
|
||||
migrations = ["sqlx/migrate"]
|
||||
|
||||
[[bin]]
|
||||
name = "migrate"
|
||||
path = "src/bin/migrate.rs"
|
||||
required-features = ["migrations"]
|
||||
|
||||
[profile.release]
|
||||
lto = "fat"
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
-- ============================================================
|
||||
-- Migration 003: OAuth 2.0 Device Authorization Grant (RFC 8628)
|
||||
-- ============================================================
|
||||
-- Adds the device_codes table to support the Device Authorization
|
||||
-- Grant flow for WebDAV/CalDAV/CardDAV client authentication.
|
||||
--
|
||||
-- Flow:
|
||||
-- 1. Client POSTs to /api/auth/device/authorize → receives device_code + user_code
|
||||
-- 2. User opens verification_uri in browser, authenticates, enters user_code
|
||||
-- 3. Client polls /api/auth/device/token with device_code
|
||||
-- 4. Once approved, client receives access_token + refresh_token
|
||||
-- ============================================================
|
||||
|
||||
-- Device code status enum
|
||||
DO $BODY$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_type t
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
|
||||
WHERE t.typname = 'device_code_status' AND n.nspname = 'auth'
|
||||
) THEN
|
||||
CREATE TYPE auth.device_code_status AS ENUM (
|
||||
'pending', -- Waiting for user to authorize
|
||||
'authorized', -- User approved, tokens ready for polling client
|
||||
'denied', -- User denied the request
|
||||
'expired' -- TTL exceeded without user action
|
||||
);
|
||||
END IF;
|
||||
END $BODY$;
|
||||
|
||||
-- Device authorization codes table
|
||||
CREATE TABLE IF NOT EXISTS auth.device_codes (
|
||||
-- Unique row ID
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
|
||||
-- RFC 8628 §3.2: device_code — long opaque token sent to the client for polling
|
||||
device_code VARCHAR(128) UNIQUE NOT NULL,
|
||||
|
||||
-- RFC 8628 §3.2: user_code — short human-readable code shown on the client
|
||||
-- and entered by the user on the verification page (e.g. "ABCD-1234")
|
||||
user_code VARCHAR(16) UNIQUE NOT NULL,
|
||||
|
||||
-- Name/description of the client requesting access (shown to user)
|
||||
client_name VARCHAR(255) NOT NULL DEFAULT 'Unknown Client',
|
||||
|
||||
-- Comma-separated scopes requested (e.g. "webdav,caldav,carddav")
|
||||
scopes VARCHAR(512) NOT NULL DEFAULT 'webdav,caldav,carddav',
|
||||
|
||||
-- Current status of the device flow
|
||||
status auth.device_code_status NOT NULL DEFAULT 'pending',
|
||||
|
||||
-- User who authorized the request (NULL until status = 'authorized')
|
||||
user_id VARCHAR(36) REFERENCES auth.users(id) ON DELETE CASCADE,
|
||||
|
||||
-- Tokens generated after authorization (NULL until status = 'authorized')
|
||||
-- Stored encrypted/hashed depending on sensitivity
|
||||
access_token TEXT,
|
||||
refresh_token TEXT,
|
||||
|
||||
-- RFC 8628 §3.2: verification_uri — full URL the user must visit
|
||||
verification_uri TEXT NOT NULL,
|
||||
|
||||
-- RFC 8628 §3.2: verification_uri_complete — URL with user_code pre-filled
|
||||
verification_uri_complete TEXT,
|
||||
|
||||
-- RFC 8628 §3.2: expires_in — encoded as an absolute timestamp
|
||||
expires_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
|
||||
-- RFC 8628 §3.2: interval — minimum polling interval in seconds
|
||||
poll_interval_secs INTEGER NOT NULL DEFAULT 5,
|
||||
|
||||
-- Last time the client polled (for slow_down enforcement)
|
||||
last_poll_at TIMESTAMP WITH TIME ZONE,
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
authorized_at TIMESTAMP WITH TIME ZONE
|
||||
);
|
||||
|
||||
-- Index for client polling by device_code (hot path)
|
||||
CREATE INDEX IF NOT EXISTS idx_device_codes_device_code
|
||||
ON auth.device_codes(device_code);
|
||||
|
||||
-- Index for user verification page lookup by user_code
|
||||
CREATE INDEX IF NOT EXISTS idx_device_codes_user_code
|
||||
ON auth.device_codes(user_code)
|
||||
WHERE status = 'pending';
|
||||
|
||||
-- Index for cleanup of expired entries
|
||||
CREATE INDEX IF NOT EXISTS idx_device_codes_expires_at
|
||||
ON auth.device_codes(expires_at)
|
||||
WHERE status = 'pending';
|
||||
|
||||
-- Index for user's authorized devices
|
||||
CREATE INDEX IF NOT EXISTS idx_device_codes_user_id
|
||||
ON auth.device_codes(user_id)
|
||||
WHERE status = 'authorized';
|
||||
|
||||
COMMENT ON TABLE auth.device_codes IS 'OAuth 2.0 Device Authorization Grant (RFC 8628) codes for DAV client authentication';
|
||||
@@ -1,43 +0,0 @@
|
||||
-- Migration 004: Add GIN trigram indexes for ILIKE/LIKE substring search
|
||||
--
|
||||
-- Eliminates full table scans on text search queries by enabling
|
||||
-- PostgreSQL's pg_trgm extension and creating GIN indexes with
|
||||
-- gin_trgm_ops on all columns used in ILIKE / LIKE '%text%' patterns.
|
||||
--
|
||||
-- CONCURRENTLY is used so that no table locks are held during index
|
||||
-- creation — zero downtime for existing installations.
|
||||
--
|
||||
-- NOTE: CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
|
||||
-- If using sqlx migrate, run this file manually:
|
||||
-- psql -f db/migrations/004_add_trigram_indexes.sql
|
||||
|
||||
-- 0. Enable the pg_trgm extension (idempotent)
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
|
||||
-- 1. Contacts — search_contacts(), get_contacts_by_email()
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_full_name_trgm
|
||||
ON carddav.contacts USING gin (full_name gin_trgm_ops);
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_first_name_trgm
|
||||
ON carddav.contacts USING gin (first_name gin_trgm_ops);
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_last_name_trgm
|
||||
ON carddav.contacts USING gin (last_name gin_trgm_ops);
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_nickname_trgm
|
||||
ON carddav.contacts USING gin (nickname gin_trgm_ops);
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_organization_trgm
|
||||
ON carddav.contacts USING gin (organization gin_trgm_ops);
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_email_text_trgm
|
||||
ON carddav.contacts USING gin ((email::text) gin_trgm_ops);
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_phone_text_trgm
|
||||
ON carddav.contacts USING gin ((phone::text) gin_trgm_ops);
|
||||
|
||||
-- 2. Calendar events — find_events_by_summary()
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_calendar_events_summary_trgm
|
||||
ON caldav.calendar_events USING gin (summary gin_trgm_ops);
|
||||
|
||||
-- 3. Files — search_files_paginated(), search_files_in_subtree(), suggest_files_by_name()
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_files_name_trgm
|
||||
ON storage.files USING gin (name gin_trgm_ops);
|
||||
|
||||
-- 4. Folders — search_folders(), list_descendant_folders(), suggest_folders_by_name()
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_folders_name_trgm
|
||||
ON storage.folders USING gin (name gin_trgm_ops);
|
||||
@@ -1,177 +0,0 @@
|
||||
# 18 - Database Migrations
|
||||
|
||||
OxiCloud uses versioned SQL files to manage database schema changes. The migration system ensures changes are versioned, trackable, consistently applied across environments, reproducible, and independent of application code.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
OxiCloud/
|
||||
├── db/
|
||||
│ └── schema.sql # Main database schema
|
||||
├── src/
|
||||
├── bin/
|
||||
│ └── migrate.rs # CLI tool for running migrations
|
||||
├── common/
|
||||
│ └── db.rs # Database connection with schema verification
|
||||
```
|
||||
|
||||
> The schema is currently applied from `db/schema.sql` at application startup (when it detects the `auth` tables don't exist). The `migrations/` directory doesn't exist yet, but `src/bin/migrate.rs` is ready to use sqlx migrations once the `migrations` feature is enabled.
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
Migration files follow this format: `YYYYMMDDHHMMSS_brief_description.sql`
|
||||
|
||||
- `YYYYMMDDHHMMSS` -- timestamp that guarantees correct ordering (year, month, day, hour, minute, second)
|
||||
- `brief_description` -- short description of the migration purpose
|
||||
- `.sql` -- SQL file extension
|
||||
|
||||
## Running Migrations
|
||||
|
||||
Migrations run via a dedicated CLI tool:
|
||||
|
||||
```bash
|
||||
cargo run --bin migrate --features migrations
|
||||
```
|
||||
|
||||
This command:
|
||||
1. Connects to the database configured in the environment
|
||||
2. Looks for migrations in the `/migrations/` directory
|
||||
3. Compares applied migrations against available ones
|
||||
4. Sequentially executes pending migrations
|
||||
5. Records applied migrations in a control table
|
||||
|
||||
## Creating New Migrations
|
||||
|
||||
To create a new migration:
|
||||
|
||||
1. Create a new file in `migrations/` following the naming convention
|
||||
2. Define the SQL changes in the file
|
||||
3. Make sure the changes are compatible with the current schema version
|
||||
4. Run the migrations
|
||||
|
||||
Example migration structure:
|
||||
|
||||
```sql
|
||||
-- Migración: Añadir tabla de etiquetas
|
||||
-- Descripción: Crea la tabla para almacenar etiquetas de archivos y sus relaciones
|
||||
|
||||
-- Crear tabla de etiquetas
|
||||
CREATE TABLE IF NOT EXISTS auth.tags (
|
||||
id SERIAL PRIMARY KEY,
|
||||
user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
color TEXT NOT NULL DEFAULT '#3498db',
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(user_id, name)
|
||||
);
|
||||
|
||||
-- Crear índices
|
||||
CREATE INDEX IF NOT EXISTS idx_tags_user_id ON auth.tags(user_id);
|
||||
|
||||
-- Tabla de relación entre archivos y etiquetas
|
||||
CREATE TABLE IF NOT EXISTS auth.file_tags (
|
||||
id SERIAL PRIMARY KEY,
|
||||
tag_id INTEGER NOT NULL REFERENCES auth.tags(id) ON DELETE CASCADE,
|
||||
file_id TEXT NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(tag_id, file_id)
|
||||
);
|
||||
|
||||
-- Comentarios de documentación
|
||||
COMMENT ON TABLE auth.tags IS 'Almacena etiquetas definidas por usuarios';
|
||||
COMMENT ON TABLE auth.file_tags IS 'Relación muchos-a-muchos entre archivos y etiquetas';
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Incremental migrations** -- each migration should represent one atomic, coherent change.
|
||||
|
||||
2. **Idempotent migrations** -- use commands that can run multiple times without errors (e.g., `CREATE TABLE IF NOT EXISTS`).
|
||||
|
||||
3. **Forward-only migrations** -- design migrations to move forward, not roll back. If you need to undo a change, create a new migration.
|
||||
|
||||
4. **Forward compatibility** -- migrations must be compatible with both the existing code and the code about to be deployed.
|
||||
|
||||
5. **Test before deploying** -- test migrations in a production-like environment before applying them.
|
||||
|
||||
6. **Documentation** -- document the purpose and key changes of each migration with comments inside the SQL file.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Checking Migration State
|
||||
|
||||
OxiCloud includes startup-time detection to verify which migrations have been applied:
|
||||
|
||||
```rust
|
||||
// Desde src/common/db.rs
|
||||
let migration_check = sqlx::query("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'auth' AND tablename = 'users')")
|
||||
.fetch_one(&pool)
|
||||
.await;
|
||||
|
||||
match migration_check {
|
||||
Ok(row) => {
|
||||
let tables_exist: bool = row.get(0);
|
||||
if !tables_exist {
|
||||
tracing::warn!("Las tablas de la base de datos no existen. Por favor, ejecuta las migraciones con: cargo run --bin migrate --features migrations");
|
||||
}
|
||||
},
|
||||
Err(_) => {
|
||||
tracing::warn!("No se pudo verificar el estado de las migraciones. Por favor, ejecuta las migraciones con: cargo run --bin migrate --features migrations");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Database connection error** -- verify the connection URL in the **DATABASE_URL** environment variable.
|
||||
|
||||
2. **Migration conflicts** -- if a migration fails, check the error messages for conflicts with the existing schema.
|
||||
|
||||
3. **Insufficient permissions** -- make sure the database user has permissions to create schemas, tables, and indexes.
|
||||
|
||||
4. **"Admin already exists" error** -- if you get this error when trying to register an admin user, follow these steps:
|
||||
|
||||
a. Connect to the PostgreSQL container:
|
||||
```bash
|
||||
# Find the container
|
||||
docker ps
|
||||
# Example: oxicloud-postgres-1
|
||||
docker exec -it oxicloud-postgres-1 bash
|
||||
```
|
||||
|
||||
b. Connect to the database:
|
||||
```bash
|
||||
psql -U postgres -d oxicloud
|
||||
```
|
||||
|
||||
c. Set the schema and delete the existing admin user:
|
||||
```sql
|
||||
SET search_path TO auth;
|
||||
DELETE FROM auth.users WHERE username = 'admin';
|
||||
```
|
||||
|
||||
d. Verify the deletion:
|
||||
```sql
|
||||
SELECT username, email, role FROM auth.users;
|
||||
```
|
||||
|
||||
e. Exit PostgreSQL:
|
||||
```sql
|
||||
\q
|
||||
exit
|
||||
```
|
||||
|
||||
f. You can now register a new admin user through the OxiCloud interface.
|
||||
|
||||
Alternatively, use the provided script:
|
||||
```bash
|
||||
cat scripts/reset_admin.sql | docker exec -i oxicloud-postgres-1 psql -U postgres -d oxicloud
|
||||
```
|
||||
|
||||
## Benefits of Migration-Based Approach
|
||||
|
||||
- **Separation of concerns** -- migrations live separately from application code.
|
||||
- **Automation** -- simplifies deployment automation and CI/CD.
|
||||
- **Change history** -- provides a clear history of schema evolution.
|
||||
- **Collaboration** -- lets multiple developers contribute schema changes in an orderly way.
|
||||
- **Multiple environments** -- guarantees identical database structures across dev, test, and production.
|
||||
@@ -1,49 +0,0 @@
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use std::env;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Configure logging
|
||||
tracing_subscriber::fmt::init();
|
||||
|
||||
// Load environment variables (.env.local first, then .env)
|
||||
if let Ok(path) = env::var("DOTENV_PATH") {
|
||||
dotenvy::from_path(Path::new(&path)).ok();
|
||||
} else {
|
||||
dotenvy::from_filename(".env.local").ok();
|
||||
dotenvy::dotenv().ok();
|
||||
}
|
||||
|
||||
// Get DATABASE_URL from environment variables
|
||||
let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be configured");
|
||||
|
||||
println!("Connecting to the database...");
|
||||
|
||||
// Create connection pool
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.acquire_timeout(Duration::from_secs(10))
|
||||
.connect(&database_url)
|
||||
.await?;
|
||||
|
||||
// Run migrations
|
||||
println!("Running migrations...");
|
||||
|
||||
// Get the directory from an environment variable or use a default value
|
||||
let migrations_dir = env::var("MIGRATIONS_DIR").unwrap_or_else(|_| "./migrations".to_string());
|
||||
println!("Migrations directory: {}", migrations_dir);
|
||||
|
||||
// Create a migrator
|
||||
let migrator = sqlx::migrate::Migrator::new(Path::new(&migrations_dir))
|
||||
.await
|
||||
.expect("Could not create the migrator");
|
||||
|
||||
// Run all pending migrations
|
||||
migrator.run(&pool).await?;
|
||||
|
||||
println!("Migrations applied successfully");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user