Spaces:
Running
Running
Commit
·
e2eee76
1
Parent(s):
68b189e
Enhance requirements and refactor migration script: add new dependencies for SQLAlchemy and Alembic, improve MongoDB migration logic with collection creation and indexing, and update logging for better traceability.
Browse files- app/api/audiobook_routes.py +9 -12
- requirements.txt +4 -1
- scripts/migrate.py +87 -20
app/api/audiobook_routes.py
CHANGED
@@ -8,22 +8,19 @@ import shutil
|
|
8 |
import logging
|
9 |
from datetime import datetime
|
10 |
from typing import Optional, List
|
11 |
-
from fastapi import APIRouter, Request, HTTPException, BackgroundTasks, UploadFile, File
|
12 |
-
from fastapi.responses import FileResponse
|
13 |
-
from sqlalchemy.orm import Session
|
14 |
-
from app.db_models.database import Audiobook, AudiobookStatus, AudiobookChunk, TextChunk
|
15 |
-
from app.services.storage import storage
|
16 |
-
from app.db import get_db, AUDIOBOOKS_COLLECTION
|
17 |
-
from app.config import AUDIO_DIR, TEXT_DIR, TEMP_DIR
|
18 |
from pydantic import BaseModel
|
19 |
-
import torchaudio
|
20 |
-
import json
|
21 |
-
import shutil
|
22 |
from motor.motor_asyncio import AsyncIOMotorDatabase
|
23 |
|
24 |
-
|
|
|
|
|
|
|
|
|
25 |
logger = logging.getLogger(__name__)
|
26 |
-
|
|
|
27 |
|
28 |
class AudiobookBase(BaseModel):
|
29 |
title: str
|
|
|
8 |
import logging
|
9 |
from datetime import datetime
|
10 |
from typing import Optional, List
|
11 |
+
from fastapi import APIRouter, Request, HTTPException, BackgroundTasks, UploadFile, File
|
12 |
+
from fastapi.responses import FileResponse
|
|
|
|
|
|
|
|
|
|
|
13 |
from pydantic import BaseModel
|
|
|
|
|
|
|
14 |
from motor.motor_asyncio import AsyncIOMotorDatabase
|
15 |
|
16 |
+
from app.db import get_db, AUDIOBOOKS_COLLECTION
|
17 |
+
from app.config import AUDIO_DIR, TEXT_DIR, TEMP_DIR
|
18 |
+
|
19 |
+
# Configure logging
|
20 |
+
logging.basicConfig(level=logging.INFO)
|
21 |
logger = logging.getLogger(__name__)
|
22 |
+
|
23 |
+
router = APIRouter()
|
24 |
|
25 |
class AudiobookBase(BaseModel):
|
26 |
title: str
|
requirements.txt
CHANGED
@@ -24,4 +24,7 @@ ffmpeg-python>=0.2.0
|
|
24 |
accelerate>=0.20.0
|
25 |
pymongo>=4.6.1
|
26 |
motor>=3.3.2
|
27 |
-
python-dotenv>=1.0.1
|
|
|
|
|
|
|
|
24 |
accelerate>=0.20.0
|
25 |
pymongo>=4.6.1
|
26 |
motor>=3.3.2
|
27 |
+
python-dotenv>=1.0.1
|
28 |
+
sqlalchemy>=2.0.0
|
29 |
+
alembic>=1.13.0
|
30 |
+
psycopg2-binary>=2.9.9
|
scripts/migrate.py
CHANGED
@@ -1,42 +1,109 @@
|
|
1 |
#!/usr/bin/env python3
|
2 |
"""
|
3 |
-
|
4 |
"""
|
5 |
import os
|
6 |
import sys
|
7 |
import logging
|
8 |
-
from
|
|
|
|
|
9 |
|
10 |
-
#
|
|
|
|
|
|
|
11 |
logging.basicConfig(
|
12 |
level=logging.INFO,
|
13 |
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
14 |
)
|
15 |
logger = logging.getLogger(__name__)
|
16 |
|
17 |
-
|
|
|
|
|
|
|
18 |
"""Run database migrations."""
|
19 |
try:
|
20 |
-
|
21 |
-
|
22 |
-
|
|
|
|
|
23 |
|
24 |
-
#
|
25 |
-
|
26 |
|
27 |
-
#
|
28 |
-
|
29 |
-
|
|
|
|
|
|
|
|
|
|
|
30 |
|
31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
32 |
|
33 |
-
|
34 |
-
|
35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
36 |
except Exception as e:
|
37 |
-
logger.error(f"
|
38 |
-
|
39 |
|
40 |
if __name__ == "__main__":
|
41 |
-
|
42 |
-
sys.exit(0 if success else 1)
|
|
|
1 |
#!/usr/bin/env python3
|
2 |
"""
|
3 |
+
MongoDB database migration script.
|
4 |
"""
|
5 |
import os
|
6 |
import sys
|
7 |
import logging
|
8 |
+
from motor.motor_asyncio import AsyncIOMotorClient
|
9 |
+
import asyncio
|
10 |
+
from datetime import datetime
|
11 |
|
12 |
+
# Add the app directory to the Python path
|
13 |
+
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
14 |
+
|
15 |
+
# Configure logging
|
16 |
logging.basicConfig(
|
17 |
level=logging.INFO,
|
18 |
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
19 |
)
|
20 |
logger = logging.getLogger(__name__)
|
21 |
|
22 |
+
# Import app configuration after setting up path
|
23 |
+
from app.db import MONGO_URI, DB_NAME, AUDIOBOOKS_COLLECTION, VOICES_COLLECTION, AUDIO_CACHE_COLLECTION
|
24 |
+
|
25 |
+
async def run_migrations():
|
26 |
"""Run database migrations."""
|
27 |
try:
|
28 |
+
logger.info(f"Starting MongoDB migrations for database: {DB_NAME}")
|
29 |
+
|
30 |
+
# Connect to MongoDB
|
31 |
+
client = AsyncIOMotorClient(MONGO_URI)
|
32 |
+
db = client[DB_NAME]
|
33 |
|
34 |
+
# Create collections if they don't exist
|
35 |
+
collections = await db.list_collection_names()
|
36 |
|
37 |
+
# Audiobooks collection
|
38 |
+
if AUDIOBOOKS_COLLECTION not in collections:
|
39 |
+
logger.info(f"Creating collection: {AUDIOBOOKS_COLLECTION}")
|
40 |
+
await db.create_collection(AUDIOBOOKS_COLLECTION)
|
41 |
+
# Create indexes
|
42 |
+
await db[AUDIOBOOKS_COLLECTION].create_index("id", unique=True)
|
43 |
+
await db[AUDIOBOOKS_COLLECTION].create_index("created_at")
|
44 |
+
await db[AUDIOBOOKS_COLLECTION].create_index("status")
|
45 |
|
46 |
+
# Voices collection
|
47 |
+
if VOICES_COLLECTION not in collections:
|
48 |
+
logger.info(f"Creating collection: {VOICES_COLLECTION}")
|
49 |
+
await db.create_collection(VOICES_COLLECTION)
|
50 |
+
# Create indexes
|
51 |
+
await db[VOICES_COLLECTION].create_index("id", unique=True)
|
52 |
+
await db[VOICES_COLLECTION].create_index("name")
|
53 |
+
await db[VOICES_COLLECTION].create_index("type")
|
54 |
|
55 |
+
# Audio cache collection
|
56 |
+
if AUDIO_CACHE_COLLECTION not in collections:
|
57 |
+
logger.info(f"Creating collection: {AUDIO_CACHE_COLLECTION}")
|
58 |
+
await db.create_collection(AUDIO_CACHE_COLLECTION)
|
59 |
+
# Create indexes
|
60 |
+
await db[AUDIO_CACHE_COLLECTION].create_index("hash", unique=True)
|
61 |
+
await db[AUDIO_CACHE_COLLECTION].create_index("created_at")
|
62 |
+
|
63 |
+
# Add any future migrations here
|
64 |
+
# Example:
|
65 |
+
# await migrate_v1_to_v2(db)
|
66 |
+
|
67 |
+
logger.info("Database migrations completed successfully")
|
68 |
+
|
69 |
+
except Exception as e:
|
70 |
+
logger.error(f"Error during migrations: {str(e)}")
|
71 |
+
raise
|
72 |
+
finally:
|
73 |
+
# Close the client connection
|
74 |
+
client.close()
|
75 |
+
|
76 |
+
async def migrate_v1_to_v2(db):
|
77 |
+
"""
|
78 |
+
Example migration function for future use.
|
79 |
+
Updates documents from v1 to v2 schema.
|
80 |
+
"""
|
81 |
+
try:
|
82 |
+
# Example: Add a new field to all documents
|
83 |
+
result = await db[AUDIOBOOKS_COLLECTION].update_many(
|
84 |
+
{"version": {"$exists": False}},
|
85 |
+
{
|
86 |
+
"$set": {
|
87 |
+
"version": "2.0",
|
88 |
+
"updated_at": datetime.utcnow()
|
89 |
+
}
|
90 |
+
}
|
91 |
+
)
|
92 |
+
logger.info(f"Migrated {result.modified_count} documents to v2")
|
93 |
+
except Exception as e:
|
94 |
+
logger.error(f"Error in v1 to v2 migration: {str(e)}")
|
95 |
+
raise
|
96 |
+
|
97 |
+
def main():
|
98 |
+
"""Main entry point for migrations."""
|
99 |
+
try:
|
100 |
+
# Run migrations
|
101 |
+
asyncio.run(run_migrations())
|
102 |
+
logger.info("Migrations completed successfully")
|
103 |
+
sys.exit(0)
|
104 |
except Exception as e:
|
105 |
+
logger.error(f"Migration failed: {str(e)}")
|
106 |
+
sys.exit(1)
|
107 |
|
108 |
if __name__ == "__main__":
|
109 |
+
main()
|
|