VeuReu commited on
Commit
46cf14b
·
verified ·
1 Parent(s): 8134888

Upload 7 files

Browse files
storage/common.py CHANGED
@@ -1,14 +1,14 @@
1
- from fastapi import HTTPException
2
- import os
3
-
4
- def validate_token(token: str):
5
- """
6
- Validate the provided token against the HF_TOKEN environment variable.
7
- Raises an HTTPException if validation fails.
8
- """
9
- HF_TOKEN = os.getenv("HF_TOKEN")
10
- if HF_TOKEN is None:
11
- raise RuntimeError("HF_TOKEN environment variable is not set on the server.")
12
-
13
- if token != HF_TOKEN:
14
  raise HTTPException(status_code=401, detail="Invalid token")
 
1
+ from fastapi import HTTPException
2
+ import os
3
+
4
+ def validate_token(token: str):
5
+ """
6
+ Validate the provided token against the HF_TOKEN environment variable.
7
+ Raises an HTTPException if validation fails.
8
+ """
9
+ HF_TOKEN = os.getenv("HF_TOKEN")
10
+ if HF_TOKEN is None:
11
+ raise RuntimeError("HF_TOKEN environment variable is not set on the server.")
12
+
13
+ if token != HF_TOKEN:
14
  raise HTTPException(status_code=401, detail="Invalid token")
storage/data_routers.py CHANGED
@@ -1,194 +1,194 @@
1
- import os
2
- import io
3
- import json
4
- import shutil
5
-
6
- import sqlite3
7
-
8
- from pathlib import Path
9
-
10
- from fastapi import APIRouter, UploadFile, File, Query, HTTPException
11
- from fastapi.responses import FileResponse, JSONResponse
12
-
13
-
14
- from storage.files.file_manager import FileManager
15
- from storage.common import validate_token
16
-
17
- router = APIRouter(prefix="/data", tags=["Data Manager"])
18
- DATA_ROOT = Path("/data")
19
- HF_TOKEN = os.getenv("HF_TOKEN")
20
-
21
-
22
- @router.get("/data_tree", tags=["Data Manager"])
23
- def get_data_tree(
24
- token: str = Query(..., description="Token required for authorization")
25
- ):
26
- """
27
- Return a formatted tree of folders and files under /data.
28
-
29
- Behavior:
30
- - Validate token.
31
- - Walk the /data directory and build a recursive tree.
32
- - Each entry includes: name, type (file/directory), and children if directory.
33
- """
34
-
35
- validate_token(token)
36
-
37
- def build_tree(path: Path):
38
- """
39
- Build a tree representation of directories and files.
40
- Prevents errors by checking if the path is a directory before iterating.
41
- """
42
-
43
- if not path.exists():
44
- return {"name": path.name, "type": "missing"}
45
-
46
- # Si es un fichero → devolvemos un nodo simple
47
- if path.is_file():
48
- return {
49
- "name": path.name,
50
- "type": "file"
51
- }
52
-
53
- # Si es un directorio → construimos sus hijos
54
- children = []
55
- try:
56
- entries = sorted(path.iterdir(), key=lambda p: p.name.lower())
57
- except Exception:
58
- # Si por cualquier razón no podemos listar, lo tratamos como file
59
- return {
60
- "name": path.name,
61
- "type": "file"
62
- }
63
-
64
- for child in entries:
65
- children.append(build_tree(child))
66
-
67
- return {
68
- "name": path.name,
69
- "type": "directory",
70
- "children": children
71
- }
72
-
73
- if not DATA_ROOT.exists():
74
- return {"error": "/data does not exist"}
75
-
76
- return build_tree(DATA_ROOT)
77
-
78
- @router.post("/create_data_item", tags=["Data Manager"])
79
- def create_data_item(
80
- path: str = Query(..., description="Full path starting with /data/"),
81
- item_type: str = Query(..., description="directory or file"),
82
- token: str = Query(..., description="Token required for authorization")
83
- ):
84
- """
85
- Create a directory or file under /data.
86
-
87
- Restrictions:
88
- - Path must start with /data/.
89
- - Writing to /data/db or /data/media (or any of their subpaths) is forbidden.
90
- - item_type must be 'directory' or 'file'.
91
-
92
- Behavior:
93
- - Validate token.
94
- - Check path validity and protection.
95
- - Create directory or empty file.
96
- - Raise error if path already exists.
97
- """
98
-
99
- validate_token(token)
100
-
101
- target = Path(path)
102
-
103
- # Validación básica
104
- if not path.startswith("/data/"):
105
- raise HTTPException(status_code=400, detail="Path must start with /data/")
106
-
107
- if item_type not in ("directory", "file"):
108
- raise HTTPException(status_code=400, detail="item_type must be 'directory' or 'file'")
109
-
110
- # Protección de carpetas sensibles
111
- protected = ["/data/db", "/data/media"]
112
-
113
- for p in protected:
114
- if path == p or path.startswith(p + "/"):
115
- raise HTTPException(
116
- status_code=403,
117
- detail=f"Access to protected path '{p}' is not allowed"
118
- )
119
-
120
- # No permitir sobrescritura
121
- if target.exists():
122
- raise HTTPException(status_code=409, detail="Path already exists")
123
-
124
- try:
125
- if item_type == "directory":
126
- target.mkdir(parents=True, exist_ok=False)
127
- else:
128
- target.parent.mkdir(parents=True, exist_ok=True)
129
- with open(target, "wb") as f:
130
- f.write(b"")
131
- except Exception as exc:
132
- raise HTTPException(status_code=500, detail=f"Failed to create item: {exc}")
133
-
134
- return {
135
- "status": "ok",
136
- "created": str(target),
137
- "type": item_type
138
- }
139
-
140
- @router.delete("/delete_path", tags=["Data Manager"])
141
- def delete_path(
142
- target: str = Query(..., description="Ruta absoluta dentro de /data a borrar"),
143
- token: str = Query(..., description="Token requerido para autorización")
144
- ):
145
- """
146
- Delete a file or directory recursively inside /data, except protected folders.
147
-
148
- Behavior:
149
- - Validate token.
150
- - Validate path starts with /data/.
151
- - Deny deletion of /data/db and /data/media (and anything inside them).
152
- - If target is a file → delete file.
153
- - If target is a directory → delete recursively.
154
- """
155
-
156
- validate_token(token)
157
-
158
- # Normalizar ruta
159
- path = Path(target).resolve()
160
-
161
- # Debe estar dentro de /data/
162
- if not str(path).startswith(str(DATA_ROOT)):
163
- raise HTTPException(status_code=400, detail="Path must be inside /data/")
164
-
165
- # Carpetas protegidas
166
- protected = [
167
- DATA_ROOT / "db",
168
- DATA_ROOT / "media",
169
- ]
170
-
171
- for p in protected:
172
- if path == p or str(path).startswith(str(p) + "/"):
173
- raise HTTPException(
174
- status_code=403,
175
- detail=f"Deletion forbidden: {p} is protected"
176
- )
177
-
178
- # Verificar existencia
179
- if not path.exists():
180
- raise HTTPException(status_code=404, detail="Target path does not exist")
181
-
182
- try:
183
- if path.is_file():
184
- path.unlink()
185
- else:
186
- shutil.rmtree(path)
187
-
188
- except Exception as exc:
189
- raise HTTPException(status_code=500, detail=f"Failed to delete: {exc}")
190
-
191
- return {
192
- "status": "ok",
193
- "deleted": str(path)
194
  }
 
1
+ import os
2
+ import io
3
+ import json
4
+ import shutil
5
+
6
+ import sqlite3
7
+
8
+ from pathlib import Path
9
+
10
+ from fastapi import APIRouter, UploadFile, File, Query, HTTPException
11
+ from fastapi.responses import FileResponse, JSONResponse
12
+
13
+
14
+ from storage.files.file_manager import FileManager
15
+ from storage.common import validate_token
16
+
17
+ router = APIRouter(prefix="/data", tags=["Data Manager"])
18
+ DATA_ROOT = Path("/data")
19
+ HF_TOKEN = os.getenv("HF_TOKEN")
20
+
21
+
22
+ @router.get("/data_tree", tags=["Data Manager"])
23
+ def get_data_tree(
24
+ token: str = Query(..., description="Token required for authorization")
25
+ ):
26
+ """
27
+ Return a formatted tree of folders and files under /data.
28
+
29
+ Behavior:
30
+ - Validate token.
31
+ - Walk the /data directory and build a recursive tree.
32
+ - Each entry includes: name, type (file/directory), and children if directory.
33
+ """
34
+
35
+ validate_token(token)
36
+
37
+ def build_tree(path: Path):
38
+ """
39
+ Build a tree representation of directories and files.
40
+ Prevents errors by checking if the path is a directory before iterating.
41
+ """
42
+
43
+ if not path.exists():
44
+ return {"name": path.name, "type": "missing"}
45
+
46
+ # Si es un fichero → devolvemos un nodo simple
47
+ if path.is_file():
48
+ return {
49
+ "name": path.name,
50
+ "type": "file"
51
+ }
52
+
53
+ # Si es un directorio → construimos sus hijos
54
+ children = []
55
+ try:
56
+ entries = sorted(path.iterdir(), key=lambda p: p.name.lower())
57
+ except Exception:
58
+ # Si por cualquier razón no podemos listar, lo tratamos como file
59
+ return {
60
+ "name": path.name,
61
+ "type": "file"
62
+ }
63
+
64
+ for child in entries:
65
+ children.append(build_tree(child))
66
+
67
+ return {
68
+ "name": path.name,
69
+ "type": "directory",
70
+ "children": children
71
+ }
72
+
73
+ if not DATA_ROOT.exists():
74
+ return {"error": "/data does not exist"}
75
+
76
+ return build_tree(DATA_ROOT)
77
+
78
+ @router.post("/create_data_item", tags=["Data Manager"])
79
+ def create_data_item(
80
+ path: str = Query(..., description="Full path starting with /data/"),
81
+ item_type: str = Query(..., description="directory or file"),
82
+ token: str = Query(..., description="Token required for authorization")
83
+ ):
84
+ """
85
+ Create a directory or file under /data.
86
+
87
+ Restrictions:
88
+ - Path must start with /data/.
89
+ - Writing to /data/db or /data/media (or any of their subpaths) is forbidden.
90
+ - item_type must be 'directory' or 'file'.
91
+
92
+ Behavior:
93
+ - Validate token.
94
+ - Check path validity and protection.
95
+ - Create directory or empty file.
96
+ - Raise error if path already exists.
97
+ """
98
+
99
+ validate_token(token)
100
+
101
+ target = Path(path)
102
+
103
+ # Validación básica
104
+ if not path.startswith("/data/"):
105
+ raise HTTPException(status_code=400, detail="Path must start with /data/")
106
+
107
+ if item_type not in ("directory", "file"):
108
+ raise HTTPException(status_code=400, detail="item_type must be 'directory' or 'file'")
109
+
110
+ # Protección de carpetas sensibles
111
+ protected = ["/data/db", "/data/media"]
112
+
113
+ for p in protected:
114
+ if path == p or path.startswith(p + "/"):
115
+ raise HTTPException(
116
+ status_code=403,
117
+ detail=f"Access to protected path '{p}' is not allowed"
118
+ )
119
+
120
+ # No permitir sobrescritura
121
+ if target.exists():
122
+ raise HTTPException(status_code=409, detail="Path already exists")
123
+
124
+ try:
125
+ if item_type == "directory":
126
+ target.mkdir(parents=True, exist_ok=False)
127
+ else:
128
+ target.parent.mkdir(parents=True, exist_ok=True)
129
+ with open(target, "wb") as f:
130
+ f.write(b"")
131
+ except Exception as exc:
132
+ raise HTTPException(status_code=500, detail=f"Failed to create item: {exc}")
133
+
134
+ return {
135
+ "status": "ok",
136
+ "created": str(target),
137
+ "type": item_type
138
+ }
139
+
140
+ @router.delete("/delete_path", tags=["Data Manager"])
141
+ def delete_path(
142
+ target: str = Query(..., description="Ruta absoluta dentro de /data a borrar"),
143
+ token: str = Query(..., description="Token requerido para autorización")
144
+ ):
145
+ """
146
+ Delete a file or directory recursively inside /data, except protected folders.
147
+
148
+ Behavior:
149
+ - Validate token.
150
+ - Validate path starts with /data/.
151
+ - Deny deletion of /data/db and /data/media (and anything inside them).
152
+ - If target is a file → delete file.
153
+ - If target is a directory → delete recursively.
154
+ """
155
+
156
+ validate_token(token)
157
+
158
+ # Normalizar ruta
159
+ path = Path(target).resolve()
160
+
161
+ # Debe estar dentro de /data/
162
+ if not str(path).startswith(str(DATA_ROOT)):
163
+ raise HTTPException(status_code=400, detail="Path must be inside /data/")
164
+
165
+ # Carpetas protegidas
166
+ protected = [
167
+ DATA_ROOT / "db",
168
+ DATA_ROOT / "media",
169
+ ]
170
+
171
+ for p in protected:
172
+ if path == p or str(path).startswith(str(p) + "/"):
173
+ raise HTTPException(
174
+ status_code=403,
175
+ detail=f"Deletion forbidden: {p} is protected"
176
+ )
177
+
178
+ # Verificar existencia
179
+ if not path.exists():
180
+ raise HTTPException(status_code=404, detail="Target path does not exist")
181
+
182
+ try:
183
+ if path.is_file():
184
+ path.unlink()
185
+ else:
186
+ shutil.rmtree(path)
187
+
188
+ except Exception as exc:
189
+ raise HTTPException(status_code=500, detail=f"Failed to delete: {exc}")
190
+
191
+ return {
192
+ "status": "ok",
193
+ "deleted": str(path)
194
  }
storage/db_routers.py CHANGED
@@ -1,137 +1,137 @@
1
- import os
2
- import io
3
-
4
- import sqlite3
5
- import zipfile
6
-
7
- from pathlib import Path
8
-
9
- from fastapi import APIRouter, UploadFile, File, Query, HTTPException
10
- from fastapi.responses import JSONResponse, StreamingResponse
11
-
12
- from storage.common import validate_token
13
-
14
- router = APIRouter(prefix="/db", tags=["Database Manager"])
15
- HF_TOKEN = os.getenv("HF_TOKEN")
16
- DB_PATH = Path("/data/db")
17
-
18
- @router.get("/download_all_db_files", tags=["Database Manager"])
19
- async def download_all_db_files(
20
- token: str = Query(..., description="Token required for authorization")
21
- ):
22
- """
23
- Download all .db files from /data/db as a ZIP archive.
24
-
25
- Steps:
26
- - Validate the token.
27
- - Verify that /data/db exists and contains .db files.
28
- - Create an in-memory ZIP containing all .db files.
29
- - Return the ZIP as a downloadable file.
30
- """
31
- validate_token(token)
32
-
33
- if not DB_PATH.exists():
34
- raise HTTPException(status_code=404, detail="DB folder does not exist")
35
-
36
- db_files = list(DB_PATH.glob("*.db"))
37
- if not db_files:
38
- raise HTTPException(status_code=404, detail="No .db files found")
39
-
40
- # Create in-memory ZIP
41
- zip_buffer = io.BytesIO()
42
-
43
- with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zipf:
44
- for file_path in db_files:
45
- zipf.write(file_path, arcname=file_path.name)
46
-
47
- zip_buffer.seek(0)
48
-
49
- return StreamingResponse(
50
- zip_buffer,
51
- media_type="application/zip",
52
- headers={
53
- "Content-Disposition": 'attachment; filename="db_files.zip"'
54
- }
55
- )
56
-
57
-
58
- @router.post("/upload_db_file", tags=["Database Manager"])
59
- async def upload_db_file(
60
- file: UploadFile = File(...),
61
- token: str = Query(..., description="Token required for authorization")
62
- ):
63
- """
64
- Upload a file to the /data/db folder.
65
-
66
- Steps:
67
- - Validate the token.
68
- - Ensure /data/db exists (create it if missing).
69
- - Save the uploaded file inside /data/db using its original filename.
70
- - Return a JSON response confirming the upload.
71
- """
72
- validate_token(token)
73
-
74
- # Crear carpeta /data/db si no existe
75
- DB_PATH.mkdir(parents=True, exist_ok=True)
76
-
77
- final_path = DB_PATH / file.filename
78
-
79
- try:
80
- # Leer contenido en memoria y guardar
81
- file_bytes = await file.read()
82
- with open(final_path, "wb") as f:
83
- f.write(file_bytes)
84
- except Exception as exc:
85
- raise HTTPException(status_code=500, detail=f"Failed to save file: {exc}")
86
-
87
- return JSONResponse(
88
- status_code=200,
89
- content={
90
- "status": "ok",
91
- "saved_to": str(final_path)
92
- }
93
- )
94
-
95
- @router.post("/execute_query", tags=["Database Manager"])
96
- async def execute_query(
97
- db_filename: str = Query(..., description="SQLite .db file name"),
98
- query: str = Query(..., description="SQL query to execute"),
99
- token: str = Query(..., description="Token required for authorization")
100
- ):
101
- """
102
- Execute a SQL query against a specified SQLite database file in /data/db.
103
-
104
- Steps:
105
- - Validate the token.
106
- - Ensure the requested .db file exists inside /data/db.
107
- - Connect to the database using sqlite3.
108
- - Execute the provided query.
109
- - Return the results as a list of dictionaries (column: value).
110
- - Capture and return errors if query execution fails.
111
- """
112
- validate_token(token)
113
-
114
- db_file_path = DB_PATH / db_filename
115
-
116
- if not db_file_path.exists() or not db_file_path.is_file():
117
- raise HTTPException(status_code=404, detail=f"Database file {db_filename} not found")
118
-
119
- try:
120
- conn = sqlite3.connect(db_file_path)
121
- conn.row_factory = sqlite3.Row # para devolver columnas por nombre
122
- cur = conn.cursor()
123
-
124
- cur.execute(query)
125
- rows = cur.fetchall()
126
-
127
- # Convert rows to list of dicts
128
- results = [dict(row) for row in rows]
129
-
130
- conn.commit()
131
- conn.close()
132
- except sqlite3.Error as e:
133
- raise HTTPException(status_code=400, detail=f"SQL error: {e}")
134
- except Exception as e:
135
- raise HTTPException(status_code=500, detail=f"Unexpected error: {e}")
136
-
137
  return JSONResponse(content={"results": results})
 
1
+ import os
2
+ import io
3
+
4
+ import sqlite3
5
+ import zipfile
6
+
7
+ from pathlib import Path
8
+
9
+ from fastapi import APIRouter, UploadFile, File, Query, HTTPException
10
+ from fastapi.responses import JSONResponse, StreamingResponse
11
+
12
+ from storage.common import validate_token
13
+
14
+ router = APIRouter(prefix="/db", tags=["Database Manager"])
15
+ HF_TOKEN = os.getenv("HF_TOKEN")
16
+ DB_PATH = Path("/data/db")
17
+
18
+ @router.get("/download_all_db_files", tags=["Database Manager"])
19
+ async def download_all_db_files(
20
+ token: str = Query(..., description="Token required for authorization")
21
+ ):
22
+ """
23
+ Download all .db files from /data/db as a ZIP archive.
24
+
25
+ Steps:
26
+ - Validate the token.
27
+ - Verify that /data/db exists and contains .db files.
28
+ - Create an in-memory ZIP containing all .db files.
29
+ - Return the ZIP as a downloadable file.
30
+ """
31
+ validate_token(token)
32
+
33
+ if not DB_PATH.exists():
34
+ raise HTTPException(status_code=404, detail="DB folder does not exist")
35
+
36
+ db_files = list(DB_PATH.glob("*.db"))
37
+ if not db_files:
38
+ raise HTTPException(status_code=404, detail="No .db files found")
39
+
40
+ # Create in-memory ZIP
41
+ zip_buffer = io.BytesIO()
42
+
43
+ with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zipf:
44
+ for file_path in db_files:
45
+ zipf.write(file_path, arcname=file_path.name)
46
+
47
+ zip_buffer.seek(0)
48
+
49
+ return StreamingResponse(
50
+ zip_buffer,
51
+ media_type="application/zip",
52
+ headers={
53
+ "Content-Disposition": 'attachment; filename="db_files.zip"'
54
+ }
55
+ )
56
+
57
+
58
+ @router.post("/upload_db_file", tags=["Database Manager"])
59
+ async def upload_db_file(
60
+ file: UploadFile = File(...),
61
+ token: str = Query(..., description="Token required for authorization")
62
+ ):
63
+ """
64
+ Upload a file to the /data/db folder.
65
+
66
+ Steps:
67
+ - Validate the token.
68
+ - Ensure /data/db exists (create it if missing).
69
+ - Save the uploaded file inside /data/db using its original filename.
70
+ - Return a JSON response confirming the upload.
71
+ """
72
+ validate_token(token)
73
+
74
+ # Crear carpeta /data/db si no existe
75
+ DB_PATH.mkdir(parents=True, exist_ok=True)
76
+
77
+ final_path = DB_PATH / file.filename
78
+
79
+ try:
80
+ # Leer contenido en memoria y guardar
81
+ file_bytes = await file.read()
82
+ with open(final_path, "wb") as f:
83
+ f.write(file_bytes)
84
+ except Exception as exc:
85
+ raise HTTPException(status_code=500, detail=f"Failed to save file: {exc}")
86
+
87
+ return JSONResponse(
88
+ status_code=200,
89
+ content={
90
+ "status": "ok",
91
+ "saved_to": str(final_path)
92
+ }
93
+ )
94
+
95
+ @router.post("/execute_query", tags=["Database Manager"])
96
+ async def execute_query(
97
+ db_filename: str = Query(..., description="SQLite .db file name"),
98
+ query: str = Query(..., description="SQL query to execute"),
99
+ token: str = Query(..., description="Token required for authorization")
100
+ ):
101
+ """
102
+ Execute a SQL query against a specified SQLite database file in /data/db.
103
+
104
+ Steps:
105
+ - Validate the token.
106
+ - Ensure the requested .db file exists inside /data/db.
107
+ - Connect to the database using sqlite3.
108
+ - Execute the provided query.
109
+ - Return the results as a list of dictionaries (column: value).
110
+ - Capture and return errors if query execution fails.
111
+ """
112
+ validate_token(token)
113
+
114
+ db_file_path = DB_PATH / db_filename
115
+
116
+ if not db_file_path.exists() or not db_file_path.is_file():
117
+ raise HTTPException(status_code=404, detail=f"Database file {db_filename} not found")
118
+
119
+ try:
120
+ conn = sqlite3.connect(db_file_path)
121
+ conn.row_factory = sqlite3.Row # para devolver columnas por nombre
122
+ cur = conn.cursor()
123
+
124
+ cur.execute(query)
125
+ rows = cur.fetchall()
126
+
127
+ # Convert rows to list of dicts
128
+ results = [dict(row) for row in rows]
129
+
130
+ conn.commit()
131
+ conn.close()
132
+ except sqlite3.Error as e:
133
+ raise HTTPException(status_code=400, detail=f"SQL error: {e}")
134
+ except Exception as e:
135
+ raise HTTPException(status_code=500, detail=f"Unexpected error: {e}")
136
+
137
  return JSONResponse(content={"results": results})
storage/embeddings_routers.py CHANGED
@@ -1,162 +1,162 @@
1
- import os
2
- import io
3
- import json
4
- import shutil
5
-
6
- import sqlite3
7
-
8
- from pathlib import Path
9
-
10
- from fastapi import APIRouter, UploadFile, File, Query, HTTPException
11
- from fastapi.responses import FileResponse, JSONResponse
12
-
13
-
14
- from storage.files.file_manager import FileManager
15
- from storage.common import validate_token
16
-
17
- router = APIRouter(prefix="/embeddings", tags=["Embeddings Manager"])
18
- EMBEDDINGS_ROOT = Path("/data/embeddings")
19
- file_manager = FileManager(EMBEDDINGS_ROOT)
20
- HF_TOKEN = os.getenv("HF_TOKEN")
21
-
22
-
23
- @router.get("/list_embeddings", tags=["Embeddings Manager"])
24
- def list_all_embeddings(
25
- token: str = Query(..., description="Token required for authorization")
26
- ):
27
- """
28
- List all embeddings stored under /data/embeddings.
29
-
30
- For each video hash folder, returns:
31
- - video: folder name (hash)
32
- - faces: true/false depending on whether faces/embeddings.json exists
33
- - voices: true/false depending on whether voices/embeddings.json exists
34
-
35
- Notes:
36
- - A video folder may contain only faces, only voices, or neither.
37
- - Missing folders are treated as false.
38
- """
39
- validate_token(token)
40
-
41
- results = []
42
-
43
- # If embeddings root does not exist, return empty list
44
- if not EMBEDDINGS_ROOT.exists():
45
- return []
46
-
47
- for video_dir in EMBEDDINGS_ROOT.iterdir():
48
- if not video_dir.is_dir():
49
- continue # Skip anything that is not a folder
50
-
51
- faces_path = video_dir / "faces" / "embeddings.json"
52
- voices_path = video_dir / "voices" / "embeddings.json"
53
-
54
- results.append({
55
- "video": video_dir.name,
56
- "faces": faces_path.exists(),
57
- "voices": voices_path.exists()
58
- })
59
-
60
- return results
61
-
62
-
63
- @router.post("/upload_embeddings", tags=["Embeddings Manager"])
64
- async def upload_embeddings(
65
- file: UploadFile = File(...),
66
- embedding_type: str = Query(..., description="faces or voices"),
67
- video_hash: str = Query(..., description="Hash of the video"),
68
- token: str = Query(..., description="Token required for authorization")
69
- ):
70
- """
71
- Upload embeddings JSON for a given video and type (faces or voices).
72
-
73
- Behavior:
74
- - Validate the token.
75
- - Validate embedding_type.
76
- - Ensure directory structure: /data/embeddings/<video_hash>/<embedding_type>/
77
- - Delete any existing .json file inside that folder.
78
- - Save the uploaded embeddings as embeddings.json.
79
- """
80
- validate_token(token)
81
-
82
- # Validación del tipo
83
- if embedding_type not in ("faces", "voices"):
84
- raise HTTPException(status_code=400, detail="embedding_type must be 'faces' or 'voices'")
85
-
86
- # Rutas objetivo
87
- video_path = EMBEDDINGS_ROOT / video_hash
88
- type_path = video_path / embedding_type
89
-
90
- # Crear carpetas si no existen
91
- type_path.mkdir(parents=True, exist_ok=True)
92
-
93
- # Eliminar JSONs previos
94
- for existing in type_path.glob("*.json"):
95
- try:
96
- existing.unlink()
97
- except Exception as exc:
98
- raise HTTPException(status_code=500, detail=f"Failed to delete old embeddings: {exc}")
99
-
100
- # Guardar como embeddings.json
101
- final_path = type_path / "embeddings.json"
102
-
103
- try:
104
- file_bytes = await file.read()
105
- with open(final_path, "wb") as f:
106
- f.write(file_bytes)
107
- except Exception as exc:
108
- raise HTTPException(status_code=500, detail=f"Failed to save embeddings: {exc}")
109
-
110
- return JSONResponse(
111
- status_code=200,
112
- content={
113
- "status": "ok",
114
- "saved_to": str(final_path)
115
- }
116
- )
117
-
118
- def get_embeddings_json(video_hash: str, embedding_type: str):
119
- """
120
- Returns the parsed embeddings.json for a given video and type.
121
-
122
- Behavior:
123
- - Validate embedding_type.
124
- - Build the file path: /data/embeddings/<video_hash>/<embedding_type>/embeddings.json
125
- - Raise HTTPException if missing.
126
- - Load and return parsed JSON.
127
- """
128
-
129
- if embedding_type not in ("faces", "voices"):
130
- raise HTTPException(status_code=400, detail="embedding_type must be 'faces' or 'voices'")
131
-
132
- target_file = EMBEDDINGS_ROOT / video_hash / embedding_type / "embeddings.json"
133
-
134
- if not target_file.exists():
135
- raise HTTPException(
136
- status_code=404,
137
- detail=f"embeddings.json not found for video={video_hash}, type={embedding_type}"
138
- )
139
-
140
- try:
141
- with open(target_file, "r", encoding="utf-8") as f:
142
- data = json.load(f)
143
- except Exception as exc:
144
- raise HTTPException(status_code=500, detail=f"Failed to read embeddings: {exc}")
145
-
146
- return data
147
-
148
-
149
- @router.get("/get_embedding", tags=["Embeddings Manager"])
150
- def get_embeddings(
151
- video_hash: str = Query(..., description="Hash of the video"),
152
- embedding_type: str = Query(..., description="faces or voices"),
153
- token: str = Query(..., description="Token required for authorization")
154
- ):
155
- """
156
- Endpoint to retrieve embeddings.json for a given video hash and type.
157
- """
158
- validate_token(token)
159
-
160
- data = get_embeddings_json(video_hash, embedding_type)
161
-
162
  return data
 
1
+ import os
2
+ import io
3
+ import json
4
+ import shutil
5
+
6
+ import sqlite3
7
+
8
+ from pathlib import Path
9
+
10
+ from fastapi import APIRouter, UploadFile, File, Query, HTTPException
11
+ from fastapi.responses import FileResponse, JSONResponse
12
+
13
+
14
+ from storage.files.file_manager import FileManager
15
+ from storage.common import validate_token
16
+
17
+ router = APIRouter(prefix="/embeddings", tags=["Embeddings Manager"])
18
+ EMBEDDINGS_ROOT = Path("/data/embeddings")
19
+ file_manager = FileManager(EMBEDDINGS_ROOT)
20
+ HF_TOKEN = os.getenv("HF_TOKEN")
21
+
22
+
23
+ @router.get("/list_embeddings", tags=["Embeddings Manager"])
24
+ def list_all_embeddings(
25
+ token: str = Query(..., description="Token required for authorization")
26
+ ):
27
+ """
28
+ List all embeddings stored under /data/embeddings.
29
+
30
+ For each video hash folder, returns:
31
+ - video: folder name (hash)
32
+ - faces: true/false depending on whether faces/embeddings.json exists
33
+ - voices: true/false depending on whether voices/embeddings.json exists
34
+
35
+ Notes:
36
+ - A video folder may contain only faces, only voices, or neither.
37
+ - Missing folders are treated as false.
38
+ """
39
+ validate_token(token)
40
+
41
+ results = []
42
+
43
+ # If embeddings root does not exist, return empty list
44
+ if not EMBEDDINGS_ROOT.exists():
45
+ return []
46
+
47
+ for video_dir in EMBEDDINGS_ROOT.iterdir():
48
+ if not video_dir.is_dir():
49
+ continue # Skip anything that is not a folder
50
+
51
+ faces_path = video_dir / "faces" / "embeddings.json"
52
+ voices_path = video_dir / "voices" / "embeddings.json"
53
+
54
+ results.append({
55
+ "video": video_dir.name,
56
+ "faces": faces_path.exists(),
57
+ "voices": voices_path.exists()
58
+ })
59
+
60
+ return results
61
+
62
+
63
+ @router.post("/upload_embeddings", tags=["Embeddings Manager"])
64
+ async def upload_embeddings(
65
+ file: UploadFile = File(...),
66
+ embedding_type: str = Query(..., description="faces or voices"),
67
+ video_hash: str = Query(..., description="Hash of the video"),
68
+ token: str = Query(..., description="Token required for authorization")
69
+ ):
70
+ """
71
+ Upload embeddings JSON for a given video and type (faces or voices).
72
+
73
+ Behavior:
74
+ - Validate the token.
75
+ - Validate embedding_type.
76
+ - Ensure directory structure: /data/embeddings/<video_hash>/<embedding_type>/
77
+ - Delete any existing .json file inside that folder.
78
+ - Save the uploaded embeddings as embeddings.json.
79
+ """
80
+ validate_token(token)
81
+
82
+ # Validación del tipo
83
+ if embedding_type not in ("faces", "voices"):
84
+ raise HTTPException(status_code=400, detail="embedding_type must be 'faces' or 'voices'")
85
+
86
+ # Rutas objetivo
87
+ video_path = EMBEDDINGS_ROOT / video_hash
88
+ type_path = video_path / embedding_type
89
+
90
+ # Crear carpetas si no existen
91
+ type_path.mkdir(parents=True, exist_ok=True)
92
+
93
+ # Eliminar JSONs previos
94
+ for existing in type_path.glob("*.json"):
95
+ try:
96
+ existing.unlink()
97
+ except Exception as exc:
98
+ raise HTTPException(status_code=500, detail=f"Failed to delete old embeddings: {exc}")
99
+
100
+ # Guardar como embeddings.json
101
+ final_path = type_path / "embeddings.json"
102
+
103
+ try:
104
+ file_bytes = await file.read()
105
+ with open(final_path, "wb") as f:
106
+ f.write(file_bytes)
107
+ except Exception as exc:
108
+ raise HTTPException(status_code=500, detail=f"Failed to save embeddings: {exc}")
109
+
110
+ return JSONResponse(
111
+ status_code=200,
112
+ content={
113
+ "status": "ok",
114
+ "saved_to": str(final_path)
115
+ }
116
+ )
117
+
118
+ def get_embeddings_json(video_hash: str, embedding_type: str):
119
+ """
120
+ Returns the parsed embeddings.json for a given video and type.
121
+
122
+ Behavior:
123
+ - Validate embedding_type.
124
+ - Build the file path: /data/embeddings/<video_hash>/<embedding_type>/embeddings.json
125
+ - Raise HTTPException if missing.
126
+ - Load and return parsed JSON.
127
+ """
128
+
129
+ if embedding_type not in ("faces", "voices"):
130
+ raise HTTPException(status_code=400, detail="embedding_type must be 'faces' or 'voices'")
131
+
132
+ target_file = EMBEDDINGS_ROOT / video_hash / embedding_type / "embeddings.json"
133
+
134
+ if not target_file.exists():
135
+ raise HTTPException(
136
+ status_code=404,
137
+ detail=f"embeddings.json not found for video={video_hash}, type={embedding_type}"
138
+ )
139
+
140
+ try:
141
+ with open(target_file, "r", encoding="utf-8") as f:
142
+ data = json.load(f)
143
+ except Exception as exc:
144
+ raise HTTPException(status_code=500, detail=f"Failed to read embeddings: {exc}")
145
+
146
+ return data
147
+
148
+
149
+ @router.get("/get_embedding", tags=["Embeddings Manager"])
150
+ def get_embeddings(
151
+ video_hash: str = Query(..., description="Hash of the video"),
152
+ embedding_type: str = Query(..., description="faces or voices"),
153
+ token: str = Query(..., description="Token required for authorization")
154
+ ):
155
+ """
156
+ Endpoint to retrieve embeddings.json for a given video hash and type.
157
+ """
158
+ validate_token(token)
159
+
160
+ data = get_embeddings_json(video_hash, embedding_type)
161
+
162
  return data
storage/media_routers.py CHANGED
@@ -624,8 +624,8 @@ def list_all_audios(
624
  return results
625
 
626
 
627
- @router.post("/upload_audio_subtype", tags=["Media Manager"])
628
- async def upload_audio_subtype(
629
  audio: UploadFile = File(...),
630
  sha1: str = Query(..., description="SHA1 of the video folder"),
631
  version: str = Query(..., description="Version: Salamandra or MoE"),
@@ -676,8 +676,8 @@ async def upload_audio_subtype(
676
  )
677
 
678
 
679
- @router.get("/download_audio_subtype", tags=["Media Manager"])
680
- def download_audio_subtype(
681
  sha1: str,
682
  version: str,
683
  token: str = Query(..., description="Token required for authorization")
@@ -794,8 +794,8 @@ def download_audio_ad(
794
  )
795
 
796
 
797
- @router.get("/list_subtype_audios", tags=["Media Manager"])
798
- def list_subtype_audios(
799
  sha1: str = Query(..., description="SHA1 of the video folder"),
800
  token: str = Query(..., description="Token required for authorization")
801
  ):
@@ -835,8 +835,8 @@ def list_subtype_audios(
835
  return results
836
 
837
 
838
- @router.post("/upload_subtype_video", tags=["Media Manager"])
839
- async def upload_subtype_video(
840
  sha1: str = Query(..., description="SHA1 associated to the media folder"),
841
  version: str = Query(..., description="Version: Salamandra or MoE"),
842
  video: UploadFile = File(...),
@@ -882,8 +882,8 @@ async def upload_subtype_video(
882
  }
883
 
884
 
885
- @router.get("/download_subtype_video", tags=["Media Manager"])
886
- def download_subtype_video(
887
  sha1: str,
888
  version: str,
889
  token: str = Query(..., description="Token required for authorization")
@@ -924,8 +924,8 @@ def download_subtype_video(
924
  )
925
 
926
 
927
- @router.get("/list_subtype_videos", tags=["Media Manager"])
928
- def list_subtype_videos(
929
  sha1: str,
930
  token: str = Query(..., description="Token required for authorization")
931
  ):
 
624
  return results
625
 
626
 
627
+ @router.post("/upload_audio_version", tags=["Media Manager"])
628
+ async def upload_audio_version(
629
  audio: UploadFile = File(...),
630
  sha1: str = Query(..., description="SHA1 of the video folder"),
631
  version: str = Query(..., description="Version: Salamandra or MoE"),
 
676
  )
677
 
678
 
679
+ @router.get("/download_audio_version", tags=["Media Manager"])
680
+ def download_audio_version(
681
  sha1: str,
682
  version: str,
683
  token: str = Query(..., description="Token required for authorization")
 
794
  )
795
 
796
 
797
+ @router.get("/list_version_audios", tags=["Media Manager"])
798
+ def list_version_audios(
799
  sha1: str = Query(..., description="SHA1 of the video folder"),
800
  token: str = Query(..., description="Token required for authorization")
801
  ):
 
835
  return results
836
 
837
 
838
+ @router.post("/upload_version_video", tags=["Media Manager"])
839
+ async def upload_version_video(
840
  sha1: str = Query(..., description="SHA1 associated to the media folder"),
841
  version: str = Query(..., description="Version: Salamandra or MoE"),
842
  video: UploadFile = File(...),
 
882
  }
883
 
884
 
885
+ @router.get("/download_version_video", tags=["Media Manager"])
886
+ def download_version_video(
887
  sha1: str,
888
  version: str,
889
  token: str = Query(..., description="Token required for authorization")
 
924
  )
925
 
926
 
927
+ @router.get("/list_version_videos", tags=["Media Manager"])
928
+ def list_version_videos(
929
  sha1: str,
930
  token: str = Query(..., description="Token required for authorization")
931
  ):