adebaumann/labhelper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed management command to remove orphan files
All checks were successful
Build containers when image tags change / build-if-image-changed (., web, containers, main container, git.baumann.gr/adebaumann/labhelper) (push) Successful in 2m13s
Details
Build containers when image tags change / build-if-image-changed (data-loader, loader, initContainers, init-container, git.baumann.gr/adebaumann/labhelper-data-loader) (push) Successful in 8s
Details

Browse Source
This commit is contained in:
Adrian A. Baumann
2026-01-17 14:46:45 +01:00
parent 2705f6c16e
commit 985460ff84
3 changed files with 112 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
argocd/configmap.yaml
View File

@@ -18,4 +18,4 @@ data:
LOGIN_REDIRECT_URL: "index" LOGIN_REDIRECT_URL: "index"
LOGOUT_REDIRECT_URL: "login" LOGOUT_REDIRECT_URL: "login"
GUNICORN_OPTS: "--access-logfile -" GUNICORN_OPTS: "--access-logfile -"
IMAGE_TAG: "0.063" IMAGE_TAG: "0.064"

2
argocd/deployment.yaml
View File

@@ -27,7 +27,7 @@ spec:
mountPath: /data mountPath: /data
containers: containers:
- name: web - name: web
image: git.baumann.gr/adebaumann/labhelper:0.063 image: git.baumann.gr/adebaumann/labhelper:0.064
imagePullPolicy: Always imagePullPolicy: Always
ports: ports:
- containerPort: 8000 - containerPort: 8000

136
boxes/management/commands/clean_orphaned_images.py
View File

@@ -4,65 +4,83 @@ from django.conf import settings
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.db.models import F from django.db.models import F
from sorl.thumbnail.models import KVStore from sorl.thumbnail.models import KVStore
from boxes.models import Thing from boxes.models import Thing, ThingFile
class Command(BaseCommand): class Command(BaseCommand):
help = 'Clean up orphaned images and thumbnails from deleted things' help = "Clean up orphaned images, files, and thumbnails from deleted things"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument( parser.add_argument(
'--dry-run', "--dry-run",
action='store_true', action="store_true",
dest='dry_run', dest="dry_run",
help='Show what would be deleted without actually deleting', help="Show what would be deleted without actually deleting",
) )
def handle(self, *args, **options): def handle(self, *args, **options):
dry_run = options.get('dry_run', False) dry_run = options.get("dry_run", False)
if dry_run: if dry_run:
self.stdout.write(self.style.WARNING('DRY RUN - No files will be deleted')) self.stdout.write(self.style.WARNING("DRY RUN - No files will be deleted"))
self.stdout.write('Finding orphaned images and thumbnails...') self.stdout.write("Finding orphaned images and thumbnails...")
media_root = settings.MEDIA_ROOT media_root = settings.MEDIA_ROOT
cache_root = os.path.join(media_root, 'cache') cache_root = os.path.join(media_root, "cache")
things_root = os.path.join(media_root, 'things') things_root = os.path.join(media_root, "things")
if not os.path.exists(things_root): if not os.path.exists(things_root):
self.stdout.write(self.style.WARNING('No things directory found')) self.stdout.write(self.style.WARNING("No things directory found"))
return return
valid_paths = set() valid_paths = set()
for thing in Thing.objects.exclude(picture__exact='').exclude(picture__isnull=True): for thing in Thing.objects.exclude(picture__exact="").exclude(
picture__isnull=True
):
if thing.picture: if thing.picture:
valid_paths.add(os.path.basename(thing.picture.name)) valid_paths.add(os.path.basename(thing.picture.name))
self.stdout.write(f'Found {len(valid_paths)} valid images in database') for thing_file in ThingFile.objects.all():
if thing_file.file:
if thing_file.file.name.startswith("things/"):
relative_path = thing_file.file.name[7:]
valid_paths.add(relative_path)
self.stdout.write(
f"Found {len(valid_paths)} valid images and files in database"
)
orphaned_thumbnail_paths = set() orphaned_thumbnail_paths = set()
db_cache_paths = set() db_cache_paths = set()
for kvstore in KVStore.objects.filter(key__startswith='sorl-thumbnail||image||'): for kvstore in KVStore.objects.filter(
key__startswith="sorl-thumbnail||image||"
):
try: try:
data = json.loads(kvstore.value) data = json.loads(kvstore.value)
name = data.get('name', '') name = data.get("name", "")
if name.startswith('things/'): if name.startswith("things/"):
filename = os.path.basename(name) filename = os.path.basename(name)
if filename not in valid_paths: if filename not in valid_paths:
image_hash = kvstore.key.split('||')[-1] image_hash = kvstore.key.split("||")[-1]
thumbnail_kvstore = KVStore.objects.filter(key=f'sorl-thumbnail||thumbnails||{image_hash}').first() thumbnail_kvstore = KVStore.objects.filter(
key=f"sorl-thumbnail||thumbnails||{image_hash}"
).first()
if thumbnail_kvstore: if thumbnail_kvstore:
thumbnail_list = json.loads(thumbnail_kvstore.value) thumbnail_list = json.loads(thumbnail_kvstore.value)
for thumbnail_hash in thumbnail_list: for thumbnail_hash in thumbnail_list:
thumbnail_image_kvstore = KVStore.objects.filter(key=f'sorl-thumbnail||image||{thumbnail_hash}').first() thumbnail_image_kvstore = KVStore.objects.filter(
key=f"sorl-thumbnail||image||{thumbnail_hash}"
).first()
if thumbnail_image_kvstore: if thumbnail_image_kvstore:
thumbnail_data = json.loads(thumbnail_image_kvstore.value) thumbnail_data = json.loads(
thumbnail_path = thumbnail_data.get('name', '') thumbnail_image_kvstore.value
if thumbnail_path.startswith('cache/'): )
thumbnail_path = thumbnail_data.get("name", "")
if thumbnail_path.startswith("cache/"):
orphaned_thumbnail_paths.add(thumbnail_path) orphaned_thumbnail_paths.add(thumbnail_path)
elif name.startswith('cache/'): elif name.startswith("cache/"):
db_cache_paths.add(name) db_cache_paths.add(name)
except (json.JSONDecodeError, KeyError, AttributeError): except (json.JSONDecodeError, KeyError, AttributeError):
pass pass
@@ -79,26 +97,30 @@ class Command(BaseCommand):
if relative_path not in valid_paths: if relative_path not in valid_paths:
deleted_count += 1 deleted_count += 1
if dry_run: if dry_run:
self.stdout.write(f'Would delete: {file_path}') self.stdout.write(f"Would delete: {file_path}")
else: else:
try: try:
os.remove(file_path) os.remove(file_path)
self.stdout.write(f'Deleted: {file_path}') self.stdout.write(f"Deleted: {file_path}")
except OSError as e: except OSError as e:
self.stdout.write(self.style.ERROR(f'Failed to delete {file_path}: {e}')) self.stdout.write(
self.style.ERROR(f"Failed to delete {file_path}: {e}")
)
for dirname in dirs: for dirname in dirs:
dir_path = os.path.join(root, dirname) dir_path = os.path.join(root, dirname)
if not os.listdir(dir_path): if not os.listdir(dir_path):
if dry_run: if dry_run:
self.stdout.write(f'Would remove empty directory: {dir_path}') self.stdout.write(f"Would remove empty directory: {dir_path}")
else: else:
try: try:
os.rmdir(dir_path) os.rmdir(dir_path)
self.stdout.write(f'Removed empty directory: {dir_path}') self.stdout.write(f"Removed empty directory: {dir_path}")
empty_dirs_removed += 1 empty_dirs_removed += 1
except OSError as e: except OSError as e:
self.stdout.write(self.style.ERROR(f'Failed to remove {dir_path}: {e}')) self.stdout.write(
self.style.ERROR(f"Failed to remove {dir_path}: {e}")
)
if os.path.exists(cache_root): if os.path.exists(cache_root):
for root, dirs, files in os.walk(cache_root, topdown=False): for root, dirs, files in os.walk(cache_root, topdown=False):
@@ -109,39 +131,69 @@ class Command(BaseCommand):
if relative_path in orphaned_thumbnail_paths: if relative_path in orphaned_thumbnail_paths:
thumbnail_deleted_count += 1 thumbnail_deleted_count += 1
if dry_run: if dry_run:
self.stdout.write(f'Would delete thumbnail (orphaned image): {file_path}') self.stdout.write(
f"Would delete thumbnail (orphaned image): {file_path}"
)
else: else:
try: try:
os.remove(file_path) os.remove(file_path)
self.stdout.write(f'Deleted thumbnail (orphaned image): {file_path}') self.stdout.write(
f"Deleted thumbnail (orphaned image): {file_path}"
)
except OSError as e: except OSError as e:
self.stdout.write(self.style.ERROR(f'Failed to delete {file_path}: {e}')) self.stdout.write(
self.style.ERROR(
f"Failed to delete {file_path}: {e}"
)
)
elif relative_path not in db_cache_paths: elif relative_path not in db_cache_paths:
thumbnail_deleted_count += 1 thumbnail_deleted_count += 1
if dry_run: if dry_run:
self.stdout.write(f'Would delete thumbnail (no db entry): {file_path}') self.stdout.write(
f"Would delete thumbnail (no db entry): {file_path}"
)
else: else:
try: try:
os.remove(file_path) os.remove(file_path)
self.stdout.write(f'Deleted thumbnail (no db entry): {file_path}') self.stdout.write(
f"Deleted thumbnail (no db entry): {file_path}"
)
except OSError as e: except OSError as e:
self.stdout.write(self.style.ERROR(f'Failed to delete {file_path}: {e}')) self.stdout.write(
self.style.ERROR(
f"Failed to delete {file_path}: {e}"
)
)
for dirname in dirs: for dirname in dirs:
dir_path = os.path.join(root, dirname) dir_path = os.path.join(root, dirname)
if not os.listdir(dir_path): if not os.listdir(dir_path):
if dry_run: if dry_run:
self.stdout.write(f'Would remove empty cache directory: {dir_path}') self.stdout.write(
f"Would remove empty cache directory: {dir_path}"
)
else: else:
try: try:
os.rmdir(dir_path) os.rmdir(dir_path)
empty_dirs_removed += 1 empty_dirs_removed += 1
except OSError as e: except OSError as e:
self.stdout.write(self.style.ERROR(f'Failed to remove {dir_path}: {e}')) self.stdout.write(
self.style.ERROR(
f"Failed to remove {dir_path}: {e}"
)
)
if dry_run: if dry_run:
self.stdout.write(self.style.WARNING(f'\nDry run complete. Would delete {deleted_count} images and {thumbnail_deleted_count} thumbnails')) self.stdout.write(
self.stdout.write(f'Would remove {empty_dirs_removed} empty directories') self.style.WARNING(
f"\nDry run complete. Would delete {deleted_count} files and {thumbnail_deleted_count} thumbnails"
)
)
self.stdout.write(f"Would remove {empty_dirs_removed} empty directories")
else: else:
self.stdout.write(self.style.SUCCESS(f'\nCleanup complete! Deleted {deleted_count} images and {thumbnail_deleted_count} thumbnails')) self.stdout.write(
self.stdout.write(f'Removed {empty_dirs_removed} empty directories') self.style.SUCCESS(
f"\nCleanup complete! Deleted {deleted_count} files and {thumbnail_deleted_count} thumbnails"
)
)
self.stdout.write(f"Removed {empty_dirs_removed} empty directories")