From 9a3f6b5db8c38dedbc6ff7932c481b25814c6b0d Mon Sep 17 00:00:00 2001 From: Nils Pukropp Date: Fri, 24 Apr 2026 18:42:53 +0200 Subject: [PATCH] migrated to redis --- .gitignore | 1 + backend/api/migrations/0031_alter_job_name.py | 18 ++++++++++ backend/api/models/system.py | 1 + backend/api/serializers/system.py | 24 +++++++++++++ backend/api/tasks.py | 27 +++++++++----- backend/api/views/admin.py | 35 +++++-------------- backend/entrypoint.sh | 7 ---- backend/holy_backend/settings/base.py | 10 ++++-- docker-compose.yml | 34 ++++++++++++++++-- 9 files changed, 112 insertions(+), 45 deletions(-) create mode 100644 backend/api/migrations/0031_alter_job_name.py diff --git a/.gitignore b/.gitignore index b76c4bd..fdfcb70 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ backend/staticfiles/ backend/media/ backend/django_cache/ backend/backups/ +backend/.seeded # React/Vite build dist/ diff --git a/backend/api/migrations/0031_alter_job_name.py b/backend/api/migrations/0031_alter_job_name.py new file mode 100644 index 0000000..b78c01d --- /dev/null +++ b/backend/api/migrations/0031_alter_job_name.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.4 on 2026-04-24 16:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0030_flavor_image_urls'), + ] + + operations = [ + migrations.AlterField( + model_name='job', + name='name', + field=models.CharField(choices=[('sync_flavors', 'Sync Holy Flavors'), ('cleanup_duplicates', 'Cleanup Duplicate Flavors'), ('backup_db', 'Database Backup'), ('seed_legacy', 'Seed Legacy Data'), ('seed_banners', 'Seed Banner Configurations')], max_length=100), + ), + ] diff --git a/backend/api/models/system.py b/backend/api/models/system.py index 3d48686..ceb7123 100644 --- a/backend/api/models/system.py +++ b/backend/api/models/system.py @@ -56,6 +56,7 @@ class Job(models.Model): ("cleanup_duplicates", "Cleanup Duplicate Flavors"), ("backup_db", "Database Backup"), ("seed_legacy", "Seed Legacy Data"), + ("seed_banners", "Seed Banner Configurations"), ] STATUS_CHOICES = [ ("pending", "Pending"), diff --git a/backend/api/serializers/system.py b/backend/api/serializers/system.py index 52daf04..74ee6cd 100644 --- a/backend/api/serializers/system.py +++ b/backend/api/serializers/system.py @@ -1,7 +1,16 @@ +from datetime import timedelta + from rest_framework import serializers from api.models import Banner, Job, SystemConfig +_INTERVAL_PERIOD_TO_TIMEDELTA = { + "days": lambda every: timedelta(days=every), + "hours": lambda every: timedelta(hours=every), + "minutes": lambda every: timedelta(minutes=every), + "seconds": lambda every: timedelta(seconds=every), +} + class SystemConfigSerializer(serializers.ModelSerializer): class Meta: @@ -17,6 +26,21 @@ class SystemConfigSerializer(serializers.ModelSerializer): class JobSerializer(serializers.ModelSerializer): name_display = serializers.CharField(source="get_name_display", read_only=True) + next_run = serializers.SerializerMethodField() + + def get_next_run(self, obj): + from django_celery_beat.models import PeriodicTask + + task = PeriodicTask.objects.filter(name=f"job:{obj.name}", enabled=True).first() + if not task or not task.interval: + return None + base = task.last_run_at or obj.last_run + if not base: + return None + delta_fn = _INTERVAL_PERIOD_TO_TIMEDELTA.get(task.interval.period) + if not delta_fn: + return None + return base + delta_fn(task.interval.every) class Meta: model = Job diff --git a/backend/api/tasks.py b/backend/api/tasks.py index 80da034..956e37a 100644 --- a/backend/api/tasks.py +++ b/backend/api/tasks.py @@ -8,7 +8,6 @@ status/output/timestamps so the admin UI keeps working. from __future__ import annotations -from datetime import timedelta from io import StringIO from typing import Any @@ -22,7 +21,12 @@ logger = get_task_logger(__name__) def _run_command_job(job_name: str, command: str, *args: str) -> str: - """Run a management command and mirror status/output into the Job row.""" + """Run a management command and mirror status/output into the Job row. + + Scheduling (next_run) is owned by django_celery_beat.PeriodicTask; the Job + serializer derives next_run from it. This function only writes execution + state: status, last_run, last_output, error_message. + """ from api.models import Job job, _ = Job.objects.get_or_create(name=job_name) @@ -43,16 +47,18 @@ def _run_command_job(job_name: str, command: str, *args: str) -> str: raise finally: job.last_output = out.getvalue() - if job.interval_hours > 0: - job.next_run = timezone.now() + timedelta(hours=job.interval_hours) - else: - job.next_run = None - job.save() + job.save(update_fields=["status", "last_output", "error_message"]) return job.last_output -@shared_task(name="api.sync_flavors") +@shared_task( + name="api.sync_flavors", + autoretry_for=(Exception,), + retry_backoff=60, + retry_backoff_max=600, + retry_kwargs={"max_retries": 2}, +) def sync_flavors_task() -> str: return _run_command_job("sync_flavors", "sync_flavors") @@ -72,6 +78,11 @@ def seed_legacy_task() -> str: return _run_command_job("seed_legacy", "seed_legacy_flavors") +@shared_task(name="api.seed_banners") +def seed_banners_task() -> str: + return _run_command_job("seed_banners", "seed_banners") + + @shared_task( name="api.send_email", autoretry_for=(Exception,), diff --git a/backend/api/views/admin.py b/backend/api/views/admin.py index 8b217d8..4676bfc 100644 --- a/backend/api/views/admin.py +++ b/backend/api/views/admin.py @@ -20,6 +20,7 @@ JOB_TASK_MAP = { "cleanup_duplicates": "api.cleanup_duplicates", "backup_db": "api.backup_db", "seed_legacy": "api.seed_legacy", + "seed_banners": "api.seed_banners", } @@ -130,35 +131,17 @@ class AdminViewSet(viewsets.ViewSet): def update_job_schedule(self, request: Request, pk=None) -> Response: try: job = Job.objects.get(pk=pk) - interval = request.data.get("interval_hours") - next_run = request.data.get("next_run") - - if interval is not None: - job.interval_hours = int(interval) - - if next_run is not None: - from django.utils import timezone - from django.utils.dateparse import parse_datetime - - parsed_dt = parse_datetime(next_run) - if parsed_dt: - if timezone.is_naive(parsed_dt): - job.next_run = timezone.make_aware( - parsed_dt, timezone.get_current_timezone() - ) - else: - job.next_run = parsed_dt - elif interval is not None and job.interval_hours > 0 and not job.next_run: - from django.utils import timezone - - job.next_run = timezone.now() - - job.save() - _sync_periodic_task(job) - return Response(JobSerializer(job).data) except Job.DoesNotExist: return Response(status=status.HTTP_404_NOT_FOUND) + interval = request.data.get("interval_hours") + if interval is not None: + job.interval_hours = int(interval) + job.save(update_fields=["interval_hours"]) + + _sync_periodic_task(job) + return Response(JobSerializer(job).data) + @action(detail=False, methods=["post"]) def send_test_email(self, request: Request) -> Response: try: diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index a36ee0b..a3ce13e 100755 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -18,15 +18,10 @@ fi echo "Applying database migrations..." python manage.py migrate --noinput -echo "Creating cache table..." -python manage.py createcachetable - echo "Collecting static files..." python manage.py collectstatic --no-input mkdir -p media/flavors -mkdir -p django_cache -chmod -R 777 django_cache # One-shot seed jobs. These should run on image first boot, not on every # restart. A marker file on the bind-mounted volume gates them. @@ -40,8 +35,6 @@ if [ ! -f "$SEED_MARKER" ]; then touch "$SEED_MARKER" else echo "Seed marker present — skipping seed commands. Trigger them from /admin if needed." - # Banners are cheap + config-driven; always re-sync them. - python manage.py seed_banners || echo "seed_banners failed (non-fatal)" fi if [ "$#" -gt 0 ]; then diff --git a/backend/holy_backend/settings/base.py b/backend/holy_backend/settings/base.py index a215850..9686882 100644 --- a/backend/holy_backend/settings/base.py +++ b/backend/holy_backend/settings/base.py @@ -87,10 +87,12 @@ DATABASES = { } } +DJANGO_CACHE_URL = os.environ.get("DJANGO_CACHE_URL", "redis://redis:6379/1") + CACHES = { "default": { - "BACKEND": "django.core.cache.backends.db.DatabaseCache", - "LOCATION": "django_ratelimit_cache", + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": DJANGO_CACHE_URL, } } @@ -189,3 +191,7 @@ CELERY_TASK_TRACK_STARTED = True CELERY_TASK_TIME_LIMIT = 30 * 60 CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" CELERY_TASK_ALWAYS_EAGER = os.environ.get("CELERY_TASK_ALWAYS_EAGER", "false").lower() == "true" +CELERY_TASK_ACKS_LATE = True +CELERY_TASK_REJECT_ON_WORKER_LOST = True +CELERY_WORKER_PREFETCH_MULTIPLIER = 1 +CELERY_RESULT_EXPIRES = 3600 diff --git a/docker-compose.yml b/docker-compose.yml index 63e2824..3d102b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ services: redis: image: redis:7-alpine container_name: holy-redis + command: redis-server --appendonly yes --save 60 1 restart: unless-stopped networks: - holy-net @@ -49,7 +50,7 @@ services: build: context: ./backend container_name: holy-worker - command: celery -A holy_backend worker -l info + command: celery -A holy_backend worker -l info --concurrency=2 restart: unless-stopped networks: - holy-net @@ -70,12 +71,18 @@ services: condition: service_healthy backend: condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "celery -A holy_backend inspect ping -d celery@$$HOSTNAME || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s beat: build: context: ./backend container_name: holy-beat - command: celery -A holy_backend beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler + command: celery -A holy_backend beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler --pidfile /tmp/celerybeat.pid restart: unless-stopped networks: - holy-net @@ -96,6 +103,29 @@ services: condition: service_healthy backend: condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/celerybeat.pid && kill -0 $$(cat /tmp/celerybeat.pid) || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + + flower: + image: mher/flower:2.0 + container_name: holy-flower + restart: unless-stopped + command: celery --broker=${CELERY_BROKER_URL:-redis://redis:6379/0} flower --port=5555 + environment: + - CELERY_BROKER_URL=${CELERY_BROKER_URL:-redis://redis:6379/0} + - CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND:-redis://redis:6379/0} + - FLOWER_BASIC_AUTH=${FLOWER_BASIC_AUTH:-admin:change-me} + ports: + - "5555:5555" + networks: + - holy-net + depends_on: + redis: + condition: service_healthy frontend: build: