import hashlib import os import re from concurrent.futures import ThreadPoolExecutor from urllib.parse import urlparse import requests from django.conf import settings from django.core.management.base import BaseCommand from django.utils.text import slugify from api.models import Category, Flavor _DOWNLOAD_WORKERS = 8 # Shopify product_type looks like "42 - Syrup Bottle" / "01 - Energy Bundle". # Strip the numeric prefix, then the trailing form-factor word, to get the # drink family ("Syrup", "Energy", "Iced Tea") that becomes a Category. _PRODUCT_TYPE_PREFIX = re.compile(r"^\s*\d+\s*[-–—]\s*") # Form-factor words are Holy's internal SKU taxonomy, NOT a pack signal: # "Energy Bundle" is a single can, "Caipirinha Crab" is its title. Pack # detection stays title-driven (see _is_pack). These only get stripped to # reveal the family. _FORM_WORDS = frozenset( {"bundle", "sachet", "sachetbox", "bottle", "box", "pack", "set", "sachets"} ) # When the product_type family itself is one of these, it's not a drink line — # funnel to "Packs and other" instead of minting a category for it. _NON_FLAVOR_TOKENS = frozenset( { "value", "shaker", "merch", "sticker", "packaging", "material", "other", "items", "mixed", "collector", "collector's", "free", "products", "naturalrabatt", "sample", "samplebox", } ) _PACKS_SLUG = "packs-and-other" _PACKS_NAME = "Packs and other" _PACK_TITLE_KEYWORDS = ( "bundle", "set", "box", "probe", "sample", "taster", "starter", "collection", "probier", ) # Accessories (cleaning brushes, spare parts) inherit a drink form-factor in # their product_type ("Reinigungsbürste – Syrup Bottle" -> "42 - Syrup Bottle"), # so family resolution would mis-shelve them as that drink. The title is the # only signal. Funnel to "Packs and other" like shakers. _ACCESSORY_TITLE_KEYWORDS = ( "bürste", "reinigung", "ersatzteil", "zubehör", ) # Fallback only — product_type covers these, but some rows lack one. _TAG_FAMILY = ( (("holy energy", "energy"), "Energy"), (("holy hydration", "hydration"), "Hydration"), (("holy iced tea", "iced tea"), "Iced Tea"), (("milkshake",), "Milkshake"), ) def _family_from_product_type(product_type: str) -> str | None: """Return the drink family from a Shopify product_type, or None. "42 - Syrup Bottle" -> "Syrup"; "Value Pack" / "Shaker" -> None (non-flavor). """ if not product_type: return None stripped = _PRODUCT_TYPE_PREFIX.sub("", product_type).strip() if not stripped: return None tokens = stripped.split() if any(tok.lower() in _NON_FLAVOR_TOKENS for tok in tokens): return None while tokens and tokens[-1].lower() in _FORM_WORDS: tokens.pop() if not tokens: return None return " ".join(tokens) class Command(BaseCommand): help = "Syncs flavors from Holy Energy Shopify API" def clean_html(self, raw_html): if not raw_html: return "" cleanr = re.compile("<.*?>") cleantext = re.sub(cleanr, "", raw_html) return cleantext.strip() def download_image_to_path(self, url, rel_path): """Download `url` to MEDIA_ROOT/rel_path. Skip if file already exists. Returns rel_path or None.""" if not url: return None abs_path = os.path.join(settings.MEDIA_ROOT, rel_path) if os.path.exists(abs_path) and os.path.getsize(abs_path) > 0: return rel_path try: os.makedirs(os.path.dirname(abs_path), exist_ok=True) response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10) response.raise_for_status() with open(abs_path, "wb") as f: f.write(response.content) return rel_path except Exception as e: self.stdout.write(self.style.WARNING(f"Failed to download {url}: {e}")) return None @staticmethod def _ext_for(url): path = urlparse(url).path ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" return ext if ext in ("jpg", "jpeg", "png", "webp", "gif") else "png" @staticmethod def _hash_for(url): path = urlparse(url).path return hashlib.sha1(path.encode("utf-8")).hexdigest()[:10] def download_gallery(self, image_urls, dir_slug): """Download every URL into flavors//_.. Returns list of rel paths. Downloads run in parallel; ordering of returned paths matches input ordering so the gallery index is preserved. """ targets: list[tuple[int, str, str]] = [] for i, url in enumerate(image_urls): if not url: continue filename = f"{i:02d}_{self._hash_for(url)}.{self._ext_for(url)}" rel = os.path.join("flavors", dir_slug, filename) targets.append((i, url, rel)) if not targets: return [] results: dict[int, str] = {} with ThreadPoolExecutor(max_workers=_DOWNLOAD_WORKERS) as ex: futures = { ex.submit(self.download_image_to_path, url, rel): i for i, url, rel in targets } for fut, i in futures.items(): saved = fut.result() if saved: results[i] = saved return [results[i] for i, _, _ in targets if i in results] @staticmethod def _category_for_family(family_name: str) -> Category: """get_or_create a Category from a family name. slugify('Iced Tea') == 'iced-tea', which matches the pre-existing slugs, so known families are reused not duplicated.""" slug = slugify(family_name) return Category.objects.get_or_create(slug=slug, defaults={"name": family_name})[0] @staticmethod def _is_pack(title_lower: str, tags_lower: list[str]) -> bool: if any(k in title_lower for k in _PACK_TITLE_KEYWORDS): return True return "shaker" in title_lower or "merch" in tags_lower def resolve_category(self, product, title_lower, tags_lower, packs_category) -> Category | None: """Pick a Category for a Shopify product. None means skip (no usable signal). Order: pack detection (title/tags) -> accessory title -> product_type family (dynamic, auto-creates) -> tag-based family fallback -> shaker tag -> skip. """ if self._is_pack(title_lower, tags_lower): return packs_category if any(k in title_lower for k in _ACCESSORY_TITLE_KEYWORDS): return packs_category family = _family_from_product_type(product.get("product_type", "")) if family: return self._category_for_family(family) for keywords, name in _TAG_FAMILY: if any(k in tags_lower for k in keywords): return self._category_for_family(name) if "shaker" in tags_lower: return packs_category return None @staticmethod def _is_syrup_variant_product(product) -> bool: """True for the single 'Syrup' article whose variants ARE the flavors. Unlike every other line, syrup has no standalone per-flavor products — it ships as one bundle ('3er Box Syrup') with a 'Flavor' option. Its variants are expanded into individual Flavor rows (see _sync_syrup_variants). """ if _family_from_product_type(product.get("product_type", "")) != "Syrup": return False has_flavor_option = any( (o.get("name") or "").lower() == "flavor" for o in product.get("options", []) ) return has_flavor_option and len(product.get("variants", [])) > 1 def _upsert_flavor( self, *, external_id, name, category, description, shop_url, is_available, image_url, image_urls, dir_slug, ) -> bool: """Find-or-create one Flavor row and refresh its fields. Returns True if created. Match order: external_id -> exact name in category -> case-insensitive name. """ flavor = Flavor.objects.filter(external_id=external_id).first() if not flavor: flavor = Flavor.objects.filter(name=name, category=category).first() if not flavor: flavor = Flavor.objects.filter(name__iexact=name, category=category).first() created = flavor is None if flavor is None: flavor = Flavor(external_id=external_id, name=name, category=category, is_legacy=False) else: if not flavor.external_id: flavor.external_id = external_id if flavor.is_legacy: flavor.is_legacy = False if ( flavor.name != name and not Flavor.objects.filter(name=name, category=category) .exclude(pk=flavor.pk) .exists() ): flavor.name = name flavor.description = description flavor.shop_url = shop_url flavor.is_available = is_available local_paths = self.download_gallery(image_urls, dir_slug) if local_paths: flavor.local_image_paths = local_paths # Drop main_image_path if it points to a file no longer in the gallery if flavor.main_image_path and flavor.main_image_path not in local_paths: flavor.main_image_path = None elif not flavor.local_image_paths: # No new paths and nothing previously stored — leave both empty flavor.main_image_path = None flavor.image_url = image_url flavor.image_urls = image_urls flavor.save() return created def _sync_syrup_variants(self, product, synced_external_ids) -> tuple[int, int]: """Expand the syrup bundle's variants into individual Syrup flavors. Each variant id becomes a Flavor.external_id (BigInt, unique) under the Syrup category; the variant's featured_image is its per-flavor image. Returns (created, updated) counts. """ category = self._category_for_family("Syrup") description = self.clean_html(product.get("body_html", "")) handle = product.get("handle") shop_url = f"https://weareholy.com/products/{handle}" if handle else None created = updated = 0 for v in product.get("variants", []): name = (v.get("option1") or v.get("title") or "").strip() variant_id = v.get("id") if not name or variant_id is None: continue featured = v.get("featured_image") or {} image_url = featured.get("src") image_urls = [image_url] if image_url else [] if self._upsert_flavor( external_id=variant_id, name=name, category=category, description=description, shop_url=shop_url, is_available=bool(v.get("available")), image_url=image_url, image_urls=image_urls, dir_slug=str(variant_id), ): created += 1 else: updated += 1 synced_external_ids.append(variant_id) return created, updated def handle(self, *args, **options): url = "https://weareholy.com/products.json?limit=250" self.stdout.write(f"Fetching from {url}...") try: response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15) response.raise_for_status() data = response.json() except Exception as e: self.stdout.write(self.style.ERROR(f"Failed to fetch data: {e}")) return products = data.get("products", []) self.stdout.write(f"Found {len(products)} products.") packs_category = Category.objects.get_or_create( slug=_PACKS_SLUG, defaults={"name": _PACKS_NAME} )[0] created_count = 0 updated_count = 0 synced_external_ids: list[int] = [] for p in products: title = p.get("title", "").strip() tags = p.get("tags", []) if isinstance(tags, str): tags = [t.strip() for t in tags.split(",")] tags_lower = [t.lower() for t in tags] title_lower = title.lower() # Syrup ships as one article whose variants are the flavors — expand them. if self._is_syrup_variant_product(p): c, u = self._sync_syrup_variants(p, synced_external_ids) created_count += c updated_count += u continue category = self.resolve_category(p, title_lower, tags_lower, packs_category) if not category: continue variants = p.get("variants", []) is_available = any(v.get("available") for v in variants) images = p.get("images", []) image_url = images[0].get("src") if images else None image_urls_list = [img.get("src") for img in images if img.get("src")] description = self.clean_html(p.get("body_html", "")) handle = p.get("handle") shop_url = f"https://weareholy.com/products/{handle}" if handle else None if self._upsert_flavor( external_id=p["id"], name=title, category=category, description=description, shop_url=shop_url, is_available=is_available, image_url=image_url, image_urls=image_urls_list, dir_slug=str(p["id"]), ): created_count += 1 else: updated_count += 1 synced_external_ids.append(p["id"]) discontinued_count = ( Flavor.objects.filter(external_id__isnull=False) .exclude(external_id__in=synced_external_ids) .update(is_available=False) ) self.stdout.write( self.style.SUCCESS( f"Finished! Created: {created_count}, Updated: {updated_count}, Marked Unavailable: {discontinued_count}" ) )