diff --git a/plugins/channel-mapparr/group_scope.py b/plugins/channel-mapparr/group_scope.py new file mode 100644 index 0000000..efb1232 --- /dev/null +++ b/plugins/channel-mapparr/group_scope.py @@ -0,0 +1,208 @@ +"""Pure, Django-free channel-group scope resolution. + +Lives outside plugin.py so the whole include/exclude behaviour table can be +unit-tested with zero mocks. plugin.py supplies the ORM rows and formats the +returns; every rule lives here. + +The contract is section 4 of +docs/superpowers/specs/2026-07-26-ignore-groups-design.md. +""" +import re +from dataclasses import dataclass + +try: + from .wildcard_match import expand_patterns +except ImportError: # loaded standalone (tests, or a non-package path) + from wildcard_match import expand_patterns + +_SPLIT = re.compile(r'[,\n]+') + + +class GroupScopeError(Exception): + """The configured scope cannot be honoured; the action must refuse to run. + + Fail-closed is right here because the scope is the operator's PRIMARY input, + not a defence-in-depth backstop: "I could not resolve your exclusion" must + never authorize touching the channels it was meant to protect. + """ + + +@dataclass(frozen=True) +class GroupScope: + """The resolved include/exclude outcome for one action run. + + ignored_names lists every group name matched by the ignore patterns + ANYWHERE, which is a SUPERSET of out_of_scope_names (the subset that was + already outside the include scope, so removing them changed nothing). + A consumer that sums the two counts, or reports len(ignored_names) as + "groups excluded from this run", will over-count. + """ + group_ids: frozenset[int] + include_ungrouped: bool + ignored_names: tuple[str, ...] = () + out_of_scope_names: tuple[str, ...] = () + info: str = "" + + +def parse_tokens(raw): + """Split a comma/newline separated setting into non-empty stripped tokens. + + Empty tokens are dropped BEFORE any emptiness test, so a stray comma reads + as "no list" rather than "a list that matched nothing" - which, under + fail-closed resolution, would hard-error every action. + """ + if not raw: + return [] + return [tok.strip() for tok in _SPLIT.split(raw) if tok.strip()] + + +def is_ignored_name(name, ignore_value): + """True if a group name the plugin is about to CREATE or write into is ignored. + + The scope filters channels out of a scan; this is the other direction - + nothing should create or adopt a group the operator declared untouchable. + """ + return is_ignored_name_tokens(name, parse_tokens(ignore_value)) + + +def is_ignored_name_tokens(name, tokens): + """Same check as is_ignored_name, but takes ALREADY-PARSED tokens. + + Lets a caller that tests many names against one setting (e.g. a per-channel + Organize loop) parse_tokens() once outside the loop instead of re-parsing + the raw setting string on every iteration. + """ + if not name: + return False + if not isinstance(name, str): + return False + if not tokens: + return False + matched, _ = expand_patterns(tokens, [name], ci_plain=True) + return bool(matched) + + +def build_name_to_ids(rows): + """Map group name -> SET of ids. + + A set, not a scalar: Dispatcharr permits two groups with the same name, and + a scalar map silently drops one of them - leaving it unprotected by an + exclusion that names it. + """ + mapping = {} + for row in rows: + name, gid = row.get('name'), row.get('id') + if name is None or gid is None: + continue + mapping.setdefault(name, set()).add(gid) + return mapping + + +def resolve_group_scope(include_value, ignore_value, group_name_to_ids, *, include_label): + """Resolve the include filter, then subtract the exclusion. + + Returns a GroupScope whose group_ids is always explicit. Raises + GroupScopeError for every refusal case in the spec's section 4 table. + """ + include_tokens = parse_tokens(include_value) + ignore_tokens = parse_tokens(ignore_value) + + # --- include --------------------------------------------------------- + if include_tokens: + # Exact, case-sensitive: unchanged from the pre-existing behaviour. + missing = [t for t in include_tokens if t not in group_name_to_ids] + target = set() + for tok in include_tokens: + target |= group_name_to_ids.get(tok, set()) + if not target: + raise GroupScopeError( + f"None of the groups named in '{include_label}' could be found: " + f"{', '.join(missing)}" + ) + include_ungrouped = False + else: + target = set() + for ids in group_name_to_ids.values(): + target |= ids + include_ungrouped = True + + # --- exclude --------------------------------------------------------- + ignored_names, out_of_scope = (), () + if ignore_tokens: + if not group_name_to_ids: + raise GroupScopeError( + "'Channel Groups to Ignore' is set, but Dispatcharr has no " + "channel groups to match it against." + ) + matched, unmatched = expand_patterns( + ignore_tokens, list(group_name_to_ids), ci_plain=True) + if unmatched: + raise GroupScopeError( + f"These entries in 'Channel Groups to Ignore' match no channel " + f"group: {', '.join(unmatched)}. Check the spelling, or use a " + f"wildcard (a group name containing a comma cannot be written " + f"literally, because the setting splits on commas)." + ) + ignored_ids = set() + for name in matched: + ignored_ids |= group_name_to_ids[name] + ignored_names = tuple(matched) + # A real group that the include filter had already excluded: a no-op, + # NOT a typo. Reported, never fatal. + out_of_scope = tuple( + n for n in matched if not (group_name_to_ids[n] & target)) + target -= ignored_ids + if not target: + raise GroupScopeError( + f"'Channel Groups to Ignore' excluded every group that " + f"'{include_label}' selected, so there is nothing left to " + f"process. Narrow the exclusion or widen the selection." + ) + + return GroupScope( + group_ids=frozenset(target), + include_ungrouped=include_ungrouped, + ignored_names=ignored_names, + out_of_scope_names=out_of_scope, + info=_describe(include_tokens, ignored_names, include_label), + ) + + +def split_rows_by_ignore(rows, ignore_value, *, group_key='channel_group'): + """Partition persisted result rows into (kept, dropped) by group NAME. + + The rename/tag actions replay a results file and never fetch channels, so + the exclusion has to be applied here too. Matching on the stored NAME rather + than an id means a stale file is still filtered after the group has been + renamed or deleted. + + Deliberately does not refuse on an unmatched token: the results file may + legitimately contain no rows from a named group, and refusing a rename for + a group absent from *this file* would be wrong. The typo case is already + caught at scan time by resolve_group_scope. + """ + rows = list(rows) + tokens = parse_tokens(ignore_value) + if not tokens: + return rows, [] + + present = sorted({r.get(group_key) for r in rows if r.get(group_key)}) + matched, _ = expand_patterns(tokens, present, ci_plain=True) + ignored = set(matched) + + kept, dropped = [], [] + for row in rows: + (dropped if row.get(group_key) in ignored else kept).append(row) + return kept, dropped + + +def _describe(include_tokens, ignored_names, include_label): + parts = [] + if include_tokens: + parts.append(f"{include_label}: {', '.join(include_tokens)}") + else: + parts.append(f"{include_label}: all groups") + if ignored_names: + parts.append(f"ignoring {len(ignored_names)} group(s): " + f"{', '.join(ignored_names)}") + return "; ".join(parts) diff --git a/plugins/channel-mapparr/matching_core.py b/plugins/channel-mapparr/matching_core.py index d7e9147..c230c90 100644 --- a/plugins/channel-mapparr/matching_core.py +++ b/plugins/channel-mapparr/matching_core.py @@ -323,6 +323,15 @@ def normalize_name(self, name, user_ignored_tags=None, ignore_quality=True, igno original_name = name + # Strip zero-width / invisible Unicode format characters (category Cf: ZERO WIDTH + # SPACE U+200B, joiners U+200C/D, word joiner U+2060, BOM U+FEFF, soft hyphen + # U+00AD, bidi marks). Some IPTV providers pad names with these around a + # decorative block glyph (e.g. "UK |BBC 1"); they are invisible + # padding that \s does not match and _DECORATOR_CATS does not cover, so they + # would otherwise survive the whole pipeline and poison the match. Removed, not + # spaced, since they are zero-width (a ZWSP inside "BBC" -> "BBC", not "BB C"). + name = ''.join(c for c in name if unicodedata.category(c) != 'Cf') + name = _LEADING_BAR_TAG_RE.sub('', name) # leading "┃CANAL+┃" bouquet tag # Map emoji-as-letters (⚽ = 'o' in "SP⚽RTS") and strip emoji decoration, before @@ -356,8 +365,18 @@ def normalize_name(self, name, user_ignored_tags=None, ignore_quality=True, igno # "SPoRTS" to "3840P" and break the word-boundary anchor. for pattern in RESOLUTION_PATTERNS: name = re.sub(pattern, '', name, flags=re.IGNORECASE) + # Replace with a SPACE, not '' (bug-126). Every QUALITY_PATTERN also + # consumes the whitespace flanking the tag, so deleting the match glues + # the tag's neighbours together whenever a token follows it: + # "SKY NEWS FHD rec" -> "SKY NEWSrec", "CNN [HD] USA" -> "CNNUSA". A + # glued token is also unreachable by a user ignore tag (\brec\b finds no + # boundary inside "NEWSrec"), so the custom-tag escape hatch silently did + # nothing. Tags at the start/end just leave an edge space, which the + # whitespace cleanup at the end of this method strips. The loop still + # terminates: a match always removes >=2 tag chars and adds at most one + # space, so each pass strictly shortens the name. for pattern in QUALITY_PATTERNS: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) + name = re.sub(pattern, ' ', name, flags=re.IGNORECASE) # Normalize spacing around numbers name = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', name) @@ -702,9 +721,21 @@ def _trailing_number(name): 'KIND', 'KING', 'KINGS', 'KISS', 'KITE', 'KNEE', 'KNEW', 'KNOW', 'KNOWN', }) + # OTA branding context that immediately follows a station callsign: a channel + # number (optionally prefixed by a broadcast suffix), e.g. "KING 5", "WAVE 3", + # "WOOD TV8", "WHO 13". Used to rescue a denylisted common-word callsign in a + # loose position WITHOUT also rescuing bare program words ("King of the Hill", + # "Doctor Who"), which never carry this trailing number. bug-098. + _OTA_NUMBER_CONTEXT = re.compile(r'^\s+(?:TV|DT|CD|LP|LD)?\s*\d{1,3}\b', re.IGNORECASE) + def _is_callsign_allowed(self, callsign): """A candidate callsign is allowed if it is not denylisted, OR the plugin - supplied a known-real callsign set that contains it (DB rescue).""" + supplied a known-real callsign set that contains it (DB rescue). + + NOTE: this full rescue is used only at the PARENTHESIZED priorities (1/1b), + where the parentheses are an unambiguous OTA signal. The end-of-name and + loose priorities deliberately do NOT use it for denylisted words -- see + bug-098 hardening in _compute_callsign_with_confidence.""" return (callsign not in self._CALLSIGN_DENYLIST or (self._known_callsigns is not None and callsign in self._known_callsigns)) @@ -746,18 +777,28 @@ def _compute_callsign_with_confidence(self, channel_name): if paren_suffix_match: return paren_suffix_match.group(1).upper(), True - # Priority 3: Callsigns at the end + # Priority 3: Callsigns at the end. A denylisted common word at the end + # ("WOLF KING", "Doctor Who") is NOT rescued here -- end position alone is + # too weak a signal for a word that is also a real callsign. Non-denylisted + # callsigns (and suffixed forms like "KING-TV") still match. bug-098. end_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\s*(?:\.[a-z]+)?\s*$', channel_name, re.IGNORECASE) if end_match: callsign = end_match.group(1).upper() - if self._is_callsign_allowed(callsign): + if callsign not in self._CALLSIGN_DENYLIST: return callsign, True - # Priority 4: Any word matching callsign pattern (low confidence) + # Priority 4: Any word matching callsign pattern (low confidence). A + # denylisted common word is rescued here ONLY in OTA branding context -- + # immediately followed by a channel number ("KING 5", "WAVE 3", "WOOD + # TV8", "WHO 13") -- never as a bare program word ("King of the Hill", + # "Doctor Who", "Will Ferrell"). bug-098. word_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\b', channel_name, re.IGNORECASE) if word_match: callsign = word_match.group(1).upper() - if self._is_callsign_allowed(callsign): + if callsign not in self._CALLSIGN_DENYLIST: + return callsign, False + if (self._known_callsigns is not None and callsign in self._known_callsigns + and self._OTA_NUMBER_CONTEXT.match(channel_name[word_match.end():])): return callsign, False return None, False diff --git a/plugins/channel-mapparr/notify_bridge.py b/plugins/channel-mapparr/notify_bridge.py new file mode 100644 index 0000000..26967fc --- /dev/null +++ b/plugins/channel-mapparr/notify_bridge.py @@ -0,0 +1,218 @@ +"""Channel-Maparr's emit layer for the Newsflasharr notification service. + +This module owns the guard boundary. The vendored client's notify() never +raises, but the code around it can, and a bug here must never break +Channel-Maparr's real work. Every public function in this file is written so a +failure is reported rather than thrown. + +Settings are read from the dict passed in on every call and never cached on an +instance. A value primed on one entry path and read back with getattr on another +fails silently, with no crash and no log line, which is a failure this codebase +has shipped before. + +Stream-Mapparr's scheduled-run timestamp file is deliberately not reproduced +here. It exists there to prove a schedule is still alive, because Newsflasharr's +own absence detector stamps a timestamp on any successful attachment send and +cannot tell a scheduled run from a button press. Channel-Maparr has no scheduler +at all, so such a file would prove nothing. +""" +import json +import os + +# The plugin key. Newsflasharr routing and deduplication both key on it, so it +# must stay stable. +SOURCE = "channel-mapparr" + +# The event name every report notification carries. Newsflasharr routing rules +# match on it, so it must stay stable. "usage_report" matches the naming the +# other report senders on this installation already use, and it does not +# collide, because every existing rule is scoped by source and event together. +EVENT = "usage_report" + +# Everything Newsflasharr needs configured before it can send mail at all. Only +# the PRESENCE of each is ever checked or reported: smtp_password is one of them +# and its value must never reach a log line or a toast. +SMTP_REQUIRED = ("smtp_server", "smtp_username", "smtp_password", "smtp_to") + +# Which runs email a report. Stream-Mapparr's third value, "scheduled", is +# absent because this plugin has no scheduler, and the default is changed to +# match: leaving "scheduled" as the default while removing it from the accepted +# set would resolve every unset value to something outside the set. +_TRIGGERS = ("never", "every_run") +_DEFAULT_TRIGGER = "every_run" + +# Which report files are emailed. A notification carries ONE attachment, so +# "both" means two emails per run and either single format means one. The +# default is a single format deliberately: an attachment bearing event bypasses +# Newsflasharr's hourly cap and its quiet hours, so the email count cannot be +# throttled from the service side and is worth keeping low here. +_FORMATS = ("html", "csv", "both") +_DEFAULT_FORMAT = "html" + +# Where the report files are kept, stated in the notification body as plain text +# so a reader with container access can find the complete set. It is NOT sent as +# the notification's url: measured on a real delivery on 2026-08-02, the email +# template renders url as a hyperlink, and this is a path inside the container, +# so the recipient got a link that could not resolve from a mail client. +REPORT_LOCATION = "/data/channel_mapparr_reports" + + +def is_enabled(settings): + """Is the Newsflasharr master toggle on? + + Public on purpose. The Email Report Now button needs this check to fail fast + before it does any work, and a caller in another module must not reach for a + private helper: nothing would pin that call, so an ordinary rename in this + file would break the button silently. + + A checkbox arrives as a string on some Dispatcharr paths, and bool("false") + is True, so a string is coerced rather than passed to bool. + """ + value = (settings or {}).get("notify_enabled", False) + if isinstance(value, str): + value = value.strip().lower() in ("true", "yes", "1", "on") + return bool(value) + + +def resolve_report_trigger(settings): + """Return "never" or "every_run", never anything else. + + An unrecognised or missing value resolves to the default rather than raising + or guessing. Dispatcharr never prunes a stored setting when its field is + removed, so a value written by an earlier version survives forever and must + not be allowed to decide behaviour. + """ + value = (settings or {}).get("notify_report_on", _DEFAULT_TRIGGER) + if not isinstance(value, str): + return _DEFAULT_TRIGGER + value = value.strip().lower() + return value if value in _TRIGGERS else _DEFAULT_TRIGGER + + +def resolve_report_format(settings): + """Return "html", "csv" or "both", never anything else.""" + value = (settings or {}).get("notify_report_format", _DEFAULT_FORMAT) + if not isinstance(value, str): + return _DEFAULT_FORMAT + value = value.strip().lower() + return value if value in _FORMATS else _DEFAULT_FORMAT + + +def unknown_setting_values(settings): + """Report stored values this module does not recognise, for the operator. + + Silently coercing an unrecognised value is how a setting quietly stops doing + what its owner believes it does. Validate Settings surfaces whatever this + returns, because a promise in a field's help text is not a surface. + """ + settings = settings if isinstance(settings, dict) else {} + problems = [] + for key, accepted in (("notify_report_on", _TRIGGERS), + ("notify_report_format", _FORMATS)): + if key not in settings: + continue + value = settings.get(key) + if isinstance(value, str) and value.strip().lower() in accepted: + continue + problems.append( + f"{key} holds an unrecognised value ({value!r}); " + f"it is being treated as the default. Accepted values: " + f"{', '.join(accepted)}.") + return problems + + +def routes_to_smtp(nf_settings, source=SOURCE, event=EVENT): + """Would a report from this plugin actually reach the email channel? + + Newsflasharr sends an event to `default_channels` when no rule matches it, + so a missing routing rule is invisible from this side: the queue write + succeeds, a delivery is recorded, and the mail goes somewhere other than the + inbox. Attachments are email only, so an unrouted report is delivered with no + file at all. This is the check that makes that visible. + + `routing_rules` is stored as a JSON string, not a list, so it is parsed + defensively; a list is accepted too in case that ever changes. A rule with no + source or no event is a wildcard and matches. Never raises. + """ + nf_settings = nf_settings if isinstance(nf_settings, dict) else {} + raw = nf_settings.get("routing_rules") + rules = raw if isinstance(raw, list) else [] + if isinstance(raw, str): + try: + rules = json.loads(raw) + except (ValueError, TypeError): + rules = [] + for rule in rules if isinstance(rules, list) else []: + if not isinstance(rule, dict): + continue + match = rule.get("match") if isinstance(rule.get("match"), dict) else {} + if match.get("source") not in (None, source): + continue + if match.get("event") not in (None, event): + continue + if any("smtp" in str(channel).lower() for channel in (rule.get("channels") or [])): + return True + return "smtp" in str(nf_settings.get("default_channels") or "").lower() + + +def should_emit(settings): + """Return (bool, reason). The reason is operator readable when False.""" + if not is_enabled(settings): + return False, "notifications to Newsflasharr are switched off" + if resolve_report_trigger(settings) == "never": + return False, "the report trigger is set to never" + return True, None + + +def emit_reports(notify_fn, settings, written): + """Emit one notification per report file. Returns {"sent", "skipped_reason"}. + + `notify_fn` is injected rather than imported so tests can observe the call + without a queue directory. + + Each path must be a caller owned, never rewritten timestamped file that + already exists on disk. An email send re-reads the attachment path on every + retry attempt, so a file rewritten in place would be a different file on the + second attempt. A path that is missing is skipped rather than sent, because a + green task result does not prove an artifact was published. + + No url is sent. An earlier version passed the report path there, reasoning + that it was a locator rather than a link. That distinction does not survive + contact with a mail client: measured on a real delivery on 2026-08-02, the + email arrived with the container path rendered as a hyperlink that could not + resolve. The same information is now stated as plain text in the body. + """ + result = {"sent": 0, "skipped_reason": None} + try: + allowed, reason = should_emit(settings) + if not allowed: + result["skipped_reason"] = reason + return result + written = written or {} + if written.get("error"): + result["skipped_reason"] = written["error"] + return result + wanted = resolve_report_format(settings) + for key, label in (("html_path", "HTML report"), ("csv_path", "CSV report")): + if wanted != "both" and not key.startswith(wanted): + continue + path = written.get(key) + if not path or not os.path.isfile(path): + continue + sent = notify_fn( + source=SOURCE, + title=f"Channel-Maparr {label} ready", + body=(f"Attached: {os.path.basename(path)}\n" + f"Kept in {REPORT_LOCATION} inside the container."), + event=EVENT, + severity="info", + kind="event", + dedup_key=None, + url=None, + attachment=path, + ) + if sent: + result["sent"] += 1 + except Exception as error: + result["skipped_reason"] = f"the emit path raised and was contained: {error}" + return result diff --git a/plugins/channel-mapparr/notify_client.py b/plugins/channel-mapparr/notify_client.py new file mode 100644 index 0000000..ba0e607 --- /dev/null +++ b/plugins/channel-mapparr/notify_client.py @@ -0,0 +1,211 @@ +"""Notifyarr caller client — vendor this single file into your plugin. + +Writes one redacted JSON event into Notifyarr's spool. Stdlib only, no +imports from Notifyarr. never raises. True = durably spooled (NOT +"delivered"). Strings you pass here are POSTed to external services +(Discord etc.): never place raw stream URLs, provider hostnames or +unfiltered exception text in title/body — redaction is best-effort +shape-matching, not a guarantee. Contract: notifier spec §3. +""" +from __future__ import annotations + +import json +import os +import re +import sys +import time +import types + +SCHEMA_V = 1 +CLIENT_VERSION = "1.2.0" +DEFAULT_BASE = "/data/newsflasharr" +MAX_BODY_BYTES = 65536 +MAX_SPOOL_FILES = 1000 +# Headroom only `critical` may consume, so a backed-up spool cannot +# silently swallow a real incident behind a flood of info events. +CRITICAL_RESERVE = 200 +COUNT_CACHE_S = 5.0 +_STATE_KEY = "_notifyarr_client_state" + +_CREDS_RE = re.compile(r"(/(?:live|movie|series)/)[^/\s?]+/[^/\s?]+(?=[/?\s]|$)", + re.IGNORECASE) +_QUERY_CREDS_RE = re.compile(r"([?&](?:username|user|password|pass)=)[^&\s]+", + re.IGNORECASE) +_BASIC_AUTH_RE = re.compile(r"(://)[^/\s@]+:[^/\s@]+(@)") +_SOURCE_RE = re.compile(r"[^a-z0-9_-]") + +# Stripping the username and password out of a provider URL leaves the edge +# hostname and the numeric stream id, and both identify the operator's provider +# account to anyone reading a Discord message or an email. Those are removed too. +# +# The rewrite is scoped to URLs, never to free text, and that scoping is the +# whole safety argument. A rule that hunts for bare hostnames in prose needs a +# curated top-level-domain allowlist, a file-extension denylist, a two-label +# minimum and public-suffix handling, because .ts, .sh, .zip and .mov are all +# real top-level domains. A host sitting between "://" and the next "/" needs +# none of that, because the URL syntax has already identified it. +_URL_RE = re.compile( + r"(?P[a-z][a-z0-9+.\-]*)://(?P[^/\s?#]*)" + r"(?P[^\s?#]*)(?P[?#][^\s]*)?", + re.IGNORECASE) + +# A URL is treated as media, and therefore rewritten, on either signal: a +# provider path segment, or a streaming file extension. Anything else is left +# alone, because destroying a link to documentation or to an issue makes +# notifications less useful while protecting nothing. +_MEDIA_PATH_RE = re.compile(r"/(?:live|movie|series)/", re.IGNORECASE) +_MEDIA_EXT_RE = re.compile( + r"\.(?:ts|m3u8|m3u|mp4|mkv|avi|flv|m4v|mov|mpd|mpg|mpeg|wmv)$", + re.IGNORECASE) + + +def _redact_media_url(match): + """Rewrite one URL if it looks like media, otherwise return it unchanged. + + The scheme, the /live/ style path segments and the file extension survive, + so a reader can still tell what kind of value was removed. The authority, + which carries any host, port and remaining userinfo, and the final path + segment, which carries the stream id, do not. + """ + whole = match.group(0) + path = match.group("path") or "" + if not (_MEDIA_PATH_RE.search(path) or _MEDIA_EXT_RE.search(path)): + return whole + + segments = path.split("/") + last = segments[-1] + ext = "" + dot = last.rfind(".") + if dot > 0: + ext = last[dot:] + segments[-1] = "" + ext + # The query string on a media URL is where a session token lives, so it is + # dropped rather than kept. On a non-media URL it is left untouched above. + return f'{match.group("scheme")}://{"/".join(segments)}' + + +def redact(text): + if text is None: + return None + out = str(text) + out = _CREDS_RE.sub(r"\1/", out) + out = _QUERY_CREDS_RE.sub(r"\1", out) + out = _BASIC_AUTH_RE.sub(r"\1:\2", out) + return _URL_RE.sub(_redact_media_url, out) + + +def _client_state(): + """Per-process mutable state parked in sys.modules — module globals are + wiped by Dispatcharr's reload ping-pong (bug-136).""" + mod = sys.modules.get(_STATE_KEY) + if mod is None: + mod = types.ModuleType(_STATE_KEY) # no __file__: loader ignores it + mod.cache = {} + sys.modules[_STATE_KEY] = mod + return mod.cache + + +def _spool_has_room(spool_dir, now, severity="info"): + """Backpressure, with a reserve that only `critical` may enter. + + This guard used to be CALLER-BLIND: past MAX_SPOOL_FILES it returned + False for everything, so a critical incident arriving while the spool + was backed up was dropped with no ledger row, no failed/ file and no + trace of any kind. The spool only fills when the collector has fallen + behind, which is exactly when a critical matters most. + + Non-critical events are refused at MAX_SPOOL_FILES; critical may use a + further CRITICAL_RESERVE on top. A critical refused even then leaves a + `spool_full` marker (see notify()), which Newsflasharr's `show_status` + reports as an error naming the age of the marker. + + That last clause used to read "so the loss is at least visible" while NO + production code opened the file -- a docstring asserting a safety property + the code did not implement, which is worse than plain silence because it + stops the next reader checking. + """ + cache = _client_state() + hit = cache.get(spool_dir) + if hit and now - hit[1] < COUNT_CACHE_S: + count = hit[0] + else: + try: + count = sum(1 for f in os.listdir(spool_dir) if f.endswith(".json")) + except OSError: + count = 0 + cache[spool_dir] = (count, now) + limit = MAX_SPOOL_FILES + if str(severity) == "critical": + limit += CRITICAL_RESERVE + return count < limit + + +def _mark_spool_full(spool_dir, severity, now): + """Leave a trace when a CRITICAL is refused. + + Best-effort and deliberately silent on failure: this runs on the + caller's hot path and must never raise. A single marker file is + rewritten rather than appended, so a sustained flood cannot itself + fill the disk. + """ + if str(severity) != "critical": + return + try: + path = os.path.join(os.path.dirname(spool_dir), "spool_full") + with open(path, "w", encoding="utf-8") as f: + f.write(str(int(now))) + except Exception: + pass + + +def _truncate_utf8(text, limit): + data = text.encode("utf-8") + if len(data) <= limit: + return text + return data[:limit].decode("utf-8", errors="ignore") + + +def notify(source, title, *, event=None, body="", severity="info", + kind="event", dedup_key=None, url=None, attachment=None, + base_dir=DEFAULT_BASE, _now_ms=None): + try: + spool_dir = os.path.join(base_dir, "spool") + os.makedirs(spool_dir, exist_ok=True) + now = time.time() + if not _spool_has_room(spool_dir, now, severity): + _mark_spool_full(spool_dir, severity, now) + return False + ts = int(_now_ms) if _now_ms is not None else int(now * 1000) + payload = {"v": SCHEMA_V, "client_v": CLIENT_VERSION, + "source": str(source), "kind": str(kind), + "severity": str(severity), "ts": ts, + "title": _truncate_utf8(redact(str(title)) or "", 1024), + "body": _truncate_utf8(redact(str(body)) or "", MAX_BODY_BYTES)} + if event: + payload["event"] = str(event) + if dedup_key: + payload["dedup_key"] = str(dedup_key) + if url: + payload["url"] = redact(str(url)) + if attachment: + payload["attachment"] = str(attachment) + cache = _client_state() + counter = cache["counter"] = (cache.get("counter", 0) + 1) % 10000 + safe = _SOURCE_RE.sub("_", str(source).lower())[:32] or "unknown" + rand8 = os.urandom(4).hex() + name = f"{ts}-{counter:04d}-{safe}-{rand8}.json" + tmp = os.path.join(spool_dir, f".tmp-{name}") + with open(tmp, "w", encoding="utf-8") as f: + json.dump(payload, f, separators=(",", ":")) + os.replace(tmp, os.path.join(spool_dir, name)) + return True + except Exception: + return False + + +def notifier_alive(base_dir=DEFAULT_BASE, max_age_s=120.0): + try: + return time.time() - os.path.getmtime( + os.path.join(base_dir, "state.json")) < max_age_s + except Exception: + return False diff --git a/plugins/channel-mapparr/plugin.json b/plugins/channel-mapparr/plugin.json index 7b2aa49..e86936e 100644 --- a/plugins/channel-mapparr/plugin.json +++ b/plugins/channel-mapparr/plugin.json @@ -1,6 +1,6 @@ { "name": "Channel Mapparr", - "version": "1.26.1791324", + "version": "1.26.2141433", "description": "Standardizes broadcast (OTA) and premium/cable channel names using network data and channel lists. Supports M3U stream import, category organization, and fuzzy matching across 42K+ channels in 11 countries.", "author": "PiratesIRC", "license": "MIT", @@ -9,11 +9,6 @@ "min_dispatcharr_version": "v0.20.0", "help_url": "https://github.com/PiratesIRC/Dispatcharr-Channel-Maparr-Plugin", "fields": [ - { - "id": "version_status", - "label": "Plugin Version", - "type": "info" - }, { "id": "channel_databases", "label": "Channel Databases", @@ -39,14 +34,22 @@ "label": "Channel Groups to Process", "type": "string", "default": "", - "help_text": "Comma-separated Dispatcharr channel group names. Leave blank to process ALL groups. Example: 'Sports, News, Movies'." + "help_text": "Comma-separated Dispatcharr channel group names. Leave blank to process ALL groups. Example: 'Sports, News, Movies'. Use 'Channel Groups to Ignore' to exclude instead." + }, + { + "id": "ignore_groups", + "label": "Channel Groups to Ignore", + "type": "string", + "default": "", + "placeholder": "Teamarr, PPV*", + "help_text": "Comma-separated. Channels in these groups are excluded from renaming, tagging, logos and Organize by Category, regardless of 'Channel Groups to Process' or 'Category Organization Groups'. Supports * and ? wildcards; matching is case-insensitive. An entry that matches no channel group refuses most actions. Organize by Category skips an ignored target group and continues. Import M3U Streams does not check entries against every group; it only refuses if its own target group is ignored." }, { "id": "category_groups", "label": "Category Organization Groups", "type": "string", "default": "", - "help_text": "Comma-separated group names the Organize-by-Category action is allowed to create or move channels into. Leave blank to use the categories defined in the channel database JSON." + "help_text": "Comma-separated group names the Organize-by-Category action is allowed to create or move channels into. Leave blank to use the categories defined in the channel database JSON. Use 'Channel Groups to Ignore' to exclude instead." }, { "id": "m3u_sources", @@ -123,6 +126,36 @@ {"value": "high", "label": "High — gentlest on database"} ], "help_text": "Throttles database writes during bulk operations. 'None' is safe for SSD/local DBs; raise if you see DB lock errors or are running on shared/slow storage." + }, + { + "id": "notify_enabled", + "label": "Send notifications to Newsflasharr", + "type": "boolean", + "default": false, + "help_text": "Requires the Newsflasharr plugin, which is what actually sends the mail. What routes where is configured in Newsflasharr's routing rules, keyed on this plugin's name. Channel Mapparr does not require Newsflasharr to be installed: with it absent or disabled, nothing is sent and nothing fails." + }, + { + "id": "notify_report_on", + "label": "Email A Report After", + "type": "select", + "default": "every_run", + "options": [ + {"value": "never", "label": "Never, do not email reports"}, + {"value": "every_run", "label": "Every run that produces an export"} + ], + "help_text": "Which runs email a report. The emailed report is built specifically for sending: it never contains your M3U source names, which the CSV exports in /data/exports do contain in their settings header. Organize by Category reports only in Dry Run, because a real run of it produces no export. This setting does nothing unless Send notifications to Newsflasharr is on above." + }, + { + "id": "notify_report_format", + "label": "Email Report Format", + "type": "select", + "default": "html", + "options": [ + {"value": "html", "label": "HTML page only, one email"}, + {"value": "csv", "label": "CSV only, one email"}, + {"value": "both", "label": "Both, which arrives as two emails"} + ], + "help_text": "Which report file to email. A notification carries one attachment, so choosing both sends two separate emails per run rather than one email with two files. The HTML page is easier to read and the CSV is easier to sort and filter. Both files are written to /data/channel_mapparr_reports either way; this setting only decides which are emailed." } ], "actions": [ @@ -202,6 +235,14 @@ "description": "Show live progress and ETA for the most recent or running operation. Reads a persistent progress file so you can check without watching container logs.", "button_label": "\u24d8 Status" }, + { + "id": "email_report_now", + "label": "Email Report Now", + "description": "Build a report from the last processed channels and queue it for email. Requires the Newsflasharr plugin installed and enabled, its email settings configured, and a routing rule sending this plugin to email. This button checks all of that first and refuses rather than queueing a report nobody receives. It changes nothing. Queued means written to Newsflasharr's queue, not yet in your inbox. It does NOT prove the automatic path works, because it runs here in the web worker using the settings currently on screen.", + "button_label": "✉ Email Now", + "button_variant": "outline", + "button_color": "cyan" + }, { "id": "clear_csv_exports", "label": "Clear CSV Exports", diff --git a/plugins/channel-mapparr/plugin.py b/plugins/channel-mapparr/plugin.py index cfd4749..69ff24d 100644 --- a/plugins/channel-mapparr/plugin.py +++ b/plugins/channel-mapparr/plugin.py @@ -12,8 +12,6 @@ import time import tempfile import threading -import urllib.request -import urllib.error from datetime import datetime # Import the fuzzy matcher module @@ -21,6 +19,16 @@ from .progress_status import ( build_status_message, load_progress, save_progress_atomic, ) +from .group_scope import ( + GroupScopeError, + build_name_to_ids, + is_ignored_name, + is_ignored_name_tokens, + parse_tokens, + resolve_group_scope, + split_rows_by_ignore, +) +from .wildcard_match import expand_patterns # Django model imports from apps.channels.models import Channel, ChannelGroup, Logo, Stream, ChannelStream @@ -40,11 +48,34 @@ # is owned by root and not writable by the dispatch uwsgi user. PROGRESS_FILE = "/data/channel_mapparr_progress.json" +# Dispatcharr clips action toasts at roughly 280 characters from the MIDDLE +# with no visual marker, so a name list that enumerates every match of a +# wildcard ignore token (e.g. "Sport*") could silently truncate the more +# important parts of the message. Cap enumeration and fall back to a count. +_MAX_NAMES_IN_MESSAGE = 5 + + +# Severity glyphs that validate_settings_action's report lines are built with. +# The final assembly classifies lines by these prefixes, so a new validation +# line MUST start with one of them or it will be silently dropped from the +# operator-facing output. +_VALIDATION_ERROR_GLYPH = "❌" # cross mark +_VALIDATION_WARNING_GLYPH = "⚠" # warning sign (with or without U+FE0F) + + +def _format_capped_name_list(names, limit=_MAX_NAMES_IN_MESSAGE): + """Join up to `limit` names, then summarize the rest as a count.""" + names = list(names) + shown = ", ".join(names[:limit]) + if len(names) > limit: + shown += f" and {len(names) - limit} more" + return shown + class PluginConfig: """Configuration constants for Channel Maparr.""" - PLUGIN_VERSION = "1.26.1791324" + PLUGIN_VERSION = "1.26.2141433" # Channel Database Settings DEFAULT_CHANNEL_DATABASES = "US" @@ -65,9 +96,20 @@ class PluginConfig: # File Paths RESULTS_FILE = "/data/channel_mapparr_loaded_channels.json" - VERSION_CHECK_FILE = "/data/channel_mapparr_version_check.json" EXPORT_DIR = "/data/exports" + # Emailed reports, delivered by the Newsflasharr plugin. + # The master toggle is OFF by default on purpose: a released plugin must not + # begin writing into another plugin's queue the moment it is upgraded. + DEFAULT_NOTIFY_ENABLED = False + # There is no scheduler in this plugin, so "scheduled" is not an option. + DEFAULT_NOTIFY_REPORT_ON = "every_run" + # One format, so one run sends one email. A notification carries a single + # attachment, and an attachment bearing event bypasses Newsflasharr's hourly + # cap and its quiet hours, so the email count cannot be throttled from the + # service side. + DEFAULT_NOTIFY_REPORT_FORMAT = "html" + # tv-logos GitHub repo for per-channel logo lookup TV_LOGOS_REPO = "tv-logo/tv-logos" TV_LOGOS_BRANCH = "main" @@ -207,48 +249,18 @@ class Plugin: # Settings rendered by UI @property def fields(self): - """Dynamically generate fields list with version check""" - # Check for updates from GitHub - version_message = "Checking for updates..." - try: - # Check if we should perform a version check (once per day) - if self._should_check_for_updates(): - # Perform the version check - latest_version = self._get_latest_version("PiratesIRC", "Dispatcharr-Channel-Maparr-Plugin") - - # Check if it's an error message - if latest_version.startswith("Error"): - version_message = f"⚠️ Could not check for updates: {latest_version}" - else: - # Save the check result - self._save_version_check(latest_version) - - # Compare versions - current = self.version - # Remove 'v' prefix if present in latest_version - latest_clean = latest_version.lstrip('v') - - if current == latest_clean: - version_message = f"✅ You are up to date (v{current})" - else: - version_message = f"🔔 Update available! Current: v{current} → Latest: {latest_version}" - else: - # Use cached version info - if self.cached_version_info: - latest_version = self.cached_version_info['latest_version'] - current = self.version - latest_clean = latest_version.lstrip('v') - - if current == latest_clean: - version_message = f"✅ You are up to date (v{current})" - else: - version_message = f"🔔 Update available! Current: v{current} → Latest: {latest_version}" - else: - version_message = "ℹ️ Version check will run on next page load" - except Exception as e: - LOGGER.debug(f"{PLUGIN_LOG_PREFIX} Error during version check: {e}") - version_message = f"⚠️ Error checking for updates: {str(e)}" - + """Build the fields list. Reads the DB for M3U source options. + + There is deliberately NO version display and NO update check here. This + property is on Dispatcharr's per-request hot path, and it used to make a + live call to GitHub's releases API (plus a /data cache write) every time + the settings page was read, so plugin settings could not render without + outbound network access and a slow or hung GitHub stalled the request. + The installed version is already shown by Dispatcharr's own plugin card, + so repeating it as a settings field was noise. + `tests/test_plugin_contract.py::test_no_update_check_remains` and + `::test_no_version_field_in_the_settings_form` keep it that way. + """ # Discover M3U sources from database m3u_source_options = [{"value": "_all", "label": "All sources (no filter)"}] try: @@ -259,12 +271,6 @@ def fields(self): # Build the fields list dynamically return [ - { - "id": "version_status", - "label": "Plugin Version", - "type": "info", - "help_text": version_message - }, { "id": "channel_databases", "label": "Channel Databases", @@ -292,7 +298,25 @@ def fields(self): "type": "string", "default": "", "placeholder": "Locals, News, Entertainment", - "help_text": "Comma-separated. Limits rename/logo actions to these groups. Leave empty for all.", + "help_text": "Comma-separated. Limits rename/logo actions to these groups. Leave empty for all. Use 'Channel Groups to Ignore' to exclude instead.", + }, + { + "id": "ignore_groups", + "label": "Channel Groups to Ignore", + "type": "string", + "default": "", + "placeholder": "Teamarr, PPV*", + "help_text": ( + "Comma-separated. Channels in these groups are excluded from " + "renaming, tagging, logos and Organize by Category, regardless " + "of 'Channel Groups to Process' or 'Category Organization " + "Groups'. Supports * and ? wildcards; matching is " + "case-insensitive. An entry that matches no channel group " + "refuses most actions. Organize by Category skips an ignored " + "target group and continues. Import M3U Streams does not " + "check entries against every group; it only refuses if its " + "own target group is ignored." + ), }, { "id": "category_groups", @@ -300,7 +324,7 @@ def fields(self): "type": "string", "default": "", "placeholder": "Locals, News, Entertainment", - "help_text": "Source groups for category-based reorganization. Leave empty for all.", + "help_text": "Source groups for category-based reorganization. Leave empty for all. Use 'Channel Groups to Ignore' to exclude instead.", }, { "id": "m3u_sources", @@ -386,6 +410,53 @@ def fields(self): ], "help_text": "Delay between DB writes during large imports to reduce server load.", }, + { + "id": "notify_enabled", + "label": "Send notifications to Newsflasharr", + "type": "boolean", + "default": PluginConfig.DEFAULT_NOTIFY_ENABLED, + "help_text": "Requires the Newsflasharr plugin, which is what " + "actually sends the mail. What routes where is " + "configured in Newsflasharr's routing rules, keyed on " + "this plugin's name. Channel Mapparr does not require " + "Newsflasharr to be installed: with it absent or " + "disabled, nothing is sent and nothing fails.", + }, + { + "id": "notify_report_on", + "label": "Email A Report After", + "type": "select", + "default": PluginConfig.DEFAULT_NOTIFY_REPORT_ON, + "options": [ + {"value": "never", "label": "Never, do not email reports"}, + {"value": "every_run", "label": "Every run that produces an export"}, + ], + "help_text": "Which runs email a report. The emailed report is built " + "specifically for sending: it never contains your M3U " + "source names, which the CSV exports in /data/exports do " + "contain in their settings header. Organize by Category " + "reports only in Dry Run, because a real run of it " + "produces no export. This setting does nothing unless " + "Send notifications to Newsflasharr is on above.", + }, + { + "id": "notify_report_format", + "label": "Email Report Format", + "type": "select", + "default": PluginConfig.DEFAULT_NOTIFY_REPORT_FORMAT, + "options": [ + {"value": "html", "label": "HTML page only, one email"}, + {"value": "csv", "label": "CSV only, one email"}, + {"value": "both", "label": "Both, which arrives as two emails"}, + ], + "help_text": "Which report file to email. A notification carries one " + "attachment, so choosing both sends two separate emails " + "per run rather than one email with two files. The HTML " + "page is easier to read and the CSV is easier to sort and " + "filter. Both files are written to " + "/data/channel_mapparr_reports either way; this setting " + "only decides which are emailed.", + }, ] # Actions for Dispatcharr UI. `label` is the action title; `button_label` @@ -467,6 +538,24 @@ def fields(self): "label": "Show Status", "description": "Show live progress and ETA for the most recent or running operation. Reads a persistent progress file so you can check without watching container logs.", "button_label": "\u24d8 Status", + "button_variant": "outline", "button_color": "blue", + }, + { + "id": "email_report_now", + "label": "Email Report Now", + "button_label": "✉ Email Now", + "button_variant": "outline", + "button_color": "cyan", + "description": "Build a report from the last processed channels and " + "queue it for email. Requires the Newsflasharr plugin " + "installed and enabled, its email settings configured, " + "and a routing rule sending this plugin to email. This " + "button checks all of that first and refuses rather than " + "queueing a report nobody receives. It changes nothing. " + "Queued means written to Newsflasharr's queue, not yet in " + "your inbox. It does NOT prove the automatic path works, " + "because it runs here in the web worker using the settings " + "currently on screen.", }, { "id": "clear_csv_exports", @@ -485,8 +574,6 @@ def __init__(self): self.group_name_map = {} # Version check cache state - self.version_check_file = PluginConfig.VERSION_CHECK_FILE - self.cached_version_info = None # Background threading self._thread = None @@ -538,94 +625,6 @@ def _resolve_threshold(self, settings, logger): logger.info(f"{PLUGIN_LOG_PREFIX} Match sensitivity: {sensitivity} (threshold: {threshold})") return threshold - def _get_latest_version(self, owner, repo): - """ - Fetches the latest release tag name from GitHub using only Python's standard library. - Returns the version string or an error message. - """ - url = f"https://api.github.com/repos/{owner}/{repo}/releases/latest" - - # Add a user-agent to avoid potential 403 Forbidden errors - headers = { - 'User-Agent': 'Dispatcharr-Plugin-Version-Checker' - } - - try: - # Create a request object with headers - req = urllib.request.Request(url, headers=headers) - - # Make the request and open the URL with a timeout - with urllib.request.urlopen(req, timeout=5) as response: - # Read the response and decode it as UTF-8 - data = response.read().decode('utf-8') - - # Parse the JSON string - json_data = json.loads(data) - - # Get the tag name - latest_version = json_data.get("tag_name") - - if latest_version: - return latest_version - else: - return "Error: 'tag_name' key not found." - - except urllib.error.HTTPError as http_err: - if http_err.code == 404: - return f"Error: Repo not found or has no releases." - else: - return f"HTTP error: {http_err.code}" - except Exception as e: - # Catch other errors like timeouts - return f"Error: {str(e)}" - - def _should_check_for_updates(self): - """ - Check if we should perform a version check (once per day). - Returns True if we should check, False otherwise. - Also loads and caches the last check data. - """ - try: - if os.path.exists(self.version_check_file): - with open(self.version_check_file, 'r') as f: - data = json.load(f) - last_check_time = data.get('last_check_time') - cached_latest_version = data.get('latest_version') - - if last_check_time and cached_latest_version: - # Check if last check was within 24 hours - last_check_dt = datetime.fromisoformat(last_check_time) - now = datetime.now() - time_diff = now - last_check_dt - - if time_diff.total_seconds() < 86400: # 24 hours in seconds - # Use cached data - self.cached_version_info = { - 'latest_version': cached_latest_version, - 'last_check_time': last_check_time - } - return False # Don't check again - - # Either file doesn't exist, or it's been more than 24 hours - return True - - except Exception as e: - LOGGER.debug(f"{PLUGIN_LOG_PREFIX} Error checking version check time: {e}") - return True # Check if there's an error - - def _save_version_check(self, latest_version): - """Save the version check result to disk with timestamp""" - try: - data = { - 'latest_version': latest_version, - 'last_check_time': datetime.now().isoformat() - } - with open(self.version_check_file, 'w') as f: - json.dump(data, f, indent=2) - LOGGER.debug(f"{PLUGIN_LOG_PREFIX} Saved version check: {latest_version}") - except Exception as e: - LOGGER.debug(f"{PLUGIN_LOG_PREFIX} Error saving version check: {e}") - def _generate_csv_settings_header(self, settings): """Generate CSV header comments with plugin settings""" # Map field IDs to their labels @@ -633,6 +632,7 @@ def _generate_csv_settings_header(self, settings): 'channel_databases': 'Channel Databases', 'match_sensitivity': 'Match Sensitivity', 'selected_groups': 'Channel Groups to Process', + 'ignore_groups': 'Channel Groups to Ignore', 'category_groups': 'Channel Groups for Category Organization', 'm3u_sources': 'M3U Sources', 'm3u_group_filter': 'M3U Group Filter', @@ -662,6 +662,399 @@ def _generate_csv_settings_header(self, settings): header_lines.append("#") return '\n'.join(header_lines) + '\n' + # ======================================== + # EMAILED REPORTS (via the Newsflasharr plugin) + # + # Everything here is written so a failure is REPORTED rather than thrown. + # Reporting is not this plugin's real work, and a bug in the report path must + # never break the run that produced the data. + # + # The modules are imported lazily rather than at the top of this file so that + # a deploy which somehow missed one of them degrades to "no reports" instead + # of breaking the whole plugin at import time. + # ======================================== + + @staticmethod + def _notify_client(): + """The vendored Newsflasharr client. Never hand-edit the vendored copy.""" + try: + from . import notify_client + except ImportError: + import notify_client + return notify_client + + @staticmethod + def _notify_bridge(): + """This plugin's emit layer.""" + try: + from . import notify_bridge + except ImportError: + import notify_bridge + return notify_bridge + + @staticmethod + def _reports(): + """The report model and renderers.""" + try: + from . import reports + except ImportError: + import reports + return reports + + def _report_dir(self): + """Where report files are written. A method so a test can redirect it.""" + return self._reports().REPORT_DIR + + def _notify_send(self, **kwargs): + """One seam in front of the vendored client's notify(). + + A seam rather than a direct call so a test can observe what would be + queued without needing a queue directory on disk. + """ + return self._notify_client().notify(**kwargs) + + def _notifier_alive(self): + """Is Newsflasharr's collector actually running? + + notify() CREATES the queue directory it writes into, so it returns True + with Newsflasharr absent, disabled, or its collector dead, and the event + then sits in a directory nobody reads. This is the one check between + queueing and the inbox that an operator can act on. + """ + try: + return bool(self._notify_client().notifier_alive()) + except Exception: + return False + + def _get_m3u_account_names(self, logger): + """Return the M3U account names, or None when the lookup FAILED. + + None and [] are deliberately different. An empty list is a legitimate + installation with no M3U accounts. None means the lookup raised, and the + caller must refuse to build a report rather than send one whose scrub was + a silent no-op: these names are the primary redaction input for an + emailed report, not a backstop. + + This is a separate read on purpose. The `fields` property also lists M3U + accounts, but it runs on Dispatcharr's per-request hot path and must not + be called from here. + """ + try: + return [row["name"] for row in M3UAccount.objects.all().values("name") + if row.get("name")] + except Exception as error: + logger.warning(f"{PLUGIN_LOG_PREFIX} Could not read the M3U account " + f"names, so no report will be built: {error}") + return None + + @staticmethod + def _read_newsflasharr_config(): + """Read Newsflasharr's stored configuration. Returns None when absent. + + Read only on another plugin's configuration row, which is allowed; + nothing here writes. Raises when the registry itself cannot be reached, + so the caller can tell "not installed" from "could not look". + """ + from apps.plugins.models import PluginConfig as StoredPluginConfig + row = StoredPluginConfig.objects.filter(key="newsflasharr").first() + if row is None: + return None + settings = getattr(row, "settings", None) + return {"enabled": bool(getattr(row, "enabled", False)), + "settings": settings if isinstance(settings, dict) else {}} + + def _newsflasharr_readiness(self): + """Everything that must be true for an emailed report to actually arrive. + + Returns a list of blocking problems, empty when the path is clear. + + This exists because every one of these failures is otherwise invisible + from this side. A missing routing rule in particular: the queue write + succeeds, Newsflasharr records a delivery, and the mail is simply sent + somewhere other than the inbox. Attachments are email only, so an + unrouted report also arrives with no file at all. + + Never echo a settings VALUE here. smtp_password is one of the keys being + checked and only its presence is ever reported. + """ + bridge = self._notify_bridge() + try: + config = self._read_newsflasharr_config() + except Exception as error: + return [f"Could not read Newsflasharr's configuration: {error}"] + if config is None: + return ["Newsflasharr is not installed, and it is what actually " + "sends the mail."] + + problems = [] + if not config.get("enabled"): + problems.append("Newsflasharr is installed but not enabled.") + nf_settings = config.get("settings") or {} + missing = [key for key in bridge.SMTP_REQUIRED + if not str(nf_settings.get(key) or "").strip()] + if missing: + problems.append("Newsflasharr's email settings are not complete " + "(missing: " + ", ".join(missing) + ").") + elif not bridge.routes_to_smtp(nf_settings): + problems.append( + f"Newsflasharr has no routing rule sending {bridge.SOURCE}'s " + f"{bridge.EVENT} to email, and email is not among its default " + "channels, so the report would be delivered somewhere else and " + "without its attachment.") + return problems + + def _resolved_databases(self, settings): + """The country codes that actually have a database file on disk. + + The raw `channel_databases` setting is free text and is never echoed into + a report. Resolving it here means the report states what was really + loaded rather than what somebody typed. + """ + raw = str(settings.get("channel_databases") + or PluginConfig.DEFAULT_CHANNEL_DATABASES) + plugin_dir = os.path.dirname(__file__) + resolved = [] + for code in [part.strip().upper() for part in raw.split(",") if part.strip()]: + if os.path.isfile(os.path.join(plugin_dir, f"{code}_channels.json")): + resolved.append(code) + return resolved + + def _build_and_emit_report(self, settings, logger, *, title, columns, rows, + export_filename=None, report_dir=None): + """Build the report files and queue one notification per file. + + Returns {"sent", "skipped_reason", "blocking_error"}. Never raises. + + `blocking_error` is set only when the operator has asked for reports and + the mail could not possibly arrive. That case has to reach the persistent + red area of the plugin card, because a four second green toast is not a + surface for it. + + Nothing is built when the report would not be sent. Building costs work, + and there is no point paying for a report nobody will receive. + """ + outcome = {"sent": 0, "skipped_reason": None, "blocking_error": None} + try: + bridge = self._notify_bridge() + allowed, reason = bridge.should_emit(settings) + if not allowed: + outcome["skipped_reason"] = reason + return outcome + + problems = self._newsflasharr_readiness() + if problems: + outcome["blocking_error"] = ("Report not queued. " + " ".join(problems)) + outcome["skipped_reason"] = outcome["blocking_error"] + logger.warning(f"{PLUGIN_LOG_PREFIX} {outcome['blocking_error']}") + return outcome + + account_names = self._get_m3u_account_names(logger) + if account_names is None: + outcome["skipped_reason"] = ( + "the M3U account name lookup failed, so the report was not " + "built rather than sent without its redaction") + return outcome + + reports = self._reports() + now = time.time() + model = reports.build_model( + title, columns, rows, + account_names=account_names, + settings=settings, + databases=self._resolved_databases(settings), + version=getattr(self, "version", PluginConfig.PLUGIN_VERSION), + now=now, + export_filename=export_filename) + written = reports.write_report( + model, report_dir or self._report_dir(), now) + if written.get("error"): + logger.warning(f"{PLUGIN_LOG_PREFIX} Report not written: " + f"{written['error']}") + outcome["skipped_reason"] = written["error"] + return outcome + + emitted = bridge.emit_reports(self._notify_send, settings, written) + outcome["sent"] = emitted["sent"] + outcome["skipped_reason"] = emitted["skipped_reason"] + logger.info(f"{PLUGIN_LOG_PREFIX} Report: {emitted['sent']} " + f"notification(s) queued for delivery") + except Exception as error: + logger.warning(f"{PLUGIN_LOG_PREFIX} Report emit suppressed: {error}") + outcome["skipped_reason"] = ( + f"the report path raised and was contained: {error}") + return outcome + + @staticmethod + def _report_outcome_clause(outcome): + """A very short clause to append to an action's own message. + + Dispatcharr shows roughly 280 characters of a toast, clipped from the + middle with no ellipsis, so this must stay small. It returns an empty + string when the operator has not switched notifications on, so somebody + who never opted in never sees report chatter. + + It says QUEUED, never sent. A True from notify() means durably written to + Newsflasharr's queue; delivery happens later on its retry ladder. + """ + outcome = outcome or {} + if outcome.get("sent"): + return f"\nReport queued ({outcome['sent']})." + reason = outcome.get("skipped_reason") or "" + if not reason or "switched off" in reason: + return "" + return f"\nReport not queued: {reason}" + + def email_report_now_action(self, settings, logger): + """Build a report from the last processed channels and queue it now. + + It BUILDS a fresh report rather than re-sending the newest files on disk. + Re-sending races the pruner: the newest file on disk is by definition old + enough to be prune eligible, so a later run could delete it while its + mail was still being retried, and the attachment would silently vanish. + + It refuses BEFORE doing any work when the mail could not arrive, because + a missing routing rule is otherwise invisible: the queue write succeeds + and the mail is simply delivered somewhere else. + + It never writes any kind of "the automatic path ran" marker. Pressing a + button must not be able to look like the ordinary path working. + """ + try: + bridge = self._notify_bridge() + if not bridge.is_enabled(settings): + return {"status": "error", + "error": "Send notifications to Newsflasharr is switched " + "off, so there is nothing to email with."} + + problems = self._newsflasharr_readiness() + if problems: + return {"status": "error", + "error": "Report not queued. " + " ".join(problems)} + + if not self._notifier_alive(): + return {"status": "error", + "error": "Newsflasharr's collector is not running, so a " + "queued report would sit unread. Check that the " + "Newsflasharr plugin is enabled and its collector " + "is running, then try again."} + + if not os.path.exists(self.results_file): + return {"status": "error", + "error": "No processed channels found. Run " + "'Load/Process Channels' first, then press this."} + + with open(self.results_file, "r") as handle: + data = json.load(handle) + changes = data.get("changes", []) + if not changes: + return {"status": "error", + "error": "The last run produced no channel changes, so " + "there is nothing to report on."} + + # Pressing the button IS the request, so the "Email A Report After" + # setting is overridden for this one call. The master toggle above is + # NOT overridden: that one is the operator's opt in. + forced = dict(settings) + forced["notify_report_on"] = "every_run" + + outcome = self._build_and_emit_report( + forced, logger, + title="Rename preview", + columns=self._RENAME_REPORT_COLUMNS, + rows=changes) + + if outcome["blocking_error"]: + return {"status": "error", "error": outcome["blocking_error"]} + if not outcome["sent"]: + return {"status": "error", + "error": "Report not queued: " + + (outcome["skipped_reason"] or "unknown reason")} + return {"status": "success", + "message": f"Report queued ({outcome['sent']}). Queued means " + "written to Newsflasharr's queue, not yet in your " + "inbox. This does not prove the automatic path " + "works."} + except Exception as error: + logger.error(f"{PLUGIN_LOG_PREFIX} Email Report Now failed: {error}") + return {"status": "error", "error": f"Email Report Now failed: {error}"} + + # The allow lists that decide what may leave the box. A row key absent from + # these pairs is never copied into a report, whatever the row carries, so a + # column added to a CSV writer later cannot start being emailed on its own. + _RENAME_REPORT_COLUMNS = [ + ("channel_id", "Channel ID"), + ("channel_number", "Channel Number"), + ("channel_group", "Group"), + ("current_name", "Current Name"), + ("new_name", "New Name"), + ("status", "Status"), + ("matcher", "Matcher"), + ("match_method", "Match Method"), + ("reason", "Reason"), + ] + + _CATEGORY_REPORT_COLUMNS = [ + ("channel_id", "Channel ID"), + ("channel_name", "Channel Name"), + ("current_group", "Current Group"), + ("new_group", "New Group"), + ("category", "Category"), + ("match_type", "Match Type"), + ("match_value", "Match Value"), + ("group_exists", "Group Exists"), + ] + + @staticmethod + def _m3u_report_rows(matched_by_category, unmatched_streams): + """Flatten the M3U import structures into report rows. + + The M3U account is deliberately NOT carried into the report. The CSV + export records it as "M3U-", which is harmless in itself, but + the report has no use for it and an allow list is only worth having if it + stays narrow. + """ + rows = [] + for category in sorted(matched_by_category or {}): + for matched in matched_by_category[category]: + stream = matched.get("stream", {}) + rows.append({ + "stream_id": stream.get("id", ""), + "stream_name": stream.get("name", ""), + "priority": stream.get("priority", 0), + "match_type": matched.get("match_type", ""), + "match_method": matched.get("match_method", ""), + "category": category, + "target_group": category, + "will_import": "Yes", + "notes": "", + }) + for unmatched in unmatched_streams or []: + stream = unmatched.get("stream", {}) + rows.append({ + "stream_id": stream.get("id", ""), + "stream_name": stream.get("name", ""), + "priority": stream.get("priority", 0), + "match_type": "", + "match_method": "No match", + "category": "", + "target_group": "", + "will_import": "No", + "notes": unmatched.get("reason", ""), + }) + return rows + + _M3U_REPORT_COLUMNS = [ + ("stream_id", "Stream ID"), + ("stream_name", "Stream Name"), + ("priority", "Priority"), + ("match_type", "Match Type"), + ("match_method", "Match Method"), + ("category", "Category"), + ("target_group", "Target Group"), + ("will_import", "Will Import"), + ("notes", "Notes"), + ] + # ======================================== # ORM HELPER METHODS # ======================================== @@ -670,12 +1063,115 @@ def _get_all_groups(self, logger): """Fetch all channel groups via Django ORM.""" return list(ChannelGroup.objects.all().values('id', 'name')) - def _get_all_channels(self, logger, group_ids=None): - """Fetch channels via Django ORM, optionally filtered by group IDs.""" + _INCLUDE_KEYS = frozenset({"selected_groups", "category_groups"}) + _INCLUDE_LABELS = { + "selected_groups": "Channel Groups to Process", + "category_groups": "Category Organization Groups", + } + + def _resolve_group_scope(self, settings, logger, include_key): + """Resolve the channel-group scope for an action. + + Raises GroupScopeError when the configured scope cannot be honoured; the + caller turns that into a visible error via _scope_error_return. + """ + if include_key not in self._INCLUDE_KEYS: + raise ValueError(f"unknown include_key {include_key!r}") + + name_to_ids = build_name_to_ids(self._get_all_groups(logger)) + scope = resolve_group_scope( + (settings.get(include_key) or ""), + (settings.get("ignore_groups") or ""), + name_to_ids, + include_label=self._INCLUDE_LABELS[include_key], + ) + logger.info(f"{PLUGIN_LOG_PREFIX} Scope: {scope.info}") + for name in scope.out_of_scope_names: + logger.info( + f"{PLUGIN_LOG_PREFIX} Ignored group '{name}' was already outside " + f"the selected scope - no effect." + ) + return scope + + def _resolve_process_scope(self, settings, logger): + """Scope for the scan / rename / logo actions.""" + return self._resolve_group_scope(settings, logger, "selected_groups") + + def _resolve_category_scope(self, settings, logger): + """Scope for the Organize-by-Category actions.""" + return self._resolve_group_scope(settings, logger, "category_groups") + + @staticmethod + def _scope_error_return(exc): + """`error`, not `message` - `status` renders nowhere on the plugin card.""" + return {"status": "error", "error": str(exc)} + + def _ignore_tokens_error(self, settings, logger): + """Refuse if 'Channel Groups to Ignore' names a group absent from the DB. + + The file-driven actions (Preview / Rename / Tag Unknown) resolve the + exclusion via split_rows_by_ignore against the group names PRESENT IN + THE RESULTS FILE, which deliberately never refuses on a token absent + from that file - a stale file may legitimately contain no rows from a + named group. But that means a typo'd token would otherwise pass those + three actions silently while every DB-scoped action (which validates + against every real group via resolve_group_scope) refuses. This + validates the tokens against the database FIRST, before the + file-scoped split, so all six mutating/preview actions agree on what + counts as an unresolvable exclusion. Returns an error dict, or None + when the tokens are fine (or there are none). + """ + tokens = parse_tokens(settings.get("ignore_groups") or "") + if not tokens: + return None + names = list(build_name_to_ids(self._get_all_groups(logger))) + _, unmatched = expand_patterns(tokens, names, ci_plain=True) + if unmatched: + return {"status": "error", "error": + "These entries in 'Channel Groups to Ignore' match no " + f"channel group: {', '.join(unmatched)}."} + return None + + def _get_all_channels(self, logger, group_ids=None, include_ungrouped=False): + """Fetch channels via Django ORM, optionally filtered by group IDs. + + group_ids=None means "no scope" (every channel). An EMPTY set means "a + scope that resolved to nothing" and returns nothing - `if group_ids:` + collapsed those two cases and silently widened the scope to every channel + in the database (bug-044). + + include_ungrouped keeps channels whose channel_group_id is NULL. A blank + include filter used to pass group_ids=None, which included them; once an + explicit id set is always passed they would silently vanish, and no + exclusion can name a NULL group anyway. + """ qs = Channel.objects.all() - if group_ids: + scoped = group_ids is not None + + if scoped and not include_ungrouped: + if not group_ids: + logger.warning( + f"{PLUGIN_LOG_PREFIX} Group scope resolved to zero groups - " + f"no channels will be processed." + ) qs = qs.filter(channel_group_id__in=group_ids) - return list(qs.values('id', 'name', 'channel_number', 'channel_group_id', 'logo_id')) + + rows = list(qs.values( + 'id', 'name', 'channel_number', 'channel_group_id', 'logo_id')) + + if scoped and include_ungrouped: + keep = set(group_ids) + rows = [ + r for r in rows + if r.get('channel_group_id') in keep + or r.get('channel_group_id') is None + ] + if not group_ids: + logger.warning( + f"{PLUGIN_LOG_PREFIX} Group scope resolved to zero groups - " + f"only ungrouped channels will be processed." + ) + return rows def _bulk_update_channels(self, updates, fields, logger): """Bulk update Channel instances. @@ -887,19 +1383,20 @@ def run(self, action, params, context): "organize_by_category": self.organize_by_category_action, "import_m3u_streams": self.import_m3u_streams_action, "plugin_status": self.plugin_status_action, + "email_report_now": self.email_report_now_action, "clear_csv_exports": self.clear_csv_exports_action, } handler = action_map.get(action) if not handler: logger.warning(f"{PLUGIN_LOG_PREFIX} Unknown action: {action}") - return {"status": "error", "message": f"Unknown action: {action}"} + return {"status": "error", "error": f"Unknown action: {action}"} logger.info(f"{PLUGIN_LOG_PREFIX} Action triggered: {action}") result = handler(settings, logger) status = result.get("status", "?") if isinstance(result, dict) else "ok" - msg = result.get("message", "")[:200] if isinstance(result, dict) else "" + msg = (result.get("message") or result.get("error", ""))[:200] if isinstance(result, dict) else "" is_bg = result.get("background", False) if isinstance(result, dict) else False logger.info(f"{PLUGIN_LOG_PREFIX} Action complete: {action} -> {status} | {msg}") @@ -915,7 +1412,7 @@ def run(self, action, params, context): except Exception as e: LOGGER.exception(f"{PLUGIN_LOG_PREFIX} Error in action '{action}': {e}") - return {"status": "error", "message": str(e)} + return {"status": "error", "error": str(e)} def load_and_process_channels_action(self, settings, logger): """Load channels from database and process them with channel data.""" @@ -926,37 +1423,32 @@ def load_and_process_channels_action(self, settings, logger): channels_loaded = self._load_channel_data(settings, logger) if not channels_loaded: - return {"status": "error", "message": "Channel databases could not be loaded. Please check your channel_databases setting and ensure the files exist."} + return {"status": "error", "error": "Channel databases could not be loaded. Please check your channel_databases setting and ensure the files exist."} logger.info(f"{PLUGIN_LOG_PREFIX} Loading channels from database...") - # Get all groups first to build name-to-id mapping + # Get all groups first to build the id-to-name mapping used below all_groups = self._get_all_groups(logger) - group_name_to_id = {g['name']: g['id'] for g in all_groups if 'name' in g and 'id' in g} group_id_to_name = {g['id']: g['name'] for g in all_groups if 'name' in g and 'id' in g} self.group_name_map = group_id_to_name - # Filter by selected groups if specified - selected_groups_str = settings.get("selected_groups", "").strip() - if selected_groups_str: - input_names = {name.strip() for name in selected_groups_str.split(',') if name.strip()} - valid_names = {n for n in input_names if n in group_name_to_id} - invalid_names = input_names - valid_names - target_group_ids = {group_name_to_id[name] for name in valid_names} - - if not target_group_ids: - return {"status": "error", "message": f"None of the specified groups could be found: {', '.join(invalid_names)}"} - - logger.info(f"{PLUGIN_LOG_PREFIX} Target group IDs: {target_group_ids}") - else: - target_group_ids = set(group_name_to_id.values()) - valid_names = set(group_name_to_id.keys()) - - # Fetch all channels and filter by group ID - all_channels = self._get_all_channels(logger, group_ids=target_group_ids if selected_groups_str else None) + # Resolve the group scope (include filter minus ignore_groups) + try: + scope = self._resolve_process_scope(settings, logger) + except GroupScopeError as exc: + return self._scope_error_return(exc) + + all_channels = self._get_all_channels( + logger, + group_ids=scope.group_ids, + include_ungrouped=scope.include_ungrouped, + ) channels_to_process = all_channels - logger.info(f"{PLUGIN_LOG_PREFIX} Filtered to {len(channels_to_process)} channels in groups: {selected_groups_str if selected_groups_str else 'all groups'}") + logger.info( + f"{PLUGIN_LOG_PREFIX} Filtered to {len(channels_to_process)} " + f"channels ({scope.info})" + ) # Store channels with proper group names for channel in channels_to_process: @@ -1125,7 +1617,9 @@ def load_and_process_channels_action(self, settings, logger): progress.update() - progress.finish() + progress.finish( + summary=f"{len(renamed_channels)} to rename, " + f"{len(skipped_channels)} skipped. Scope: {scope.info}") # Log completion logger.info(f"{PLUGIN_LOG_PREFIX} Processing complete. {len(renamed_channels)} to rename, {len(skipped_channels)} skipped.") @@ -1162,7 +1656,7 @@ def load_and_process_channels_action(self, settings, logger): except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error loading and processing channels: {e}") - return {"status": "error", "message": f"Error loading and processing channels: {e}"} + return {"status": "error", "error": f"Error loading and processing channels: {e}"} def preview_changes_action(self, settings, logger): """Export a CSV showing the preview of channel renaming changes.""" @@ -1170,14 +1664,30 @@ def preview_changes_action(self, settings, logger): if not os.path.exists(self.results_file): - return {"status": "error", "message": "No processed channels found. Please run 'Load/Process Channels' first."} + return {"status": "error", "error": "No processed channels found. Please run 'Load/Process Channels' first."} with open(self.results_file, 'r') as f: data = json.load(f) all_changes = data.get('changes', []) + # A token matching no DB group must refuse here too, or a typo + # renames/tags every excluded channel while every other action + # (which validates against the DB) refuses (blocker: fail-open). + guard = self._ignore_tokens_error(settings, logger) + if guard: + return guard + + # Dry run must reflect the same exclusion the real run applies, or + # the preview contradicts what Rename/Tag Unknown actually does. + all_changes, ignored_rows = split_rows_by_ignore( + all_changes, settings.get("ignore_groups")) + if not all_changes: + if ignored_rows: + return {"status": "success", "message": + f"No changes to preview; all {len(ignored_rows)} " + f"pending change(s) are in ignored groups."} return {"status": "success", "message": "No changes to preview."} # Create export directory if it does not exist @@ -1214,6 +1724,11 @@ def preview_changes_action(self, settings, logger): 'Match Method': change.get('match_method', ''), 'Reason': change.get('reason', '') }) + # Absence of a group's rows proves nothing on its own - + # record how many were excluded so the CSV is self-describing. + if ignored_rows: + csvfile.write( + f"# Excluded by ignore: {len(ignored_rows)} row(s)\n") os.replace(tmp_path, csv_path) except Exception: if tmp_path and os.path.exists(tmp_path): @@ -1225,14 +1740,29 @@ def preview_changes_action(self, settings, logger): renamed_count = sum(1 for c in all_changes if c.get('status') == 'Renamed') skipped_count = sum(1 for c in all_changes if c.get('status') == 'Skipped') - return { - "status": "success", - "message": f"✓ Preview exported to: {csv_filename}\n\n{renamed_count} channels will be renamed, {skipped_count} will be skipped." - } + preview_message = f"✓ Preview exported to: {csv_filename}\n\n{renamed_count} channels will be renamed, {skipped_count} will be skipped." + if ignored_rows: + preview_message += f"\n{len(ignored_rows)} row(s) in ignored groups were excluded." + + # The export is confirmed on disk above, so a report may now be built + # from the same rows. It is built from the ROWS, never by re-reading + # the CSV, whose settings header names the configured M3U sources. + outcome = self._build_and_emit_report( + settings, logger, + title="Rename preview", + columns=self._RENAME_REPORT_COLUMNS, + rows=all_changes, + export_filename=csv_filename) + preview_message += self._report_outcome_clause(outcome) + + result = {"status": "success", "message": preview_message} + if outcome["blocking_error"]: + result["error"] = outcome["blocking_error"] + return result except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error exporting preview: {e}") - return {"status": "error", "message": f"Error exporting preview: {e}"} + return {"status": "error", "error": f"Error exporting preview: {e}"} def rename_channels_action(self, settings, logger): """Apply the standardized names to channels.""" @@ -1247,7 +1777,7 @@ def rename_channels_action(self, settings, logger): return self.preview_changes_action(settings, logger) if not os.path.exists(self.results_file): - return {"status": "error", "message": "No processed channels found. Please run 'Load/Process Channels' first."} + return {"status": "error", "error": "No processed channels found. Please run 'Load/Process Channels' first."} with open(self.results_file, 'r') as f: data = json.load(f) @@ -1255,7 +1785,29 @@ def rename_channels_action(self, settings, logger): all_changes = data.get('changes', []) channels_to_rename = [c for c in all_changes if c.get('status') == 'Renamed'] + # A token matching no DB group must refuse here too (see + # preview_changes_action for why) - this is the action that + # actually writes the renames. + guard = self._ignore_tokens_error(settings, logger) + if guard: + return guard + + # These actions replay a persisted file and never fetch channels, so + # the exclusion has to be applied here too; the file may predate the + # current ignore_groups value. + channels_to_rename, ignored_rows = split_rows_by_ignore( + channels_to_rename, settings.get("ignore_groups")) + if ignored_rows: + logger.info( + f"{PLUGIN_LOG_PREFIX} Skipped {len(ignored_rows)} channel(s) " + f"in ignored groups." + ) + if not channels_to_rename: + if ignored_rows: + return {"status": "success", "message": + f"No channels renamed; all {len(ignored_rows)} " + f"pending change(s) are in ignored groups."} return {"status": "success", "message": "No channels need to be renamed."} # Bulk update using ORM @@ -1266,6 +1818,9 @@ def rename_channels_action(self, settings, logger): self._trigger_frontend_refresh(settings, logger) message_parts = [f"✓ Successfully renamed {len(updates)} channels."] + if ignored_rows: + message_parts.append( + f"Skipped {len(ignored_rows)} channel(s) in ignored groups.") if channels_to_rename: message_parts.append("\n**Sample Changes:**") for change in channels_to_rename[:5]: @@ -1277,7 +1832,7 @@ def rename_channels_action(self, settings, logger): except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error renaming channels: {e}") - return {"status": "error", "message": f"Error renaming channels: {e}"} + return {"status": "error", "error": f"Error renaming channels: {e}"} def rename_unknown_channels_action(self, settings, logger): """Append suffix to channels that could not be matched (OTA and premium/cable).""" @@ -1285,7 +1840,7 @@ def rename_unknown_channels_action(self, settings, logger): if not os.path.exists(self.results_file): - return {"status": "error", "message": "No processed channels found. Please run 'Load/Process Channels' first."} + return {"status": "error", "error": "No processed channels found. Please run 'Load/Process Channels' first."} # Get suffix with default fallback matching the field default suffix = settings.get("unknown_suffix", PluginConfig.DEFAULT_UNKNOWN_SUFFIX) @@ -1295,7 +1850,7 @@ def rename_unknown_channels_action(self, settings, logger): # Only reject if suffix is None or empty after strip if not suffix or not suffix.strip(): - return {"status": "error", "message": "No suffix configured. Please set 'Suffix for Unknown Channels' in plugin settings. Default is ' [Unk]' (with leading space)."} + return {"status": "error", "error": "No suffix configured. Please set 'Suffix for Unknown Channels' in plugin settings. Default is ' [Unk]' (with leading space)."} with open(self.results_file, 'r') as f: data = json.load(f) @@ -1303,17 +1858,47 @@ def rename_unknown_channels_action(self, settings, logger): all_changes = data.get('changes', []) skipped_channels = [c for c in all_changes if c.get('status') == 'Skipped'] + # A token matching no DB group must refuse here too (see + # preview_changes_action for why) - this is the action that + # actually writes the "[Unk]" suffix renames. + guard = self._ignore_tokens_error(settings, logger) + if guard: + return guard + + # These actions replay a persisted file and never fetch channels, so + # the exclusion has to be applied here too; the file may predate the + # current ignore_groups value. + skipped_channels, ignored_rows = split_rows_by_ignore( + skipped_channels, settings.get("ignore_groups")) + if ignored_rows: + logger.info( + f"{PLUGIN_LOG_PREFIX} Skipped {len(ignored_rows)} channel(s) " + f"in ignored groups." + ) + if not skipped_channels: + if ignored_rows: + return {"status": "success", "message": + f"No unknown channels renamed; all {len(ignored_rows)} " + f"pending change(s) are in ignored groups."} return {"status": "success", "message": "No unknown channels to rename."} # Bulk update using ORM updates = [{'id': ch['channel_id'], 'name': ch['current_name'] + suffix} for ch in skipped_channels] + if settings.get("dry_run_mode", False): + return {"status": "success", "message": + f"Dry Run: would tag {len(updates)} unknown channel(s) " + f"with suffix '{suffix}'. No changes written."} + logger.info(f"{PLUGIN_LOG_PREFIX} Adding suffix '{suffix}' to {len(updates)} unknown channels...") self._bulk_update_channels(updates, ['name'], logger) self._trigger_frontend_refresh(settings, logger) message_parts = [f"✓ Successfully added suffix '{suffix}' to {len(updates)} unknown channels."] + if ignored_rows: + message_parts.append( + f"Skipped {len(ignored_rows)} channel(s) in ignored groups.") if skipped_channels: message_parts.append("\n**Sample Changes:**") for change in skipped_channels[:5]: @@ -1326,7 +1911,7 @@ def rename_unknown_channels_action(self, settings, logger): except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error renaming unknown channels: {e}") - return {"status": "error", "message": f"Error renaming unknown channels: {e}"} + return {"status": "error", "error": f"Error renaming unknown channels: {e}"} def apply_logos_action(self, settings, logger): """Apply default logo to channels without logos.""" @@ -1336,7 +1921,7 @@ def apply_logos_action(self, settings, logger): default_logo = settings.get("default_logo", "").strip() if not default_logo: - return {"status": "error", "message": "No default logo configured. Please set 'Default Logo' in plugin settings."} + return {"status": "error", "error": "No default logo configured. Please set 'Default Logo' in plugin settings."} # Get all logos from database logger.info(f"{PLUGIN_LOG_PREFIX} Fetching all logos from database...") @@ -1364,23 +1949,23 @@ def apply_logos_action(self, settings, logger): return { "status": "error", - "message": f"Logo '{default_logo}' not found in logo manager.\n\nSearched through {len(all_logos)} logos. Check the Dispatcharr logs to see available logo names." + "error": f"Logo '{default_logo}' not found in logo manager.\n\nSearched through {len(all_logos)} logos. Check the Dispatcharr logs to see available logo names." } # Fetch FRESH channel data from database logger.info(f"{PLUGIN_LOG_PREFIX} Fetching current channel data from database...") - # Get groups to filter - selected_groups_str = settings.get("selected_groups", "").strip() - target_group_ids = None - if selected_groups_str: - all_groups = self._get_all_groups(logger) - group_name_to_id = {g['name']: g['id'] for g in all_groups if 'name' in g and 'id' in g} - input_names = {name.strip() for name in selected_groups_str.split(',') if name.strip()} - target_group_ids = {group_name_to_id[name] for name in input_names if name in group_name_to_id} - - # Get all channels - all_channels = self._get_all_channels(logger, group_ids=target_group_ids) + # Resolve the group scope (include filter minus ignore_groups) + try: + scope = self._resolve_process_scope(settings, logger) + except GroupScopeError as exc: + return self._scope_error_return(exc) + + all_channels = self._get_all_channels( + logger, + group_ids=scope.group_ids, + include_ungrouped=scope.include_ungrouped, + ) # Filter channels without logos or with "Default" logo (ID 0) channels_without_logos = [] @@ -1398,6 +1983,11 @@ def apply_logos_action(self, settings, logger): # Bulk update using ORM updates = [{'id': ch['id'], 'logo_id': int(logo_id)} for ch in channels_without_logos] + if settings.get("dry_run_mode", False): + return {"status": "success", "message": + f"Dry Run: would apply default logo to {len(updates)} " + f"channel(s). No changes written."} + logger.info(f"{PLUGIN_LOG_PREFIX} Applying logo ID {logo_id} to {len(updates)} channels...") self._bulk_update_channels(updates, ['logo_id'], logger) @@ -1416,7 +2006,7 @@ def apply_logos_action(self, settings, logger): except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error applying logos: {e}") - return {"status": "error", "message": f"Error applying logos: {e}"} + return {"status": "error", "error": f"Error applying logos: {e}"} def apply_tv_logos_action(self, settings, logger): """Assign per-channel logos by fuzzy-matching channel names to the @@ -1430,17 +2020,18 @@ def apply_tv_logos_action(self, settings, logger): country_codes_str = settings.get("channel_databases", PluginConfig.DEFAULT_CHANNEL_DATABASES).strip() country_codes = [c.strip().upper() for c in country_codes_str.split(',') if c.strip()] if not country_codes: - return {"status": "error", "message": "No country databases selected. Set 'Channel Databases' first."} - - selected_groups_str = settings.get("selected_groups", "").strip() - target_group_ids = None - if selected_groups_str: - all_groups = self._get_all_groups(logger) - group_name_to_id = {g['name']: g['id'] for g in all_groups if 'name' in g and 'id' in g} - input_names = {name.strip() for name in selected_groups_str.split(',') if name.strip()} - target_group_ids = {group_name_to_id[name] for name in input_names if name in group_name_to_id} + return {"status": "error", "error": "No country databases selected. Set 'Channel Databases' first."} - all_channels = self._get_all_channels(logger, group_ids=target_group_ids) + try: + scope = self._resolve_process_scope(settings, logger) + except GroupScopeError as exc: + return self._scope_error_return(exc) + + all_channels = self._get_all_channels( + logger, + group_ids=scope.group_ids, + include_ungrouped=scope.include_ungrouped, + ) channels_without_logos = [ ch for ch in all_channels if ch.get('logo_id') in (None, 0, '0') @@ -1474,11 +2065,18 @@ def apply_tv_logos_action(self, settings, logger): country_filelists.append((cc.lower(), country_dir, files)) if not country_filelists: - return {"status": "error", "message": "No tv-logos file lists could be fetched. Check network access or repo path."} + return {"status": "error", "error": "No tv-logos file lists could be fetched. Check network access or repo path."} + + # Hoisted above the loop: a dry run must create NOTHING, including + # the Logo catalog rows the real run creates on a first-time match + # (previously created even under Dry Run, orphaning them in the + # catalog if the operator then declined the real run). + dry_run = settings.get("dry_run_mode", False) progress = ProgressTracker(len(channels_without_logos), "apply_tv_logos", logger) assigned = 0 no_match = 0 + would_create = 0 channel_updates = [] for ch in channels_without_logos: @@ -1502,6 +2100,12 @@ def apply_tv_logos_action(self, settings, logger): logo = existing_logos_by_url.get(matched_url) if not logo: + if dry_run: + # No Logo.objects.create - a dry run must write nothing. + would_create += 1 + assigned += 1 + progress.update() + continue try: logo = Logo.objects.create(name=name, url=matched_url) existing_logos_by_url[matched_url] = logo @@ -1514,17 +2118,26 @@ def apply_tv_logos_action(self, settings, logger): assigned += 1 progress.update() + if dry_run: + summary = ( + f"Dry Run: would apply logos to {assigned} channel(s) " + f"({no_match} had no match, {would_create} would need a " + f"new Logo catalog entry). No changes written." + ) + progress.finish(summary=f"{summary} Scope: {scope.info}") + return {"status": "success", "message": summary} + if channel_updates: self._bulk_update_channels(channel_updates, ['logo_id'], logger) self._trigger_frontend_refresh(settings, logger) summary = f"Assigned {assigned} logos, {no_match} channels had no match." - progress.finish(summary=summary) + progress.finish(summary=f"{summary} Scope: {scope.info}") return {"status": "success", "message": f"✓ {summary}"} except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error applying tv-logos: {e}") - return {"status": "error", "message": f"Error applying tv-logos: {e}"} + return {"status": "error", "error": f"Error applying tv-logos: {e}"} def category_groups_dry_run_action(self, settings, logger): """Export a CSV showing which channels would be moved to which category-based groups.""" @@ -1534,27 +2147,24 @@ def category_groups_dry_run_action(self, settings, logger): # Load channel data to get categories channels_loaded = self._load_channel_data(settings, logger) if not channels_loaded: - return {"status": "error", "message": "Channel databases could not be loaded."} + return {"status": "error", "error": "Channel databases could not be loaded."} # Get all groups and channels all_groups = self._get_all_groups(logger) group_name_to_id = {g['name']: g['id'] for g in all_groups if 'name' in g and 'id' in g} group_id_to_name = {g['id']: g['name'] for g in all_groups if 'name' in g and 'id' in g} - # Filter by category groups if specified - category_groups_str = settings.get("category_groups", "").strip() - if category_groups_str: - input_names = {name.strip() for name in category_groups_str.split(',') if name.strip()} - valid_names = {n for n in input_names if n in group_name_to_id} - target_group_ids = {group_name_to_id[name] for name in valid_names} - - if not target_group_ids: - return {"status": "error", "message": f"None of the specified category groups could be found."} - else: - target_group_ids = set(group_name_to_id.values()) - - # Get all channels and filter by group - all_channels = self._get_all_channels(logger, group_ids=target_group_ids) + # Resolve the group scope (include filter minus ignore_groups) + try: + scope = self._resolve_category_scope(settings, logger) + except GroupScopeError as exc: + return self._scope_error_return(exc) + + all_channels = self._get_all_channels( + logger, + group_ids=scope.group_ids, + include_ungrouped=scope.include_ungrouped, + ) channels_to_process = all_channels # Build category mapping from channel databases @@ -1604,6 +2214,10 @@ def category_groups_dry_run_action(self, settings, logger): # Process channels and determine moves moves = [] + ignored_targets = set() + # Parsed once, not per-channel: is_ignored_name_tokens skips the + # re-parse is_ignored_name would otherwise do on every iteration. + ignore_tokens = parse_tokens(settings.get("ignore_groups") or "") for channel in channels_to_process: channel_name = channel.get('name', '') channel_id = channel.get('id') @@ -1653,6 +2267,12 @@ def category_groups_dry_run_action(self, settings, logger): if category: new_group_name = category + # The exclusion also forbids writing INTO a group - the + # preview must match what the real run would refuse to do. + if is_ignored_name_tokens(new_group_name, ignore_tokens): + ignored_targets.add(new_group_name) + continue + # Check if group exists group_exists = new_group_name in group_name_to_id @@ -1670,7 +2290,13 @@ def category_groups_dry_run_action(self, settings, logger): }) if not moves: - return {"status": "success", "message": "No channels need to be moved to category-based groups."} + message = "No channels need to be moved to category-based groups." + if ignored_targets: + message += ( + f" Skipped {len(ignored_targets)} ignored target group(s): " + f"{_format_capped_name_list(sorted(ignored_targets))}." + ) + return {"status": "success", "message": message} # Create export directory export_dir = PluginConfig.EXPORT_DIR @@ -1681,25 +2307,47 @@ def category_groups_dry_run_action(self, settings, logger): csv_filename = f"channel_mapparr_category_groups_preview_{timestamp}.csv" csv_path = os.path.join(export_dir, csv_filename) - with open(csv_path, 'w', newline='', encoding='utf-8') as csvfile: - # Write settings header as comments - csvfile.write(self._generate_csv_settings_header(settings)) + # Written atomically (temporary file, then rename) like the other two + # CSV writers. A plain open() leaves a TRUNCATED file at the final + # path if the write fails part way, with no temporary file to clean + # up, and that truncated file is the one an operator would later + # believe is complete. It also means there is no single moment at + # which the export is confirmed written, which is what the emailed + # report below waits for. + tmp_path = None + try: + with tempfile.NamedTemporaryFile(mode='w', newline='', encoding='utf-8', + dir=export_dir, suffix='.csv', + delete=False) as csvfile: + tmp_path = csvfile.name + # Write settings header as comments + csvfile.write(self._generate_csv_settings_header(settings)) + # `scope` is already resolved above in this same function, so + # this is not a re-parse: record what the ignore/include + # filters actually MATCHED, not just the raw setting text + # already echoed by the header above. + csvfile.write(f"# Ignore resolved to: {scope.info}\n") - fieldnames = ['Channel ID', 'Channel Name', 'Current Group', 'New Group', 'Category', 'Match Type', 'Match Value', 'Group Exists'] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + fieldnames = ['Channel ID', 'Channel Name', 'Current Group', 'New Group', 'Category', 'Match Type', 'Match Value', 'Group Exists'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - for move in moves: - writer.writerow({ - 'Channel ID': move['channel_id'], - 'Channel Name': move['channel_name'], - 'Current Group': move['current_group'], - 'New Group': move['new_group'], - 'Category': move['category'], - 'Match Type': move['match_type'], - 'Match Value': move['match_value'], - 'Group Exists': move['group_exists'] - }) + writer.writeheader() + for move in moves: + writer.writerow({ + 'Channel ID': move['channel_id'], + 'Channel Name': move['channel_name'], + 'Current Group': move['current_group'], + 'New Group': move['new_group'], + 'Category': move['category'], + 'Match Type': move['match_type'], + 'Match Value': move['match_value'], + 'Group Exists': move['group_exists'] + }) + os.replace(tmp_path, csv_path) + except Exception: + if tmp_path and os.path.exists(tmp_path): + os.unlink(tmp_path) + raise logger.info(f"{PLUGIN_LOG_PREFIX} Category groups preview CSV exported to {csv_path}") @@ -1710,14 +2358,36 @@ def category_groups_dry_run_action(self, settings, logger): broadcast_count = sum(1 for m in moves if 'Broadcast' in m['match_type']) premium_count = sum(1 for m in moves if 'Premium' in m['match_type']) - return { - "status": "success", - "message": f"✓ Preview exported to: {csv_filename}\n\n{len(moves)} channels will be moved ({broadcast_count} broadcast, {premium_count} premium).\n{new_groups_needed} new groups will be created." - } + message = ( + f"✓ Preview exported to: {csv_filename}\n\n{len(moves)} channels will " + f"be moved ({broadcast_count} broadcast, {premium_count} premium).\n" + f"{new_groups_needed} new groups will be created." + ) + if ignored_targets: + message += ( + f"\nSkipped {len(ignored_targets)} ignored target group(s): " + f"{_format_capped_name_list(sorted(ignored_targets))}." + ) + + # The export is confirmed on disk above. Only the Dry Run branch of + # Organize by Category produces an export at all, so only this branch + # can report; a real run has no rows to report on. + outcome = self._build_and_emit_report( + settings, logger, + title="Category organization preview", + columns=self._CATEGORY_REPORT_COLUMNS, + rows=moves, + export_filename=csv_filename) + message += self._report_outcome_clause(outcome) + + result = {"status": "success", "message": message} + if outcome["blocking_error"]: + result["error"] = outcome["blocking_error"] + return result except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error generating category groups preview: {e}") - return {"status": "error", "message": f"Error generating category groups preview: {e}"} + return {"status": "error", "error": f"Error generating category groups preview: {e}"} def organize_by_category_action(self, settings, logger): """Create groups based on category names and move matching channels to those groups.""" @@ -1734,27 +2404,24 @@ def organize_by_category_action(self, settings, logger): # Load channel data to get categories channels_loaded = self._load_channel_data(settings, logger) if not channels_loaded: - return {"status": "error", "message": "Channel databases could not be loaded."} + return {"status": "error", "error": "Channel databases could not be loaded."} # Get all groups and channels all_groups = self._get_all_groups(logger) group_name_to_id = {g['name']: g['id'] for g in all_groups if 'name' in g and 'id' in g} group_id_to_name = {g['id']: g['name'] for g in all_groups if 'name' in g and 'id' in g} - # Filter by category groups if specified - category_groups_str = settings.get("category_groups", "").strip() - if category_groups_str: - input_names = {name.strip() for name in category_groups_str.split(',') if name.strip()} - valid_names = {n for n in input_names if n in group_name_to_id} - target_group_ids = {group_name_to_id[name] for name in valid_names} - - if not target_group_ids: - return {"status": "error", "message": f"None of the specified category groups could be found."} - else: - target_group_ids = set(group_name_to_id.values()) - - # Get all channels and filter by group - all_channels = self._get_all_channels(logger, group_ids=target_group_ids) + # Resolve the group scope (include filter minus ignore_groups) + try: + scope = self._resolve_category_scope(settings, logger) + except GroupScopeError as exc: + return self._scope_error_return(exc) + + all_channels = self._get_all_channels( + logger, + group_ids=scope.group_ids, + include_ungrouped=scope.include_ungrouped, + ) channels_to_process = all_channels # Build category mapping from channel databases @@ -1805,6 +2472,10 @@ def organize_by_category_action(self, settings, logger): # Process channels and determine moves moves = [] groups_needed = set() + ignored_targets = set() + # Parsed once, not per-channel: is_ignored_name_tokens skips the + # re-parse is_ignored_name would otherwise do on every iteration. + ignore_tokens = parse_tokens(settings.get("ignore_groups") or "") for channel in channels_to_process: channel_name = channel.get('name', '') @@ -1847,6 +2518,12 @@ def organize_by_category_action(self, settings, logger): if category: new_group_name = category + # The exclusion also forbids writing INTO a group - never + # create or adopt a target the operator declared untouchable. + if is_ignored_name_tokens(new_group_name, ignore_tokens): + ignored_targets.add(new_group_name) + continue + # Track groups that need to be created if new_group_name not in group_name_to_id: groups_needed.add(new_group_name) @@ -1860,7 +2537,13 @@ def organize_by_category_action(self, settings, logger): }) if not moves: - return {"status": "success", "message": "No channels need to be moved to category-based groups."} + message = "No channels need to be moved to category-based groups." + if ignored_targets: + message += ( + f" Skipped {len(ignored_targets)} ignored target group(s): " + f"{_format_capped_name_list(sorted(ignored_targets))}." + ) + return {"status": "success", "message": message} # Create new groups if needed using ORM created_groups = [] @@ -1884,7 +2567,7 @@ def organize_by_category_action(self, settings, logger): }) if not updates: - return {"status": "error", "message": "Failed to create necessary groups. Please check logs."} + return {"status": "error", "error": "Failed to create necessary groups. Please check logs."} # Apply the moves using ORM logger.info(f"{PLUGIN_LOG_PREFIX} Moving {len(updates)} channels to category-based groups...") @@ -1902,11 +2585,17 @@ def organize_by_category_action(self, settings, logger): if len(moves) > 5: message_parts.append(f"...and {len(moves) - 5} more.") + if ignored_targets: + message_parts.append( + f"\nSkipped {len(ignored_targets)} ignored target group(s): " + f"{_format_capped_name_list(sorted(ignored_targets))}." + ) + return {"status": "success", "message": "\n".join(message_parts)} except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error organizing channels by category: {e}") - return {"status": "error", "message": f"Error organizing channels by category: {e}"} + return {"status": "error", "error": f"Error organizing channels by category: {e}"} # ======================================== # M3U STREAM IMPORT METHODS @@ -2237,6 +2926,21 @@ def _match_streams_to_categories(self, streams, settings, logger): return matched_by_category, unmatched_streams + @staticmethod + def _check_group_destinations_not_ignored(names, ignore_value): + """Refuse rather than create or adopt a group the operator declared untouchable. + + The scope filters channels out of a scan; this is the other direction - + import must not write INTO a group listed in 'Channel Groups to Ignore'. + """ + blocked = sorted({name for name in names if is_ignored_name(name, ignore_value)}) + if blocked: + raise GroupScopeError( + f"Import would create or write into group(s) listed in 'Channel " + f"Groups to Ignore': {_format_capped_name_list(blocked)}. Change " + f"the import target or remove them from the ignore list." + ) + def _ensure_category_groups_exist(self, categories, settings, logger): """ Ensure all category-based channel groups exist in Dispatcharr. @@ -2248,7 +2952,14 @@ def _ensure_category_groups_exist(self, categories, settings, logger): dict: Mapping of category name to group ID """ # Check if custom group name is specified - custom_group_name = settings.get("m3u_custom_group_name", "").strip() + custom_group_name = (settings.get("m3u_custom_group_name") or "").strip() + + # The exclusion also forbids writing INTO a group. Refuse rather than + # create or adopt a group the operator declared untouchable. + self._check_group_destinations_not_ignored( + [custom_group_name] if custom_group_name else list(categories), + settings.get("ignore_groups"), + ) # Fetch existing groups existing_groups = self._get_all_groups(logger) @@ -2603,7 +3314,7 @@ def import_m3u_streams_dry_run_action(self, settings, logger): streams = self._fetch_streams_from_m3u_sources(settings, logger) if not streams: - return {"status": "error", "message": "No streams found in specified M3U sources"} + return {"status": "error", "error": "No streams found in specified M3U sources"} # Step 2: Match streams to categories matched_by_category, unmatched_streams = self._match_streams_to_categories( @@ -2613,11 +3324,22 @@ def import_m3u_streams_dry_run_action(self, settings, logger): if not matched_by_category: return { "status": "error", - "message": f"No streams matched to channel databases. {len(unmatched_streams)} unmatched streams." + "error": f"No streams matched to channel databases. {len(unmatched_streams)} unmatched streams." } # Step 3: Check which category groups exist categories = list(matched_by_category.keys()) + + # The preview must match what the real run would refuse to do. + custom_group_name = (settings.get("m3u_custom_group_name") or "").strip() + try: + self._check_group_destinations_not_ignored( + [custom_group_name] if custom_group_name else categories, + settings.get("ignore_groups"), + ) + except GroupScopeError as exc: + return self._scope_error_return(exc) + existing_groups = self._get_all_groups(logger) existing_group_names = {group['name'] for group in existing_groups} @@ -2652,7 +3374,7 @@ def import_m3u_streams_dry_run_action(self, settings, logger): except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} M3U import dry run failed: {e}") - return {"status": "error", "message": f"Dry run failed: {str(e)}"} + return {"status": "error", "error": f"Dry run failed: {str(e)}"} def _do_import_m3u_streams(self, settings, logger): """Core M3U import logic.""" @@ -2662,7 +3384,7 @@ def _do_import_m3u_streams(self, settings, logger): streams = self._fetch_streams_from_m3u_sources(settings, logger) if not streams: - return {"status": "error", "message": "No streams found in specified M3U sources"} + return {"status": "error", "error": "No streams found in specified M3U sources"} # Step 2: Match streams to categories matched_by_category, unmatched_streams = self._match_streams_to_categories( @@ -2672,7 +3394,7 @@ def _do_import_m3u_streams(self, settings, logger): if not matched_by_category: return { "status": "error", - "message": f"No streams matched to channel databases. {len(unmatched_streams)} unmatched streams." + "error": f"No streams matched to channel databases. {len(unmatched_streams)} unmatched streams." } # Step 3: Ensure category groups exist @@ -2709,29 +3431,43 @@ def _do_import_m3u_streams(self, settings, logger): total_success = sum(1 for imp in import_results['imports'] if imp['status'] == 'success') total_failed = sum(1 for imp in import_results['imports'] if imp['status'] == 'failed') - return { - "status": "success", - "message": f"✓ M3U import complete!\n\n" - f"Channels created: {total_success}\n" - f"Failed: {total_failed}\n" - f"Unmatched streams skipped: {len(unmatched_streams)}\n" - f"Categories: {len(categories)}\n\n" - f"Results exported to: {csv_filename}" - } + message = (f"✓ M3U import complete!\n\n" + f"Channels created: {total_success}\n" + f"Failed: {total_failed}\n" + f"Unmatched streams skipped: {len(unmatched_streams)}\n" + f"Categories: {len(categories)}\n\n" + f"Results exported to: {csv_filename}") + + # Only the COMPLETED import reports, not the dry run, so one import + # produces one report rather than two. The title says results, because + # _export_m3u_import_preview hardcodes the word preview into the export + # filename and header for both of its callers. + outcome = self._build_and_emit_report( + settings, logger, + title="M3U import results", + columns=self._M3U_REPORT_COLUMNS, + rows=self._m3u_report_rows(matched_by_category, unmatched_streams), + export_filename=csv_filename) + message += self._report_outcome_clause(outcome) + + result = {"status": "success", "message": message} + if outcome["blocking_error"]: + result["error"] = outcome["blocking_error"] + return result def _do_import_m3u_streams_bg(self, settings, logger): """Background wrapper for M3U import.""" try: result = self._do_import_m3u_streams(settings, logger) self._last_bg_result = result - msg = result.get("message", "Import complete.") + msg = result.get("message") or result.get("error", "Import complete.") logger.info(f"{PLUGIN_LOG_PREFIX} IMPORT COMPLETED: {msg}") send_websocket_update('updates', 'update', { "type": "plugin", "plugin": self.name, "message": msg }) except Exception as e: - self._last_bg_result = {"status": "error", "message": str(e)} + self._last_bg_result = {"status": "error", "error": str(e)} logger.exception(f"{PLUGIN_LOG_PREFIX} Import error: {e}") send_websocket_update('updates', 'update', { "type": "plugin", "plugin": self.name, @@ -2744,8 +3480,19 @@ def import_m3u_streams_action(self, settings, logger): if dry_run: return self.import_m3u_streams_dry_run_action(settings, logger) + # The real import runs in a background thread whose result the card + # never shows, so the destination must be validated BEFORE + # backgrounding or a refusal would be silently swallowed. + custom_group_name = (settings.get("m3u_custom_group_name") or "").strip() + if custom_group_name: + try: + self._check_group_destinations_not_ignored( + [custom_group_name], settings.get("ignore_groups")) + except GroupScopeError as exc: + return self._scope_error_return(exc) + if not self._try_start_thread(self._do_import_m3u_streams_bg, (copy.deepcopy(settings), logger)): - return {"status": "error", "message": "An operation is already running. Please wait for it to finish."} + return {"status": "error", "error": "An operation is already running. Please wait for it to finish."} return { "status": "ok", @@ -2794,6 +3541,89 @@ def validate_settings_action(self, settings, logger): validation_results.append(f"❌ DB error") error_count += 1 + # 2b. Group scope (ignore_groups exclusion) - the first group + # validation in this action. Kept to one capped line per branch so + # a wildcard exclusion matching many groups cannot blow the ~280 + # char toast budget. + # + # ignore_dupe_error is set only when 2b's failure comes from the + # ignore filter ITSELF (an unmatched token, or no groups exist at + # all) - those two GroupScopeError messages don't mention + # include_label, so _resolve_category_scope below would raise the + # BYTE-IDENTICAL text and double-print/double-count one + # misconfigured setting as "2 error(s)". The "excluded every + # group that '' selected" message DOES depend on + # include_label (process vs category can differ), so that one is + # deliberately NOT deduped here. + ignore_dupe_error = None + ignore_summary = None + try: + scope = self._resolve_process_scope(settings, logger) + # Report only the names that actually removed something from + # this run, never the raw ignored_names count - it is a + # SUPERSET of out_of_scope_names, so a wildcard matching + # nothing but already-out-of-scope groups would otherwise + # print the same name list twice in one message. + out_of_scope = set(scope.out_of_scope_names) + effective = [n for n in scope.ignored_names if n not in out_of_scope] + if effective: + # Also folded into the SUCCESS toast by the assembly below. + # Confirming what the exclusion actually resolved to is the + # reason it is surfaced here at all, and a clean run would + # otherwise report nothing but "OK". + ignore_summary = (f"Ignoring {len(effective)} group(s): " + f"{_format_capped_name_list(effective)}") + validation_results.append(f"✅ Ignore: {ignore_summary}") + if scope.out_of_scope_names: + # ONE warning for the whole condition, not one per name - + # this is benign (an ignore entry that already had no + # effect), and counting per-name made a healthy config + # read as "Validation completed with 10 warning(s)". + warning_count += 1 + # No name list here: the names are already logged by + # _resolve_group_scope, and a capped list on top of the + # `effective` line above (which already fired) was the + # single offender that pushed a real operator's message + # over Dispatcharr's ~280 char clip. + validation_results.append( + f"⚠️ Ignore: {len(scope.out_of_scope_names)} " + f"name{'' if len(scope.out_of_scope_names) == 1 else 's'} " + f"had no effect (already outside the selected scope)") + except GroupScopeError as exc: + # The same exception covers an unresolvable INCLUDE filter + # (e.g. a typo'd selected_groups) as well as an unresolvable + # ignore filter, and every other consumer of this exception + # surfaces its raw wording with no prefix - the message + # already names the setting it came from (e.g. "'Channel + # Groups to Process'"), so prepending "Ignore:" would + # mislabel an include-filter typo as an ignore problem. + validation_results.append(f"❌ {exc}") + error_count += 1 + exc_text = str(exc) + if ("Channel Groups to Ignore" in exc_text + and "excluded every group" not in exc_text): + ignore_dupe_error = exc_text + + # 2c. Category scope (category_groups) - without this, a + # category_groups typo, or an exclusion that empties the category + # scope, validated GREEN here and only failed RED on Organize by + # Category. Only resolved when the setting is non-blank, so the + # common case (no category filter configured) costs nothing; + # when configured it costs exactly one line either way, to stay + # inside the ~260-char regression budget alongside 2b. Skipped + # entirely when 2b already reported the identical ignore-filter + # failure (see ignore_dupe_error above) - re-resolving would just + # print the same complaint twice and report "2 error(s)" for one + # broken setting. + category_groups_str = (settings.get("category_groups") or "").strip() + if category_groups_str and ignore_dupe_error is None: + try: + self._resolve_category_scope(settings, logger) + validation_results.append("✅ Category: OK") + except GroupScopeError as exc: + validation_results.append(f"❌ {exc}") + error_count += 1 + # 3. M3U filters (only show count if configured) m3u_info = [] @@ -2819,25 +3649,70 @@ def validate_settings_action(self, settings, logger): if dry_run: validation_results.append("ℹ️ Dry Run: ON") - # Generate summary - if error_count == 0 and warning_count == 0: - validation_results.insert(0, "✅ All settings validated successfully!") - status = "success" - elif error_count == 0: - validation_results.insert(0, f"⚠️ Validation completed with {warning_count} warning(s)") - status = "success" - else: - validation_results.insert(0, f"❌ Validation failed: {error_count} error(s), {warning_count} warning(s)") - status = "error" - - validation_results.insert(1, "") - - message = "\n".join(validation_results) - - return { - "status": status, - "message": message - } + # 5. Emailed reports. Reported ONLY when something is wrong, which + # matches the errors-and-warnings-only contract of this action. Every + # line here MUST start with a recognised glyph: a line starting with + # anything else is dropped from the operator-facing output AND trips + # the bookkeeping-drift warning below on every single run. + bridge = self._notify_bridge() + for problem in bridge.unknown_setting_values(settings): + validation_results.append(f"{_VALIDATION_WARNING_GLYPH} {problem}") + warning_count += 1 + if bridge.is_enabled(settings): + for problem in self._newsflasharr_readiness(): + validation_results.append( + f"{_VALIDATION_WARNING_GLYPH} Emailed reports: {problem}") + warning_count += 1 + if self._get_m3u_account_names(logger) is None: + validation_results.append( + f"{_VALIDATION_WARNING_GLYPH} Emailed reports: the M3U " + "account name lookup is failing, so no report will be " + "built. Those names are what is removed from a report " + "before it is emailed.") + warning_count += 1 + + # Report ONLY what the operator has to act on. + # + # Dispatcharr renders `error` persistently at the bottom of the + # plugin card and `message` as a transient toast. Returning the + # whole readout in `error` therefore parked a wall of mostly-OK + # lines under the settings form on every failure. So: a failure + # returns the failing lines and nothing else, and a clean run says + # so in a toast and leaves nothing behind. + # + # Severity is read from the glyph each line is built with, which is + # the contract for every append above. `_VALIDATION_ERROR_GLYPH` + # and `_VALIDATION_WARNING_GLYPH` name it so a new line cannot + # quietly opt out, and the assertion below fails loudly if a + # counter was incremented without a matching line (or vice versa). + errors = [ln for ln in validation_results + if ln.startswith(_VALIDATION_ERROR_GLYPH)] + warnings = [ln for ln in validation_results + if ln.startswith(_VALIDATION_WARNING_GLYPH)] + + if len(errors) != error_count or len(warnings) != warning_count: + logger.warning( + f"{PLUGIN_LOG_PREFIX} validate_settings bookkeeping drift: " + f"{error_count} error_count vs {len(errors)} error line(s), " + f"{warning_count} warning_count vs {len(warnings)} warning line(s)" + ) + + if errors: + header = (f"Validation failed, {len(errors)} error(s)" + + (f" and {len(warnings)} warning(s)" if warnings else "") + + ":") + return {"status": "error", + "error": "\n".join([header] + errors + warnings)} + + suffix = f" {ignore_summary}." if ignore_summary else "" + + if warnings: + return {"status": "success", + "message": f"✅ Settings OK.{suffix}\n" + f"{len(warnings)} warning(s):\n" + "\n".join(warnings)} + + return {"status": "success", + "message": f"✅ All settings validated successfully.{suffix}"} except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error during settings validation: {e}") @@ -2845,7 +3720,7 @@ def validate_settings_action(self, settings, logger): traceback.print_exc() return { "status": "error", - "message": f"Validation error: {e}\n\nSee logs for details." + "error": f"Validation error: {e}\n\nSee logs for details." } def plugin_status_action(self, settings, logger): @@ -2891,4 +3766,4 @@ def clear_csv_exports_action(self, settings, logger): except Exception as e: logger.error(f"{PLUGIN_LOG_PREFIX} Error clearing CSV exports: {e}") - return {"status": "error", "message": f"Error clearing CSV exports: {e}"} + return {"status": "error", "error": f"Error clearing CSV exports: {e}"} diff --git a/plugins/channel-mapparr/reports.py b/plugins/channel-mapparr/reports.py new file mode 100644 index 0000000..7235c10 --- /dev/null +++ b/plugins/channel-mapparr/reports.py @@ -0,0 +1,455 @@ +"""Report model and rendering for Channel-Maparr's emailed reports. + +Newsflasharr sends an attachment verbatim and unredacted, so anything reaching +this model can leave the box by email. Two structural decisions follow from that, +and both are pinned by tests: + +The model is built by COPYING a named allow-list of columns out of the row dicts +the actions already hold in memory. It is never built by reading a CSV export. +The exports in /data/exports open with a settings header that names the +configured M3U sources, which on a real installation is the provider hostname, +so a report built by re-reading an export would carry that hostname by +construction. Building from an allow-list also means a column added to a CSV +writer later cannot start being emailed on its own. + +The account-name scrub is the PRIMARY redaction input here, not a backstop. In +Stream-Mapparr the raw stream names never carried an account label and the scrub +was a second line of defence, so an empty account list degraded safely. That is +not true here, so build_model refuses when the account name list is None, which +is how a caller reports that the lookup failed. +""" +import csv +import datetime +import html +import io +import os +import re +import time + +# Reports are written here, deliberately NOT under /data/logos: Dispatcharr's +# nginx serves that tree unauthenticated to the entire local network. +REPORT_DIR = "/data/channel_mapparr_reports" + +# Every report file starts with this. The pruner matches on it, so a copied and +# unrenamed prefix would make pruning match nothing and the directory would grow +# forever with no error. +FILENAME_PREFIX = "channel_mapparr_report_" + +# How many of each file type to keep. +KEEP_REPORTS = 8 + +# Newsflasharr re-reads an attachment path on every delivery retry, across a +# documented worst case of 30 + 300 + 1800 seconds. A report file younger than +# this is never pruned, however many newer ones exist, because deleting it would +# strip the attachment from mail that is already queued. +RETRY_WINDOW_SECONDS = 2400 + +# The row cap, applied ONCE before rendering. A render, measure, drop rows, +# re-render loop is deliberately not used: an M3U import can carry seventeen +# thousand rows, and three of the four paths that build a report run inside the +# uWSGI request, where a pure Python loop performs no input or output and so +# never yields. Under gevent that freezes the whole worker, not just the request +# that started it. +# +# At a measured density of about 136 bytes per CSV row this is roughly 270 KB of +# CSV, well inside Newsflasharr's 1048576 byte attachment cap, and the larger +# HTML rendering still fits. Anyone raising this constant must measure in UTF-8 +# BYTES of the written file, not characters: the matcher supports Cyrillic and +# CJK names, which are two to three times longer in bytes than in characters. +MAX_REPORT_ROWS = 2000 + +# The container path of the exports. Named in the report so a reader knows where +# the complete, unredacted file is. It is a locator, not a browsable link, and it +# is never a Windows drive mapping. +EXPORTS_LOCATION = "/data/exports" + +# The only settings that reach the report. Everything else is dropped, including +# m3u_sources, which is the measured leak, and default_logo, which is an operator +# typed URL and the most likely route for a local network address to enter this +# plugin at all. +_SAFE_SETTINGS = ( + ("dry_run_mode", "Dry run"), + ("match_sensitivity", "Match sensitivity"), +) + +# Matches a bare IPv4 address. +_IPV4_RE = re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}\b") + +# Matches an IPv6 address, including the compressed forms. Deliberately requires +# either a double colon or at least three colon separated groups, so an ordinary +# clock time such as 20:30 is not mistaken for an address. +_IPV6_RE = re.compile( + r"\b(?:[0-9A-Fa-f]{1,4}:){2,7}[0-9A-Fa-f]{1,4}\b" + r"|(?:[0-9A-Fa-f]{1,4}:)+:(?:[0-9A-Fa-f]{1,4})?" + r"|::(?:[0-9A-Fa-f]{1,4}:)*[0-9A-Fa-f]{1,4}" +) + +# Collapses the run of spaces left behind when a value is removed from the middle +# of a name. +_MULTISPACE_RE = re.compile(r"\s{2,}") + + +def sanitise_label(label, account_names): + """Remove an M3U account name wherever it appears, and nothing else. + + Matching is case insensitive and is not limited to the bracketed form, + because an account name on a real installation is a literal provider + hostname and "ESPN backup provider.tv" leaks exactly as much as + "ESPN [provider.tv]". + + Account names are matched longest first: "provider.tv" is a prefix of + "provider.tv-alt1", and matching the shorter one first would leave a "-alt1" + fragment behind. + + An unknown bracketed value is left alone. On this installation a bracketed + value in a channel name holds the market, and for an over the air station the + market is its whole identity, so removing every bracketed group would collapse + dozens of distinct stations into one indistinguishable name. + """ + text = str(label if label is not None else "") + for account in sorted([a for a in (account_names or []) if a], key=len, reverse=True): + escaped = re.escape(str(account)) + text = re.sub(r"\s*\[" + escaped + r"\]", "", text, flags=re.IGNORECASE) + text = re.sub(r"\s*\(" + escaped + r"\)", "", text, flags=re.IGNORECASE) + text = re.sub(escaped, "", text, flags=re.IGNORECASE) + return _MULTISPACE_RE.sub(" ", text).strip() + + +def _scrub(value, account_names): + """Apply every content rule to one free text value.""" + cleaned = sanitise_label(value, account_names) + cleaned = _IPV4_RE.sub("", cleaned) + cleaned = _IPV6_RE.sub("", cleaned) + return _MULTISPACE_RE.sub(" ", cleaned).strip() + + +def build_model(title, columns, rows, *, account_names, settings, databases, + version, now, export_filename=None): + """Build the report model from in-memory rows. + + `columns` is a list of (row key, display header) pairs and IS the allow list. + A key absent from it is never copied, whatever the row carries. + + `account_names` of None means the M3U account lookup failed, and this raises + rather than sending an unscrubbed report. An empty list is a different thing + and is allowed: an installation can legitimately have no M3U accounts. + """ + if account_names is None: + raise ValueError( + "account_names is None, which means the M3U account lookup failed. " + "The report is not built, because the account names are the primary " + "redaction input and an empty scrub would ship unredacted names.") + + rows = list(rows or []) + total_rows = len(rows) + kept = rows[:MAX_REPORT_ROWS] + + entries = [] + for row in kept: + entries.append([_scrub(row.get(key), account_names) for key, _ in columns]) + + settings = settings if isinstance(settings, dict) else {} + summary = [("Plugin version", str(version)), + ("Generated", _fmt_ts(now)), + ("Databases loaded", ", ".join(str(d) for d in (databases or [])) or "none")] + for key, label in _SAFE_SETTINGS: + summary.append((label, str(settings.get(key, "")))) + summary.append(("Rows", f"{len(entries)} of {total_rows}")) + + return { + "title": str(title), + "generated_ts": float(now or 0), + "summary": summary, + "headers": [header for _, header in columns], + "entries": entries, + "total_rows": total_rows, + "shown_rows": len(entries), + "truncated": total_rows > len(entries), + "export_filename": export_filename, + } + + +def truncation_notice(model): + """The one line stating that rows were dropped, or None when none were. + + It names a count and a filename only. Never the M3U source, never the group + scope, and never a Windows drive mapping. + """ + if not model.get("truncated"): + return None + name = model.get("export_filename") or "the export file" + return (f"Showing the first {model['shown_rows']} of {model['total_rows']} rows. " + f"The complete file is {name} in {EXPORTS_LOCATION} inside the container.") + + +# --------------------------------------------------------------------------- # +# Rendering +# --------------------------------------------------------------------------- # + +# Styling is inlined because the page is read from a file path or inside an email +# client, where an external stylesheet would not resolve. Colours and layout +# follow Stream-Mapparr's report so the two look like one family. +_CSS = """ +:root { color-scheme: light dark; --accent: #2a78d6; } +body { font: 15px/1.5 system-ui, -apple-system, Segoe UI, sans-serif; + margin: 0; padding: 24px; background: #fbfbfd; color: #16181d; } +@media (prefers-color-scheme: dark) { + :root { --accent: #3987e5; } + body { background: #14161a; color: #e8eaed; } + th { background: #1e2127 !important; } + tr:nth-child(even) td { background: #191c21; } + .card { background: #1a1d22 !important; border-color: #2a2e35 !important; } + .notice { background: #2a2410 !important; border-color: #5a4a18 !important; } +} +h1 { font-size: 22px; margin: 0 0 4px; } +.sub { opacity: .7; font-size: 15px; margin-bottom: 20px; } +.card { background: #fff; border: 1px solid #e3e5ea; border-radius: 10px; + padding: 14px 16px; margin-bottom: 18px; } +.notice { background: #fff8e1; border: 1px solid #e8d9a0; border-radius: 8px; + padding: 10px 14px; margin-bottom: 18px; font-size: 14px; } +table { border-collapse: collapse; width: 100%; font-size: 15px; } +.scroll { overflow-x: auto; } +th, td { text-align: left; padding: 6px 10px; border-bottom: 1px solid #e6e8ec; + vertical-align: top; } +th { background: #f2f3f6; } +th.sortable { cursor: pointer; user-select: none; white-space: nowrap; } +th.sortable::after { content: " \\2195"; opacity: .35; font-size: 12px; } +th.sortable[aria-sort="ascending"]::after { content: " \\2191"; opacity: 1; } +th.sortable[aria-sort="descending"]::after { content: " \\2193"; opacity: 1; } +.empty { opacity: .7; font-style: italic; } +.note { font-size: 14px; opacity: .7; margin-top: 20px; } +dl.meta { margin: 0; display: grid; grid-template-columns: auto 1fr; gap: 2px 14px; } +dl.meta dt { opacity: .7; } +dl.meta dd { margin: 0; } +""" + + +def _esc(value): + return html.escape(str(value if value is not None else ""), quote=True) + + +def _fmt_ts(ts): + """Render a time in UTC, labelled as such. + + Deliberately not local time: this module has no access to Dispatcharr's + configured timezone, and a bare unlabelled clock that silently means UTC is + how a reader gets the day wrong. + """ + try: + moment = datetime.datetime.fromtimestamp(float(ts or 0), datetime.timezone.utc) + return moment.strftime("%Y-%m-%d %H:%M UTC") + except (TypeError, ValueError, OSError, OverflowError): + return "unknown" + + +# Click to sort, embedded in the page rather than loaded from anywhere. The page +# is opened from a file path or from a mail attachment, so an external request +# would not resolve and would disclose that the report had been opened. +# +# This is an ADDITION, not a requirement. Every row is present in the markup, so +# a reader whose mail client strips scripts still sees the whole table; they +# simply cannot reorder it. Mail clients do strip scripts, so sorting works when +# the attachment is saved and opened in a browser, which is the ordinary way to +# read an HTML attachment. +# +# The comparison reads each cell's data-v attribute, which holds the same value +# the cell displays. Two values that both parse as numbers are compared as +# numbers, because comparing them as text puts 10 before 2. +_SORT_SCRIPT = """ +(function () { + var table = document.querySelector('table'); + if (!table || !table.tBodies.length) { return; } + var headers = [].slice.call(table.querySelectorAll('th.sortable')); + function value(row, index) { + var cell = row.children[index]; + if (!cell) { return ''; } + var raw = cell.getAttribute('data-v'); + return raw === null ? cell.textContent : raw; + } + function compare(a, b, index) { + var x = value(a, index), y = value(b, index); + var nx = Number(x), ny = Number(y); + if (x !== '' && y !== '' && !isNaN(nx) && !isNaN(ny)) { return nx - ny; } + return String(x).localeCompare(String(y), undefined, + { numeric: true, sensitivity: 'base' }); + } + function apply(header, index) { + var ascending = header.getAttribute('aria-sort') !== 'ascending'; + headers.forEach(function (other) { other.setAttribute('aria-sort', 'none'); }); + header.setAttribute('aria-sort', ascending ? 'ascending' : 'descending'); + var body = table.tBodies[0]; + var rows = [].slice.call(body.rows); + rows.sort(function (a, b) { + return ascending ? compare(a, b, index) : compare(b, a, index); + }); + rows.forEach(function (row) { body.appendChild(row); }); + } + headers.forEach(function (header, index) { + header.addEventListener('click', function () { apply(header, index); }); + header.addEventListener('keydown', function (event) { + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault(); + apply(header, index); + } + }); + }); +})(); +""" + + +def render_html(model): + """Render the model to one self contained HTML page. + + The table is sortable by clicking or keyboard-activating a column header. + See _SORT_SCRIPT for what that does and what it deliberately does not do. + """ + rows = [] + for entry in model.get("entries") or []: + cells = "".join(f"{_esc(cell)}" for cell in entry) + rows.append(f"{cells}") + headers = "".join( + "{_esc(h)}" + for h in model.get("headers") or []) + if rows: + table = ("
" + "" + headers + "" + "" + "".join(rows) + "" + "
") + else: + table = "

This run produced no rows.

" + + meta = "".join(f"
{_esc(k)}
{_esc(v)}
" + for k, v in model.get("summary") or []) + + notice = truncation_notice(model) + notice_html = f"
{_esc(notice)}
\n" if notice else "" + + return ( + "\n\n\n" + "\n" + "\n" + f"Channel-Maparr: {_esc(model.get('title'))}\n" + f"\n\n\n" + f"

Channel-Maparr: {_esc(model.get('title'))}

\n" + f"
{_esc(model.get('shown_rows', 0))} row(s) shown
\n" + + notice_html + + f"
{meta}
\n" + f"
{table}
\n" + "

Click a column heading to sort by it, or focus it and " + "press Enter. Sorting needs the page open in a browser; a mail client " + "previewing this file shows every row but cannot reorder them.

\n" + "

Names in this report are shown without their M3U source " + "label, and the plugin settings that name your M3U sources are not " + f"included. The complete export, which does include them, stays in " + f"{EXPORTS_LOCATION} inside the container and is not emailed.

\n" + f"\n" + "\n\n" + ) + + +# A cell beginning with any of these is evaluated as a formula by Excel, +# LibreOffice and Google Sheets when the file is opened. +_FORMULA_LEADS = ("=", "+", "-", "@") + + +def _csv_safe(value): + """Prefix a formula shaped cell with an apostrophe so it stays text.""" + text = str(value if value is not None else "") + if text[:1] in _FORMULA_LEADS: + return "'" + text + return text + + +def render_csv(model): + """Render the model to CSV text, with the same content rules as the HTML.""" + buf = io.StringIO() + writer = csv.writer(buf, lineterminator="\n") + writer.writerow([f"# Channel-Maparr: {model.get('title')}"]) + for key, value in model.get("summary") or []: + writer.writerow([f"# {key}: {value}"]) + notice = truncation_notice(model) + if notice: + writer.writerow([f"# {notice}"]) + writer.writerow([]) + writer.writerow(list(model.get("headers") or [])) + for entry in model.get("entries") or []: + writer.writerow([_csv_safe(cell) for cell in entry]) + return buf.getvalue() + + +# --------------------------------------------------------------------------- # +# Writing +# --------------------------------------------------------------------------- # + +def _atomic_write(path, text): + """Write through a temporary file and rename, so no partial file is ever + visible at the destination path.""" + tmp = f"{path}.tmp-{os.getpid()}" + try: + with open(tmp, "w", encoding="utf-8", newline="") as handle: + handle.write(text) + os.replace(tmp, path) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def _prune(dirpath, suffix, keep=KEEP_REPORTS, now=None): + """Keep the newest `keep` report files with this suffix and delete the rest, + except any file still young enough that a delivery retry could need it. + + The age guard is required, not defensive tidiness. Newsflasharr copies + nothing: it re-reads the attachment path on every retry attempt across a + worst case of 2130 seconds. Several actions run back to back produce several + report pairs within minutes and can push an earlier one past the keep count + while its mail is still being retried. The result would be an email arriving + with its attachment missing, which Newsflasharr records as a degrade rather + than an error. + + Never raises: losing an old report must not fail the run that produced a new + one. + """ + try: + moment = time.time() if now is None else now + entries = [os.path.join(dirpath, name) for name in os.listdir(dirpath) + if name.startswith(FILENAME_PREFIX) and name.endswith(suffix)] + entries.sort(key=os.path.getmtime, reverse=True) + for stale in entries[keep:]: + try: + if moment - os.path.getmtime(stale) < RETRY_WINDOW_SECONDS: + continue # a queued delivery may still re-read this path + os.unlink(stale) + except OSError: + pass + except OSError: + pass + + +def write_report(model, report_dir, now): + """Write the HTML and CSV reports and return their paths. + + Returns {"html_path", "csv_path", "error"}. Never raises: reporting is not + the plugin's real work, and a failure here is reported rather than thrown. + + Both files carry the run's timestamp in their name and are never rewritten, + because an email send re-reads the attachment path on every retry attempt. + """ + result = {"html_path": None, "csv_path": None, "error": None} + try: + os.makedirs(report_dir, exist_ok=True) + stamp = datetime.datetime.fromtimestamp( + float(now or 0), datetime.timezone.utc).strftime("%Y%m%d_%H%M%S") + base = os.path.join(report_dir, f"{FILENAME_PREFIX}{stamp}") + html_path, csv_path = base + ".html", base + ".csv" + _atomic_write(html_path, render_html(model)) + _atomic_write(csv_path, render_csv(model)) + result["html_path"], result["csv_path"] = html_path, csv_path + _prune(report_dir, ".html") + _prune(report_dir, ".csv") + except Exception as error: + result["error"] = f"could not write the report: {error}" + return result diff --git a/plugins/channel-mapparr/wildcard_match.py b/plugins/channel-mapparr/wildcard_match.py new file mode 100644 index 0000000..e05d522 --- /dev/null +++ b/plugins/channel-mapparr/wildcard_match.py @@ -0,0 +1,46 @@ +"""Pure, Django-free glob matching for plugin list settings. + +Lives outside plugin.py so the offline unittest harness (which prepends +EPG-Janitor/ to sys.path) can import and test it without Dispatcharr/Django. +""" +import fnmatch + + +def expand_patterns(tokens, available_names, ci_plain): + """Resolve user tokens (some glob, some literal) against available_names. + + A token containing '*' or '?' is a glob, matched case-insensitively via + fnmatch.fnmatchcase on lowercased strings. Any other token is a literal: + case-insensitive when ci_plain is True, else case-sensitive exact. + + Returns (matched_names, unmatched_tokens): + - matched_names: names matching >=1 token, ordered by + (lowest matching token index, then original available_names order), + de-duplicated. + - unmatched_tokens: tokens that matched no name, in input order. + """ + avail = list(available_names) + avail_order = {name: i for i, name in enumerate(avail)} + matched_idx = {} # name -> lowest token index that matched it + matched_token_idx = set() # token indices that matched >=1 name + + for ti, tok in enumerate(tokens): + is_glob = ("*" in tok) or ("?" in tok) + tok_l = tok.lower() + for name in avail: + if is_glob: + hit = fnmatch.fnmatchcase(name.lower(), tok_l) + elif ci_plain: + hit = name.lower() == tok_l + else: + hit = name == tok + if hit: + matched_token_idx.add(ti) + if name not in matched_idx or ti < matched_idx[name]: + matched_idx[name] = ti + + matched_names = sorted( + matched_idx, key=lambda n: (matched_idx[n], avail_order[n])) + unmatched_tokens = [t for i, t in enumerate(tokens) + if i not in matched_token_idx] + return matched_names, unmatched_tokens