diff --git a/model_validation/README.md b/model_validation/README.md index 8c3412e5..4b0b1844 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -293,14 +293,28 @@ it. Every property is optional, so a block sets only what it changes. synthesized_inputs: audio: sample_rate: 48000 # default 44100 - channels: 2 # default 1 + num_channels: 2 # default 1 duration: 5.0 # default 2.0 ext: .flac # default .wav midi: num_notes: 8 # default 2 note_duration: 0.25 # default 0.5 + instrument: 24 # default 0 + channel: 0 # default 0 ``` +The MIDI `instrument` is the General MIDI program number, written as a program +change, and it is where both HARP and a model read a note's instrument from. +A model that processes only one instrument family needs it set to a program it +accepts, since the default of 0 is a grand piano. + +Channels are counted from 0 here, matching the MIDI wire format, so they run 0 +to 15 wherever validation reads or reports one. Most DAWs count the same +channels from 1, which is why the General MIDI drum channel is 9 here and is +the channel commonly called "channel 10". Both `instrument` and `channel` are +checked as configuration, so a value outside its range is reported before any +model runs rather than being written into a file the model cannot parse. + Audio and MIDI are the only media a block can configure, because they are the only inputs built from properties. A generic file input (`gr.File`) accepting `.txt` or `.json` is handed a fixed placeholder file, which has nothing worth @@ -403,7 +417,7 @@ with no rules is still subject to the default structural checks: "Output Audio": ext: .wav # extension: a string, or a list of accepted ones min_bytes: 10000 # minimum file size - channels: 1 # exact channel count (1 = mono, 2 = stereo) + num_channels: 1 # exact channel count (1 = mono, 2 = stereo) sample_rate: 44100 # exact sample rate, in Hz min_duration: 1.5 # minimum length, in seconds max_duration: 10.0 # maximum length, in seconds @@ -419,14 +433,24 @@ The following is the full vocabulary along with which output types each rule cov |---|---|---| | `ext` | any file output | Extension matches (string, or list of accepted extensions) | | `min_bytes` | any file output | File is at least this many bytes | -| `channels` | audio output | Exact channel count | +| `num_channels` | audio output | Exact channel count | | `sample_rate` | audio output | Exact sample rate in Hz | | `bit_depth` | audio output | Exact PCM bit depth (16, 24, ...), and errors on compressed formats such as MP3 or OGG | | `min_rms_db` | audio output | RMS level is at least this many dBFS | | `min_duration` / `max_duration` | audio or MIDI output | Length in seconds is within bounds | | `min_notes` | MIDI output | At least this many note-on events | +| `note_instruments` | MIDI output | Notes use only these General MIDI program numbers (one, or a list of accepted) | +| `note_channels` | MIDI output | Notes use only these channels (one, or a list of accepted) | | `min_labels` | JSON (`LabelList`) output | At least this many labels returned | +`note_instruments` and `note_channels` assert coverage rather than presence. +Every instrument or channel the notes carry has to be one the rule lists, and +the rule may list values the file never uses. A file with no notes satisfies +both, since requiring notes is what `min_notes` is for. They name the notes +because a MIDI file has no single instrument or channel of its own, only notes +that each use one, and the prefix keeps them from reading as counts the way +`num_channels` does. + **Targeting outputs.** A model with several outputs gets one block per output label, each checked independently. Use `"*"` instead of a label to apply rules to every output a rule covers. With mixed outputs, `"*"` sends `min_bytes` to diff --git a/model_validation/config.yml b/model_validation/config.yml index ac68343f..ccde4b2a 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -53,13 +53,15 @@ # # needed to depart from them # audio: # sample_rate: 44100 # Hz -# channels: 1 # 1 = mono, 2 = stereo +# num_channels: 1 # 1 = mono, 2 = stereo # duration: 2.0 # seconds # ext: .wav # any format this libsndfile build can write. # # One it cannot write is a configuration error # midi: # num_notes: 2 # ascending notes from middle C # note_duration: 0.5 # seconds per note, at 120 BPM +# instrument: 0 # General MIDI program number, 0 to 127 +# channel: 0 # MIDI channel, 0 to 15 (9 = drums) # ext: .mid # # This block can also be set per model and per test case, with each level @@ -135,14 +137,16 @@ # "Output Audio": # ext: .wav # extension: one, or a list of accepted # min_bytes: 1000 # minimum file size, in bytes -# channels: 1 # exact channel count +# num_channels: 1 # exact channel count # sample_rate: 44100 # exact sample rate, in Hz # bit_depth: 16 # exact PCM bit depth (PCM formats only) # min_rms_db: -60 # minimum RMS level, in dBFS # min_duration: 1.5 # minimum length, in seconds # max_duration: 10.0 # maximum length, in seconds # "Output Midi": -# min_notes: 1 # minimum note-on count +# min_notes: 1 # minimum note-on count +# note_instruments: 24 # program number(s) the notes may use +# note_channels: 0 # channel(s) the notes may use # "Output Labels": # min_labels: 1 # minimum labels in a pyharp LabelList. 0 # # permits an empty list but still requires @@ -160,9 +164,10 @@ # Which expect rules apply to which output types: # # ext, min_bytes any file output -# channels, sample_rate, bit_depth, min_rms_db audio outputs +# num_channels, sample_rate, bit_depth, +# min_rms_db audio outputs # min_duration, max_duration audio and MIDI outputs -# min_notes MIDI outputs +# min_notes, note_instruments, note_channels MIDI outputs # min_labels JSON (LabelList) outputs # # Applying a rule to a NAMED output whose type it does not cover fails the diff --git a/model_validation/src/assets.py b/model_validation/src/assets.py index 9f2b5322..9510d2e4 100644 --- a/model_validation/src/assets.py +++ b/model_validation/src/assets.py @@ -35,16 +35,36 @@ # Configurable properties of synthesized inputs, with their default values AUDIO_DEFAULTS = { "sample_rate": 44100, # Hz - "channels": 1, # 1 = mono, 2 = stereo, ... + "num_channels": 1, # 1 = mono, 2 = stereo, ... "duration": 2.0, # seconds "ext": ".wav", # container/format to write } MIDI_DEFAULTS = { "num_notes": 2, # ascending notes from middle C "note_duration": 0.5, # seconds per note (at the default 120 BPM) + "instrument": 0, # General MIDI program number (0 = grand piano) + "channel": 0, # MIDI channel (see MIDI_DRUM_CHANNEL) "ext": ".mid", } +# Channels are numbered 0 to 15 throughout validation, as they are on the +# wire and in mido, rather than the 1 to 16 a DAW displays. The two namings +# are one apart, which is why the General MIDI drum channel is 9 here and is +# the same channel widely called "channel 10". HARP counts from 1 internally +# (juce::MidiMessage::getChannel), so a note written here on channel 9 is the +# one HARP reports as channel 10 and marks as drums. +MIDI_DRUM_CHANNEL = 9 + +# The MIDI properties written into the file as numbers, with the range each +# accepts and how to describe it. Both land in bytes the MIDI spec reserves +# for a limited range, so an out-of-range value corrupts the stream rather +# than failing on the way out, which is why they are checked as configuration. +MIDI_RANGES = { + "instrument": (0, 127, "a General MIDI program number (0 is a grand " + "piano, 24 a nylon guitar)"), + "channel": (0, 15, "a MIDI channel counted from 0"), +} + # The media kinds a `synthesized_inputs` block can configure. The text and # JSON placeholders below are deliberately absent, as they only ever fill a # generic file input and have nothing worth varying. @@ -127,6 +147,30 @@ def synthesized_input_blocks(config: dict) -> list: if (block := (mapping or {}).get("synthesized_inputs"))] +def check_midi_number(prop: str, value, where: str) -> None: + """ + Reject a numeric MIDI property that falls outside the range it accepts. + + Args: + prop (str): The property being checked, a key of MIDI_RANGES. + value: The configured value. + where (str): Where it was written, for reporting. + + Raises: + ValueError: If the value is not a whole number within the range. + """ + + low, high, description = MIDI_RANGES[prop] + + # bool is a subclass of int, so `channel: true` would otherwise pass as + # channel 1 rather than being reported as the mistake it is + if isinstance(value, bool) or not isinstance(value, int) \ + or not low <= value <= high: + raise ValueError( + f"synthesized_inputs midi {prop} {value!r} (set in {where}) is " + f"not {description}. Give a whole number from {low} to {high}") + + def check_synthesized_inputs(config: dict) -> None: """ Reject `synthesized_inputs` settings that nothing would act on. @@ -134,10 +178,11 @@ def check_synthesized_inputs(config: dict) -> None: Only audio and MIDI inputs are built from properties. Text and JSON inputs are fixed placeholder files, and any other input a model declares has to come from a test case's `files` entry. An unrecognized media kind or - property name would therefore be read by nothing at all, and an audio - format this installation cannot write would leave models without inputs. - Reporting all three here turns them into one configuration error naming - the setting, rather than a set of models with skipped test cases. + property name would therefore be read by nothing at all, an audio format + this installation cannot write would leave models without inputs, and an + out-of-range MIDI number would corrupt the file it is written into. + Reporting them here turns each into one configuration error naming the + setting, rather than a set of models with skipped or misleading cases. Args: config (dict): Parsed configuration. @@ -167,6 +212,11 @@ def check_synthesized_inputs(config: dict) -> None: if kind == "audio" and (props or {}).get("ext"): exts.setdefault(str(props["ext"]), where) + if kind == "midi": + for prop in MIDI_RANGES: + if (props or {}).get(prop) is not None: + check_midi_number(prop, props[prop], where) + unwritable = {ext: where for ext, where in exts.items() if not can_write_audio(ext)} @@ -296,15 +346,16 @@ def audio(self, overrides: dict | None = None, ext: str | None = None) -> Path | props = self.resolve("audio", overrides) sample_rate = int(props["sample_rate"]) - channels = int(props["channels"]) + num_channels = int(props["num_channels"]) duration = float(props["duration"]) ext = (ext or props["ext"]).lower() - path = (self.workdir / - f"test_input_{sample_rate}hz_{channels}ch_{duration:g}s{ext}") + path = (self.workdir / f"test_input_{sample_rate}hz_{num_channels}ch" + f"_{duration:g}s{ext}") - return self.cached(("audio", sample_rate, channels, duration, ext), - lambda: write_audio(path, duration, sample_rate, channels)) + return self.cached(("audio", sample_rate, num_channels, duration, ext), + lambda: write_audio(path, duration, sample_rate, + num_channels)) def midi(self, overrides: dict | None = None, ext: str | None = None) -> Path: """ @@ -321,12 +372,18 @@ def midi(self, overrides: dict | None = None, ext: str | None = None) -> Path: props = self.resolve("midi", overrides) num_notes = int(props["num_notes"]) note_duration = float(props["note_duration"]) + instrument = int(props["instrument"]) + channel = int(props["channel"]) ext = (ext or props["ext"]).lower() - path = self.workdir / f"test_input_{num_notes}n_{note_duration:g}s{ext}" + path = (self.workdir / + f"test_input_{num_notes}n_{note_duration:g}s" + f"_inst{instrument}_ch{channel}{ext}") - return self.cached(("midi", num_notes, note_duration, ext), - lambda: write_midi(path, num_notes, note_duration)) + return self.cached( + ("midi", num_notes, note_duration, instrument, channel, ext), + lambda: write_midi(path, num_notes, note_duration, instrument, + channel)) def for_file_types(self, file_types: list, overrides: dict | None = None) -> Path | None: @@ -400,7 +457,7 @@ def write_bytes(path: Path, data: bytes) -> Path: def write_audio(path: Path, duration: float, sample_rate: int, - channels: int) -> Path: + num_channels: int) -> Path: """ Write a sine sweep, which is a valid input for any audio model. @@ -408,7 +465,7 @@ def write_audio(path: Path, duration: float, sample_rate: int, path (Path): Destination path, whose extension selects the format. duration (float): Length of the sweep in seconds. sample_rate (int): Sample rate in Hz. - channels (int): Number of (identical) channels to write. + num_channels (int): Number of (identical) channels to write. Returns: path (Path): The written file, for chaining. @@ -419,12 +476,14 @@ def write_audio(path: Path, duration: float, sample_rate: int, # Sweep from 220 Hz up one octave over the clip freq = 220.0 + 220.0 * numpy.arange(n) / n mono = 0.5 * numpy.sin(2 * numpy.pi * freq * t) - soundfile.write(str(path), numpy.tile(mono[:, None], (1, channels)), sample_rate) + soundfile.write(str(path), numpy.tile(mono[:, None], (1, num_channels)), + sample_rate) return path -def write_midi(path: Path, num_notes: int, note_duration: float) -> Path: +def write_midi(path: Path, num_notes: int, note_duration: float, + instrument: int = 0, channel: int = 0) -> Path: """ Write a standard MIDI file (format 0) of ascending notes from middle C. @@ -432,6 +491,8 @@ def write_midi(path: Path, num_notes: int, note_duration: float) -> Path: path (Path): Destination .mid path. num_notes (int): Number of notes to write. note_duration (float): Seconds each note sounds, at 120 BPM. + instrument (int): General MIDI program number the notes are played with + channel (int): MIDI channel the notes are written on, 0 to 15. Returns: path (Path): The written file, for chaining. @@ -448,12 +509,13 @@ def delta(value): value >>= 7 return bytes(out) - events = bytearray([0x00, 0xC0, 0x00]) # program change: grand piano + # The channel numbering matches the low nibble of a status byte directly + events = bytearray([0x00, 0xC0 | channel, instrument]) # program change for i in range(max(0, num_notes)): - pitch = 60 + (i * 2) % 24 # ascending from middle C - events += bytes([0x00, 0x90, pitch, 0x64]) # note on - events += delta(ticks) + bytes([0x80, pitch, 0x40]) - events += bytes([0x00, 0xFF, 0x2F, 0x00]) # end of track + pitch = 60 + (i * 2) % 24 # from middle C + events += bytes([0x00, 0x90 | channel, pitch, 0x64]) # note on + events += delta(ticks) + bytes([0x80 | channel, pitch, 0x40]) + events += bytes([0x00, 0xFF, 0x2F, 0x00]) # end of track header = b"MThd" + struct.pack(">IHHH", 6, 0, 1, MIDI_TICKS_PER_BEAT) track = b"MTrk" + struct.pack(">I", len(events)) + bytes(events) diff --git a/model_validation/src/audio.py b/model_validation/src/audio.py index c934700d..484746aa 100644 --- a/model_validation/src/audio.py +++ b/model_validation/src/audio.py @@ -65,10 +65,10 @@ def read_audio_props(label: str, path: str) -> dict: path (str): Path to the audio file. Returns: - props (dict): channels, sample_rate, duration (seconds), rms_db (RMS - level in dBFS, or -inf for digital silence), subtype (libsndfile - encoding name), and bit_depth (int, or None for compressed - encodings). + props (dict): num_channels, sample_rate, duration (seconds), rms_db + (RMS level in dBFS, or -inf for digital silence), subtype + (libsndfile encoding name), and bit_depth (int, or None for + compressed encodings). Raises: AssertionError: If the file cannot be decoded or contains no audio. @@ -95,7 +95,7 @@ def read_audio_props(label: str, path: str) -> dict: rms = math.sqrt(float((data * data).sum()) / data.size) return { - "channels": channels, + "num_channels": channels, "sample_rate": sample_rate, "duration": frames / sample_rate, "rms_db": 20 * math.log10(rms) if rms > 0 else float("-inf"), diff --git a/model_validation/src/expectations.py b/model_validation/src/expectations.py index 9486f983..3cba69e8 100644 --- a/model_validation/src/expectations.py +++ b/model_validation/src/expectations.py @@ -36,20 +36,29 @@ "min_bytes": FILE_TYPES, "min_duration": AUDIO | MIDI, "max_duration": AUDIO | MIDI, - "channels": AUDIO, + "num_channels": AUDIO, "sample_rate": AUDIO, "bit_depth": AUDIO, "min_rms_db": AUDIO, "min_notes": MIDI, + "note_instruments": MIDI, + "note_channels": MIDI, "min_labels": JSON, } +# The note_* rules, mapped to the decoded property each reads. Both name the +# notes rather than the file, since a MIDI file carries no single instrument +# or channel of its own, only notes that each use one. +NOTE_VALUE_RULES = {"note_instruments": "instruments", + "note_channels": "channels"} + # Rule groups, by what they need decoded. Duration rules are shared: they read # from whichever of audio/MIDI props matches the output's type. DURATION_RULES = {"min_duration", "max_duration"} DECODED_RULES = { - "audio_track": {"channels", "sample_rate", "bit_depth", "min_rms_db"} | DURATION_RULES, - "midi_track": {"min_notes"} | DURATION_RULES, + "audio_track": {"num_channels", "sample_rate", "bit_depth", + "min_rms_db"} | DURATION_RULES, + "midi_track": {"min_notes"} | set(NOTE_VALUE_RULES) | DURATION_RULES, } # Key selecting every compatible output rather than one named output @@ -138,6 +147,34 @@ def resolve_expect_targets(expect: dict, out_types: dict) -> list: return targets +def check_note_values(label: str, rule: str, found: list, allowed) -> None: + """ + Assert the notes use only the instruments or channels a rule permits. + + The check is one of coverage rather than presence: every value the notes + carry has to be one the rule lists, and a rule can list values the file + does not use. A file with no notes therefore satisfies it, since + requiring notes is what `min_notes` is for. + + Args: + label (str): Output label the rule applies to. + rule (str): The rule name, a key of NOTE_VALUE_RULES. + found (list): The values the notes actually use. + allowed: The permitted value, or a list of them. + + Raises: + AssertionError: If the notes use a value the rule does not list. + """ + + noun = rule.removeprefix("note_") + allowed = [allowed] if isinstance(allowed, int) else list(allowed) + unexpected = sorted(set(found) - set(allowed)) + + assert not unexpected, \ + (f"output '{label}' has notes using {noun} {unexpected}, expected " + f"only {sorted(allowed)}") + + def check_duration(label: str, duration, rules: dict) -> None: """ Apply the shared min_duration / max_duration rules to a decoded length. @@ -208,10 +245,10 @@ def check_expectations(label: str, otype: str, value, rules: dict) -> None: props = reader(label, require_file(label, value)) if props is not None and otype == "audio_track": - if "channels" in rules: - assert props["channels"] == rules["channels"], \ - (f"output '{label}': expected {rules['channels']} channel(s), " - f"got {props['channels']}") + if "num_channels" in rules: + assert props["num_channels"] == rules["num_channels"], \ + (f"output '{label}': expected {rules['num_channels']} " + f"channel(s), got {props['num_channels']}") if "sample_rate" in rules: assert props["sample_rate"] == rules["sample_rate"], \ @@ -237,6 +274,10 @@ def check_expectations(label: str, otype: str, value, rules: dict) -> None: (f"output '{label}' has {props['num_notes']} note(s), expected " f"at least {rules['min_notes']}") + for rule, prop in NOTE_VALUE_RULES.items(): + if rule in rules: + check_note_values(label, rule, props[prop], rules[rule]) + if props is not None: check_duration(label, props["duration"], rules) diff --git a/model_validation/src/midi.py b/model_validation/src/midi.py index d72af2d5..c41f58ec 100644 --- a/model_validation/src/midi.py +++ b/model_validation/src/midi.py @@ -11,7 +11,8 @@ __all__ = [ - 'read_midi_props' + 'read_midi_props', + 'note_voices' ] @@ -25,8 +26,9 @@ def read_midi_props(label: str, path: str) -> dict: Returns: props (dict): num_tracks, num_notes (note-on events with non-zero - velocity), and duration (seconds, or None when it cannot be - determined, e.g. an asynchronous format-2 file). + velocity), duration (seconds, or None when it cannot be + determined, e.g. an asynchronous format-2 file), and the sorted + instruments and channels the notes use (see note_voices). Raises: AssertionError: If the file cannot be parsed as MIDI. @@ -46,8 +48,44 @@ def read_midi_props(label: str, path: str) -> dict: # length is undefined for asynchronous (format 2) files duration = None + instruments, channels = note_voices(midi) + return { "num_tracks": len(midi.tracks), "num_notes": num_notes, "duration": duration, + "instruments": instruments, + "channels": channels, } + + +def note_voices(midi) -> tuple: + """ + Collect the instruments and channels the notes are played with. + + A note's instrument is the program last set on its channel. Program + changes apply only to their own channel, so each channel is tracked + separately. A file may spread one channel's events over several tracks + that play together, so the tracks are merged into a single time-ordered + stream before scanning. Notes preceding any program change on their + channel report the General MIDI default of 0. + + Args: + midi (mido.MidiFile): The parsed file. + + Returns: + instruments (list): Sorted program numbers the notes use, 0 to 127. + channels (list): Sorted channels the notes use, 0 to 15. + """ + + programs = {} + instruments, channels = set(), set() + + for msg in mido.merge_tracks(midi.tracks): + if msg.type == "program_change": + programs[msg.channel] = msg.program + elif msg.type == "note_on" and msg.velocity > 0: + instruments.add(programs.get(msg.channel, 0)) + channels.add(msg.channel) + + return sorted(instruments), sorted(channels)