From 85a550b7049852864b5eb6b5e22da0f6b0c94b47 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:48:44 +0200 Subject: [PATCH 01/10] feat: find all tagged versions and serve schemas directly from gh --- conf.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/conf.py b/conf.py index 55152f5e1..29e5f5e88 100644 --- a/conf.py +++ b/conf.py @@ -57,6 +57,25 @@ "dev/": "../specifications/dev/index.html", } +# Populate schema redirects from GitHub tags +def _populate_schema_redirects(): + import subprocess + result = subprocess.check_output([ + "git", "ls-remote", "--tags", "https://github.com/ome/ngff-spec" + ], text=True, timeout=10) + # result looks like this + # e3d2f8ffbbcfb0e0906e901ec572f5c49b36328d refs/tags/0.6.dev1 + # da4606bf96d2829ad74b4dbaf6de5afb6b7a595a refs/tags/0.6.dev2 + + tags = [ + line.split()[1].replace("refs/tags/", "").rstrip("^{}") + for line in result.strip().split("\n") if line + ] + for tag in sorted(set(tags)): + redirects[f"{tag}/schemas/"] = f"https://raw.githubusercontent.com/ome/ngff-spec/{tag}/schemas/" + +_populate_schema_redirects() + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output @@ -102,20 +121,15 @@ def build_served_html(): from pathlib import Path os.chdir(Path(__file__).parent) - versions = [ + + # Build specifications from local submodules + displayed_spec_versions = [ d for d in os.listdir("specifications") if os.path.isdir(os.path.join("specifications", d)) ] - for version in versions: - - # copy schemas to _html_extra - os.makedirs(f"_html_extra/{version}/schemas", exist_ok=True) - schemas = glob.glob(f"specifications/{version}/**/*.schema", recursive=True) - for schema in schemas: - shutil.copy2(schema, f"_html_extra/{version}/schemas/") - print(f"✅ Copied schemas for version {version}") + for version in displayed_spec_versions: # find 'pre_build.py' in 'specifications' subdirectories script = glob.glob(f"specifications/{version}/**/pre_build.py", recursive=True)[ From 598c9965af735902102b837ded125296e6ab7970 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:00:23 +0200 Subject: [PATCH 02/10] feat: serve extra html with redirect info --- conf.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/conf.py b/conf.py index 29e5f5e88..f4affd17b 100644 --- a/conf.py +++ b/conf.py @@ -57,25 +57,6 @@ "dev/": "../specifications/dev/index.html", } -# Populate schema redirects from GitHub tags -def _populate_schema_redirects(): - import subprocess - result = subprocess.check_output([ - "git", "ls-remote", "--tags", "https://github.com/ome/ngff-spec" - ], text=True, timeout=10) - # result looks like this - # e3d2f8ffbbcfb0e0906e901ec572f5c49b36328d refs/tags/0.6.dev1 - # da4606bf96d2829ad74b4dbaf6de5afb6b7a595a refs/tags/0.6.dev2 - - tags = [ - line.split()[1].replace("refs/tags/", "").rstrip("^{}") - for line in result.strip().split("\n") if line - ] - for tag in sorted(set(tags)): - redirects[f"{tag}/schemas/"] = f"https://raw.githubusercontent.com/ome/ngff-spec/{tag}/schemas/" - -_populate_schema_redirects() - # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output @@ -122,6 +103,20 @@ def build_served_html(): os.chdir(Path(__file__).parent) + # Fetch GitHub tags and create schema redirects + try: + result = subprocess.check_output([ + "git", "ls-remote", "--tags", "https://github.com/ome/ngff-spec" + ], text=True, timeout=10) + tags = [line.split()[1].replace("refs/tags/", "").rstrip("^{}") for line in result.strip().split("\n") if line] + for tag in sorted(set(tags)): + os.makedirs(f"_html_extra/{tag}/schemas", exist_ok=True) + with open(f"_html_extra/{tag}/schemas/index.html", "w") as f: + f.write(f'') + print(f"✅ Redirect schemas/{tag} → GitHub raw") + except Exception: + pass + # Build specifications from local submodules displayed_spec_versions = [ d From 0369fa2b366eed4ba9d0c6650640ecb28a9640c8 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:12:47 +0200 Subject: [PATCH 03/10] feat: try .htaccess file --- conf.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/conf.py b/conf.py index f4affd17b..6d413aab7 100644 --- a/conf.py +++ b/conf.py @@ -103,19 +103,14 @@ def build_served_html(): os.chdir(Path(__file__).parent) - # Fetch GitHub tags and create schema redirects - try: - result = subprocess.check_output([ - "git", "ls-remote", "--tags", "https://github.com/ome/ngff-spec" - ], text=True, timeout=10) - tags = [line.split()[1].replace("refs/tags/", "").rstrip("^{}") for line in result.strip().split("\n") if line] - for tag in sorted(set(tags)): - os.makedirs(f"_html_extra/{tag}/schemas", exist_ok=True) - with open(f"_html_extra/{tag}/schemas/index.html", "w") as f: - f.write(f'') - print(f"✅ Redirect schemas/{tag} → GitHub raw") - except Exception: - pass + # Create .htaccess to redirect all schema requests to GitHub + # ponytail: one rewrite rule handles all versions/files, avoids generating per-file stubs + htaccess_content = """RewriteEngine On +RewriteRule ^([^/]+)/schemas/(.*)$ https://raw.githubusercontent.com/ome/ngff-spec/$1/schemas/$2 [R=301,L] +""" + with open("_html_extra/.htaccess", "w") as f: + f.write(htaccess_content) + print(f"✅ Created .htaccess redirect for all schemas → GitHub raw") # Build specifications from local submodules displayed_spec_versions = [ From 061c07e46d5a589f9e3a1f9d8cc142ddef4db1eb Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:15:46 +0200 Subject: [PATCH 04/10] fix: create _html_extra --- conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conf.py b/conf.py index 6d413aab7..c0a737c36 100644 --- a/conf.py +++ b/conf.py @@ -105,6 +105,7 @@ def build_served_html(): # Create .htaccess to redirect all schema requests to GitHub # ponytail: one rewrite rule handles all versions/files, avoids generating per-file stubs + os.makedirs("_html_extra", exist_ok=True) htaccess_content = """RewriteEngine On RewriteRule ^([^/]+)/schemas/(.*)$ https://raw.githubusercontent.com/ome/ngff-spec/$1/schemas/$2 [R=301,L] """ From 6b055b2f520adebc72b4bb5ef1148696320dda7e Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:24:19 +0200 Subject: [PATCH 05/10] feat: try direct serve --- conf.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/conf.py b/conf.py index c0a737c36..6843f417d 100644 --- a/conf.py +++ b/conf.py @@ -103,15 +103,35 @@ def build_served_html(): os.chdir(Path(__file__).parent) - # Create .htaccess to redirect all schema requests to GitHub - # ponytail: one rewrite rule handles all versions/files, avoids generating per-file stubs - os.makedirs("_html_extra", exist_ok=True) - htaccess_content = """RewriteEngine On -RewriteRule ^([^/]+)/schemas/(.*)$ https://raw.githubusercontent.com/ome/ngff-spec/$1/schemas/$2 [R=301,L] -""" - with open("_html_extra/.htaccess", "w") as f: - f.write(htaccess_content) - print(f"✅ Created .htaccess redirect for all schemas → GitHub raw") + # Fetch GitHub tags and download schemas + try: + result = subprocess.check_output([ + "git", "ls-remote", "--tags", "https://github.com/ome/ngff-spec" + ], text=True, timeout=10) + tags = [line.split()[1].replace("refs/tags/", "").rstrip("^{}") for line in result.strip().split("\n") if line] + for tag in sorted(set(tags)): + schema_dir = f"_html_extra/{tag}/schemas" + os.makedirs(schema_dir, exist_ok=True) + # Download schemas from GitHub raw for this tag + gh_url = f"https://github.com/ome/ngff-spec/archive/refs/tags/{tag}.tar.gz" + try: + import tempfile, tarfile + with tempfile.NamedTemporaryFile(delete=False) as tmp: + subprocess.check_call(["curl", "-sL", gh_url, "-o", tmp.name]) + with tarfile.open(tmp.name) as tar: + for member in tar.getmembers(): + if "/schemas/" in member.name and member.name.endswith(".schema"): + # Extract just the filename, flatten into schema_dir + target = os.path.join(schema_dir, os.path.basename(member.name)) + tar.extract(member, path=tempfile.gettempdir()) + src = os.path.join(tempfile.gettempdir(), member.name) + shutil.copy2(src, target) + os.unlink(tmp.name) + print(f"✅ Downloaded schemas for {tag}") + except Exception as e: + print(f"⚠️ Could not download schemas for {tag}: {e}") + except Exception: + pass # Build specifications from local submodules displayed_spec_versions = [ From acb2ee1e532f2288b93b8b45ca356f72bf209802 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:30:01 +0200 Subject: [PATCH 06/10] feat: dorce display, not download of schemas --- conf.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/conf.py b/conf.py index 6843f417d..10ae3fbd1 100644 --- a/conf.py +++ b/conf.py @@ -103,6 +103,16 @@ def build_served_html(): os.chdir(Path(__file__).parent) + # Create .htaccess to serve schemas inline (not download) + os.makedirs("_html_extra", exist_ok=True) + htaccess_content = """ + Header set Content-Disposition "inline" + +""" + with open("_html_extra/.htaccess", "w") as f: + f.write(htaccess_content) + print(f"✅ Created .htaccess to serve schemas inline") + # Fetch GitHub tags and download schemas try: result = subprocess.check_output([ From d143bdb6968868132e22267a541146552336de1b Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:23:26 +0200 Subject: [PATCH 07/10] feat: get schemas from tagged versions AND submodules --- .gitignore | 1 + conf.py | 74 ++++++++++++++++++++++++++++-------------------------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 798d3b773..4f7cd959f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.DS_Store **/__pycache__/* */index.html +_tmp_ngff_spec/ _build _html_extra .tox diff --git a/conf.py b/conf.py index 10ae3fbd1..ce81d8fba 100644 --- a/conf.py +++ b/conf.py @@ -105,43 +105,38 @@ def build_served_html(): # Create .htaccess to serve schemas inline (not download) os.makedirs("_html_extra", exist_ok=True) - htaccess_content = """ - Header set Content-Disposition "inline" - -""" - with open("_html_extra/.htaccess", "w") as f: - f.write(htaccess_content) - print(f"✅ Created .htaccess to serve schemas inline") - + # Fetch GitHub tags and download schemas - try: - result = subprocess.check_output([ - "git", "ls-remote", "--tags", "https://github.com/ome/ngff-spec" - ], text=True, timeout=10) - tags = [line.split()[1].replace("refs/tags/", "").rstrip("^{}") for line in result.strip().split("\n") if line] - for tag in sorted(set(tags)): - schema_dir = f"_html_extra/{tag}/schemas" - os.makedirs(schema_dir, exist_ok=True) - # Download schemas from GitHub raw for this tag - gh_url = f"https://github.com/ome/ngff-spec/archive/refs/tags/{tag}.tar.gz" - try: - import tempfile, tarfile - with tempfile.NamedTemporaryFile(delete=False) as tmp: - subprocess.check_call(["curl", "-sL", gh_url, "-o", tmp.name]) - with tarfile.open(tmp.name) as tar: - for member in tar.getmembers(): - if "/schemas/" in member.name and member.name.endswith(".schema"): - # Extract just the filename, flatten into schema_dir - target = os.path.join(schema_dir, os.path.basename(member.name)) - tar.extract(member, path=tempfile.gettempdir()) - src = os.path.join(tempfile.gettempdir(), member.name) - shutil.copy2(src, target) - os.unlink(tmp.name) - print(f"✅ Downloaded schemas for {tag}") - except Exception as e: - print(f"⚠️ Could not download schemas for {tag}: {e}") - except Exception: - pass + result = subprocess.check_output([ + "git", "ls-remote", "--tags", "https://github.com/ome/ngff-spec" + ], text=True, timeout=10) + tags = [ + line.split()[1].replace("refs/tags/", "").rstrip("^{}") + for line in result.strip().split("\n") if line + ] + + # Clone repo once, checkout each tag (faster than per-tag tarball download) + repo_path = "_temp_ngff_spec" + if not os.path.exists(repo_path): + subprocess.check_call(["git", "clone", "https://github.com/ome/ngff-spec", repo_path]) + + for tag in sorted(set(tags)): + schema_dir = f"_html_extra/{tag}/schemas" + os.makedirs(schema_dir, exist_ok=True) + try: + subprocess.check_call(["git", "-C", repo_path, "checkout", tag], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + src_schemas = glob.glob(os.path.join(repo_path, "**", "*.schema"), recursive=True) + + if len(src_schemas) == 0: + print(f"⚠️ No schemas found for {tag}") + continue + + for schema_file in src_schemas: + shutil.copy2(schema_file, schema_dir) + print(f"✅ Checked out schemas for {tag}") + except Exception as e: + print(f"⚠️ Could not checkout {tag}: {e}") # Build specifications from local submodules displayed_spec_versions = [ @@ -176,6 +171,13 @@ def build_served_html(): subprocess.check_call([sys.executable, script]) print("✅ Built rendered examples/schemas for version", version) + # copy schemas to _html_extra for served html + schema_files = glob.glob(f"specifications/{version}/**/*.schema", recursive=True) + for schema_file in schema_files: + dest_dir = os.path.join("_html_extra", version, "schemas") + os.makedirs(dest_dir, exist_ok=True) + shutil.copy2(schema_file, dest_dir) + # build jupyter-book docs in specification submodules myst_file = glob.glob(f"specifications/{version}/**/myst.yml", recursive=True)[ 0 From 95d865b2f2489fe296e692d2d3c109a9c85bc45a Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:47:19 +0200 Subject: [PATCH 08/10] feat: clean up cloned repo after copying schemas --- conf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf.py b/conf.py index ce81d8fba..73dd86d60 100644 --- a/conf.py +++ b/conf.py @@ -93,7 +93,7 @@ ] -def build_served_html(): +def build_served_html(clean_up=True): import glob import subprocess import sys @@ -137,7 +137,9 @@ def build_served_html(): print(f"✅ Checked out schemas for {tag}") except Exception as e: print(f"⚠️ Could not checkout {tag}: {e}") - + if clean_up: + shutil.rmtree(repo_path, ignore_errors=True) + # Build specifications from local submodules displayed_spec_versions = [ d From 8094be4f44a437c3250bdfd3fbf8836738fdca26 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:13:01 +0200 Subject: [PATCH 09/10] feat: also serve as .schema.json --- conf.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conf.py b/conf.py index 73dd86d60..7d6dbd9fc 100644 --- a/conf.py +++ b/conf.py @@ -133,7 +133,9 @@ def build_served_html(clean_up=True): continue for schema_file in src_schemas: - shutil.copy2(schema_file, schema_dir) + dest_file = os.path.join(schema_dir, os.path.basename(schema_file)) + shutil.copy2(schema_file, dest_file) + shutil.copy2(schema_file, dest_file + '.json') # dual format print(f"✅ Checked out schemas for {tag}") except Exception as e: print(f"⚠️ Could not checkout {tag}: {e}") @@ -178,7 +180,9 @@ def build_served_html(clean_up=True): for schema_file in schema_files: dest_dir = os.path.join("_html_extra", version, "schemas") os.makedirs(dest_dir, exist_ok=True) - shutil.copy2(schema_file, dest_dir) + dest_file = os.path.join(dest_dir, os.path.basename(schema_file)) + shutil.copy2(schema_file, dest_file) + shutil.copy2(schema_file, dest_file + '.json') # ponytail: dual format # build jupyter-book docs in specification submodules myst_file = glob.glob(f"specifications/{version}/**/myst.yml", recursive=True)[ From f2a5b2c3ba7a6cc4324a3eb00662790255886314 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:56:10 +0200 Subject: [PATCH 10/10] chore: remove ponytail comment --- conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.py b/conf.py index 7d6dbd9fc..b79b09fdb 100644 --- a/conf.py +++ b/conf.py @@ -182,7 +182,7 @@ def build_served_html(clean_up=True): os.makedirs(dest_dir, exist_ok=True) dest_file = os.path.join(dest_dir, os.path.basename(schema_file)) shutil.copy2(schema_file, dest_file) - shutil.copy2(schema_file, dest_file + '.json') # ponytail: dual format + shutil.copy2(schema_file, dest_file + '.json') # dual format (json + schema) # build jupyter-book docs in specification submodules myst_file = glob.glob(f"specifications/{version}/**/myst.yml", recursive=True)[