export to flat-manager

.github/workflows/lint.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install mypy ruff tomlkit
+        pip install mypy ruff tomlkit keyring
     - name: Lint with Ruff
       run: ruff check flatpaker
     - name: Lint with mypy

README.md

@@ -46,8 +46,11 @@ to provide even better security by using Wayland instead of X11 (or XWayland).
 2. Download any mods or addons (optional)
 3. Generate a toml description `flatpaker generate com.developer.game "Game Name" engine archive.zip`
 4. Edit the generated description to fill in missing information
-5. run `flatpaker build-runtimes --install` (which adds the runtimes and sdks)
-6. run `flatpaker build --install *.toml` or `flatpaker build --export --gpg-sign *.toml` (for local install or for export to a shared repo)
+5. run `flatpaker build-runtimes --export=install` (which adds the runtimes and sdks)
+6. Run one of:
+  - `flatpaker build --export=install *.toml` to install for your user
+  - `flatpaker build --export=repo --gpg-sign *.toml` to export to a local ostree repo
+  - `flatpaker build --export=flat-manager *.toml` to export to a flat-manager instance
 
 ### Toml Format
 
@@ -181,15 +184,44 @@ That file must be written to `$XDG_CONFIG_HOME/flatpaker/config.toml` (if unset
 
   # The absolute path to a repo to write to. overwritten by the --repo option
   repo = "/path/to/a/repo/to/export"
+
+  # The default export mode
+  # May be one of: "none", "install", "repo", "flat-manager"
+  export = "none"
+
+[flat-manager]
+  # The address that the flat-manager instance listens on
+  remote = "https://flat-manager.example.com:8080"
+
+  # The default repo on that flat-manager instance to push to
+  repo = "stable"
+
+  # Only one of the following may be set. This is the repo key that
+  # will be passed to the flat-manager client.
+
+  # A pair of strings to pass to `keyring.get_password()`
+  # The token can be written into the keyring with the command line tool
+  # `keyring set <serivce name> <keyname>`.
+  token-keyring = ["service name", "keyname"]
+
+  # A file containing the key
+  # Both environment variables and `~` can be used here
+  # This is the only form that can be set on the command line
+  token-file = "/secret.d/flat-manager/1"
+
+  # The keyfile written straight into the config file
+  token-str = "ABC123"
 ```
 
 
 ## What is required?
 
 - python >= 3.10
+- python-keyring (if using flat-manager export with a secret stored in a keyring)
 - python-tomlkit
 - flatpak-builder
 - flatpak
+- flat-manager-client (if using flat-manager export)
 
 ### Schema
 

flatpaker/actions/build_flatpak.py

@@ -14,7 +14,7 @@
 
 if typing.TYPE_CHECKING:
     from flatpaker.description import Description, EngineName
-    from flatpaker.entry import BaseBuildArguments, BuildArguments
+    from flatpaker.entry import BuildFlatpakConfig
 
     JsonWriterImpl = typing.Callable[[Description, pathlib.Path, str, pathlib.Path, pathlib.Path], None]
 
@@ -30,7 +30,8 @@ def select_impl(name: EngineName) -> JsonWriterImpl:
     return mod.write_rules
 
 
-def _build(args: BaseBuildArguments, description: Description) -> None:
+def _build(args: BuildFlatpakConfig, path: pathlib.Path) -> None:
+    description = load_description(path)
     # TODO: This could be common
     appid = f"{description.common.reverse_url}.{util.sanitize_name(description.common.name)}"
 
@@ -43,29 +44,44 @@ def _build(args: BaseBuildArguments, description: Description) -> None:
         write_build_rules(description, workdir, appid, desktop_file, appdata_file)
 
         build_command: list[str] = [
-            'flatpak-builder', '--force-clean', '--user', 'build',
+            'flatpak-builder', '--install-deps-from=flathub',
+            '--force-clean', '--user', 'build',
             (workdir / f'{appid}.json').absolute().as_posix(),
         ]
 
-        if args.export:
-            build_command.extend(['--repo', args.repo])
-            if args.gpg:
-                build_command.extend(['--gpg-sign', args.gpg])
-        if args.install:
-            build_command.extend(['--install'])
+        repo = args.repo
+
+        match args.export:
+            case 'repo':
+                build_command.extend(['--repo', repo])
+                if args.gpg:
+                    build_command.extend(['--gpg-sign', args.gpg])
+            case 'install':
+                build_command.extend(['--install'])
+            case 'flat-manager':
+                # Use a temporary repo for each runtime and app
+                # This simplifies uploading with flat-manager-client
+                repos = pathlib.Path.cwd() / '.flat-manager-repos'
+                repos.mkdir(exist_ok=True)
+                repo = repos.joinpath(appid).as_posix()
+                build_command.extend(['--repo', repo])
 
         subprocess.run(build_command, check=True)
+
+        if args.export == 'flat-manager':
+            assert args.flat_manager is not None
+            util.export_to_flat_manager(repo, args.flat_manager)
+
         if args.cleanup:
             shutil.rmtree('build', ignore_errors=True)
 
 
-def build_flatpak(args: BuildArguments) -> bool:
+def build_flatpak(args: BuildFlatpakConfig) -> bool:
     success = True
 
     for d in args.descriptions:
         try:
-            description = load_description(d)
-            _build(args, description)
+            _build(args, d)
         except Exception:
             if not args.keep_going:
                 raise

flatpaker/actions/build_runtime.py

@@ -11,41 +11,60 @@
 from flatpaker import util
 
 if typing.TYPE_CHECKING:
-    from ..entry import BaseBuildArguments, BuildRuntimeArguments
+    from ..entry import BuildRuntimeConfig
 
 
-def _build_runtime(args: BaseBuildArguments, sdk: pathlib.Path,
+_RUNTIME_ID_BASE = 'com.github.dcbaker.flatpaker'
+
+
+def _get_renpy_branch(sdk: str) -> str:
+    if '8' in sdk:
+        return '8'
+    elif '7.py2' in sdk:
+        return '7'
+    elif '7.py3' in sdk:
+        return '7-PY3'
+    raise RuntimeError("Unknown Ren'Py branch")
+
+
+def _build_runtime(args: BuildRuntimeConfig, sdk: pathlib.Path,
                    need_platform_workaround: bool) -> None:
     build_command: list[str] = [
         'flatpak-builder', '--force-clean', '--user',
         '--install-deps-from=flathub', 'build', sdk.as_posix()]
 
-    if args.export:
-        build_command.extend(['--repo', args.repo])
-        if args.gpg:
-            build_command.extend(['--gpg-sign', args.gpg])
-    if args.install:
-        build_command.extend(['--install'])
+    repo = args.repo
+
+    match args.export:
+        case 'repo':
+            build_command.extend(['--repo', repo])
+            if args.gpg:
+                build_command.extend(['--gpg-sign', args.gpg])
+        case 'install':
+            build_command.extend(['--install'])
+        case 'flat-manager':
+            # Use a temporary repo for each runtime and app
+            # This simplifies uploading with flat-manager-client
+            repos = pathlib.Path.cwd() / '.flat-manager-repos'
+            repos.mkdir(exist_ok=True)
+            repo = repos.joinpath(sdk.name).as_posix()
+            build_command.extend(['--repo', repo])
 
     subprocess.run(build_command, check=True)
 
+    if args.export == 'flat-manager':
+        assert args.flat_manager is not None
+        util.export_to_flat_manager(repo, args.flat_manager)
+
+    platform_id = sdk.name.removeprefix(_RUNTIME_ID_BASE).removeprefix('.').split('.', maxsplit=1)[0]
     # Work around https://github.com/flatpak/flatpak-builder/issues/630
-    if need_platform_workaround and args.install and 'Sdk' in sdk.name:
-        if '8' in sdk.name:
-            branch = '8'
-        elif '7.py2' in sdk.name:
-            branch = '7'
-        elif '7.py3' in sdk.name:
-            branch = '7-PY3'
-        else:
-            raise RuntimeError('Unexpected Sdk')
-
-        repo = args.repo if args.export else pathlib.Path('.flatpak-builder/cache').absolute().as_posix()
-        platform_id = '.'.join(sdk.name.split('.', maxsplit=5)[:-1])
+    if need_platform_workaround and args.export == 'install' and platform_id == 'RenPy':
+        repo = pathlib.Path('.flatpak-builder/cache').absolute().as_posix()
+        branch = _get_renpy_branch(sdk.name)
 
         install_command = [
             'flatpak', 'install', '--user', '-y', '--noninteractive',
-            '--reinstall', repo, f'{platform_id}.Platform//{branch}',
+            '--reinstall', repo, f'{_RUNTIME_ID_BASE}.{platform_id}.Platform//{branch}',
         ]
         subprocess.run(install_command, check=True)
 
@@ -68,24 +87,16 @@ def _need_platform_workaround() -> bool:
     return tuple(int(v) for v in raw_ver.split('.')) < (1, 4, 5)
 
 
-def build_runtimes(args: BuildRuntimeArguments) -> bool:
-    command = [
-        'flatpak', 'install', '--no-auto-pin', '--user',
-        f'org.freedesktop.Platform//{util.RUNTIME_VERSION}',
-        f'org.freedesktop.Sdk//{util.RUNTIME_VERSION}',
-    ]
-    subprocess.run(command, check=True)
-
-    basename = 'com.github.dcbaker.flatpaker'
+def build_runtimes(args: BuildRuntimeConfig) -> bool:
     runtimes: list[str] = []
     if 'rpgmaker' in args.runtimes:
-        runtimes.append(f'{basename}.RPGM.Platform.yml')
+        runtimes.append(f'{_RUNTIME_ID_BASE}.RPGM.Platform.yml')
     if 'renpy8' in args.runtimes:
-        runtimes.append(f'{basename}.RenPy.8.Sdk.yml')
+        runtimes.append(f'{_RUNTIME_ID_BASE}.RenPy.8.Sdk.yml')
     if 'renpy7' in args.runtimes:
-        runtimes.append(f'{basename}.RenPy.7.py2.Sdk.yml')
+        runtimes.append(f'{_RUNTIME_ID_BASE}.RenPy.7.py2.Sdk.yml')
     if 'renpy7-py3' in args.runtimes:
-        runtimes.append(f'{basename}.RenPy.7.py3.Sdk.yml')
+        runtimes.append(f'{_RUNTIME_ID_BASE}.RenPy.7.py3.Sdk.yml')
 
     success = True
 

flatpaker/actions/generate.py

@@ -15,10 +15,10 @@
 if typing.TYPE_CHECKING:
     import tomlkit.items  # noqa: TC004
 
-    from flatpaker.entry import GenerateArguments
+    from flatpaker.entry import GenerateConfig
 
 
-def generate(args: GenerateArguments) -> bool:
+def generate(args: GenerateConfig) -> bool:
     name = f'{args.url}.{util.sanitize_name(args.appname)}'
     projectdir = pathlib.Path(name)
     sourcedir = projectdir / 'sources'
@@ -49,7 +49,7 @@ def add(table: tomlkit.items.Table, key: str, entry: object,
     doc.add('appdata', appdata)
 
     archives: list[tomlkit.items.Table] = []
-    for src in [args.archive] + args.archives:
+    for src in args.archives:
         archive = tomlkit.table()
         add(archive, 'path', os.path.join(sourcedir.name, os.path.basename(src)))
         add(archive, 'sha256', util.sha256(pathlib.Path(src)))
@@ -93,7 +93,7 @@ def add(table: tomlkit.items.Table, key: str, entry: object,
         tomlkit.dump(doc, f)
 
     # move files after writing the toml, so we don't move things then fail
-    for srcs, subdir in [([args.archive] + args.archives + args.files, sourcedir),
+    for srcs, subdir in [(args.archives + args.files, sourcedir),
                          (args.patches, patchdir)]:
         for src in srcs:
             srcp = pathlib.Path(src)

flatpaker/config.py

@@ -9,18 +9,64 @@
 import tomlkit
 
 if typing.TYPE_CHECKING:
+    ExportMode = typing.Literal['none', 'repo', 'install', 'flat-manager']
+
+    FlatManager = typing.TypedDict(
+        'FlatManager',
+        {
+            'remote': str,
+            'repo': str,
+            'token-file': str,
+            'token-str': str,
+            'token-keyring': tuple[str, str]
+        },
+        total=False,
+    )
 
     Common = typing.TypedDict(
         'Common',
         {
             'gpg-key': str,
             'repo': str,
+            'export': ExportMode,
         },
         total=False,
     )
 
-    class Config(typing.TypedDict):
-        common: Common
+    Config = typing.TypedDict(
+        'Config',
+        {
+            'common': Common,
+            'flat-manager': FlatManager,
+        },
+    )
+
+
+def _load_flat_manager(raw: dict[str, object]) -> FlatManager:
+    token_file = raw.get('token-file', None)
+    token_str = raw.get('token-str', None)
+    token_key = raw.get('token-keyring', None)
+
+    if len([k for k in [token_file, token_str, token_key] if k is not None]) > 1:
+        raise TypeError('Configuration file may only contain one of: '
+                        '"flat-manager.token-file", "flat-manager.token-str", or '
+                        '"flat-manager.token-key"')
+
+    if token_file is not None and not isinstance(token_file, str):
+        raise TypeError('Configuration key "flat-manager.token-file" must be a string')
+    if token_str is not None and not isinstance(token_str, str):
+        raise TypeError('Configuration key "flat-manager.token-str" must be a string')
+    if token_key is not None:
+        if not isinstance(token_key, list):
+            raise TypeError('Configuration key "flat-manager.token-key" must be a a list')
+        if any(not isinstance(k, str) for k in token_key):
+            raise TypeError('Configuration key "flat-manager.token-key" elements must be strings')
+        if len(token_key) != 2:
+            raise TypeError('Configuration key "flat-manager.token-key" must be an array '
+                            'with exactly two elements')
+        raw['token-keyring'] = tuple(token_key)
+
+    return typing.cast('FlatManager', raw)
 
 
 def load_config() -> Config:
@@ -36,4 +82,10 @@ def load_config() -> Config:
 
     if 'common' not in raw:
         raw['common'] = {}
+
+    if fm := raw.get('flat-manager'):
+        raw['flat-manager'] = _load_flat_manager(fm)
+    else:
+        raw['flat-manager'] = {}
+
     return typing.cast('Config', raw)

flatpaker/description.py

@@ -88,13 +88,13 @@ class Description:
     sources: Sources
 
 
-def load_description(name: str) -> Description:
-    relpath = pathlib.Path(name).parent.absolute()
+def load_description(path: pathlib.Path) -> Description:
+    relpath = path.parent.absolute()
 
     # TODO: the cast to Any leaves us with the same
     #       validation problem with had previous, but without the hints.
     #       I wish python had something like serde
-    with open(name, 'rb') as f:
+    with path.open('rb') as f:
         d = typing.cast('typing.Any', tomlkit.load(f))
 
     quirks = Quirks(**d.get('quirks', {}))