From 1c7bb320b8505604f3cf9fffd471826ebb459894 Mon Sep 17 00:00:00 2001 From: Aaron Cassar Date: Fri, 21 Aug 2026 11:50:09 -0400 Subject: [PATCH 1/7] feat: add Google Drive backend module Adds a gdrive module implementing the RootFs contract against the Google Drive API v3 with the drive.file scope. Authentication uses Google's OAuth device flow against the user's own OAuth client, with tokens in Obsidian secret storage and a bearer-token request middleware that refreshes and retries once on 401. Listing fetches all visible files in one paginated query and assembles the tree client-side; uploads use multipart for small files and sequential resumable sessions for large ones; deletes go to the Drive trash by default. Includes an in-memory Drive API mock and contract tests, a module spec page, and registry plus docs sidebar entries. --- bun.lock | 11 + docs/.vitepress/config.ts | 1 + docs/.vitepress/i18n.ts | 1 + docs/src/pages/en/deep-dive/modules/gdrive.md | 83 +++ modules.json | 9 + packages/gdrive/package.json | 25 + packages/gdrive/src/gdrive/api.ts | 73 +++ packages/gdrive/src/gdrive/auth-http.ts | 10 + packages/gdrive/src/gdrive/auth.ts | 263 ++++++++++ .../gdrive/src/gdrive/check-connection.ts | 19 + packages/gdrive/src/gdrive/fs.ts | 494 ++++++++++++++++++ packages/gdrive/src/gdrive/read-stream.ts | 111 ++++ packages/gdrive/src/gdrive/upload.ts | 142 +++++ packages/gdrive/src/handle-input.ts | 30 ++ packages/gdrive/src/i18n.ts | 37 ++ packages/gdrive/src/index.ts | 130 +++++ packages/gdrive/src/setting.ts | 291 +++++++++++ packages/gdrive/test/auth.test.ts | 208 ++++++++ packages/gdrive/test/fs-gdrive.test.ts | 240 +++++++++ packages/gdrive/test/mock-drive.ts | 386 ++++++++++++++ packages/gdrive/test/mocks.ts | 5 + packages/gdrive/tsconfig.json | 10 + packages/gdrive/tsdown.config.ts | 14 + 23 files changed, 2593 insertions(+) create mode 100644 docs/src/pages/en/deep-dive/modules/gdrive.md create mode 100644 packages/gdrive/package.json create mode 100644 packages/gdrive/src/gdrive/api.ts create mode 100644 packages/gdrive/src/gdrive/auth-http.ts create mode 100644 packages/gdrive/src/gdrive/auth.ts create mode 100644 packages/gdrive/src/gdrive/check-connection.ts create mode 100644 packages/gdrive/src/gdrive/fs.ts create mode 100644 packages/gdrive/src/gdrive/read-stream.ts create mode 100644 packages/gdrive/src/gdrive/upload.ts create mode 100644 packages/gdrive/src/handle-input.ts create mode 100644 packages/gdrive/src/i18n.ts create mode 100644 packages/gdrive/src/index.ts create mode 100644 packages/gdrive/src/setting.ts create mode 100644 packages/gdrive/test/auth.test.ts create mode 100644 packages/gdrive/test/fs-gdrive.test.ts create mode 100644 packages/gdrive/test/mock-drive.ts create mode 100644 packages/gdrive/test/mocks.ts create mode 100644 packages/gdrive/tsconfig.json create mode 100644 packages/gdrive/tsdown.config.ts diff --git a/bun.lock b/bun.lock index 45894ee4..cb721a66 100644 --- a/bun.lock +++ b/bun.lock @@ -43,6 +43,15 @@ "uni-kv": "../../uni-kv.tgz", }, }, + "packages/gdrive": { + "name": "gdrive", + "version": "0.0.1", + "devDependencies": { + "@hesprs/sync-engine-sdk": "workspace:*", + "@repo/shared": "workspace:*", + "hash-wasm": "^4.12.0", + }, + }, "packages/i18n": { "name": "i18n", "version": "0.0.1", @@ -687,6 +696,8 @@ "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "gdrive": ["gdrive@workspace:packages/gdrive"], + "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], "get-tsconfig": ["get-tsconfig@5.0.0-beta.5", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-/6gFNr0N04nob252sTQxyFLi3eKFRqIg1I87YcqAMT1i6SQrSF6KujUEQrtrjMV0H/eejTCltLdDSTEMzHbnsQ=="], diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index d1fa33b9..e85cbaba 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -156,6 +156,7 @@ const localeConfig = configGenerator((t) => { items: [ { link: `${deepDive}/modules/webdav`, text: t('webdav') }, { link: `${deepDive}/modules/s3`, text: t('s3') }, + { link: `${deepDive}/modules/gdrive`, text: t('gdrive') }, { link: `${deepDive}/modules/encryption`, text: t('encryption') }, { lint: `${deepDive}/modules/smart-marge`, text: t('smartMerge') }, ], diff --git a/docs/.vitepress/i18n.ts b/docs/.vitepress/i18n.ts index fb2892fc..c174214d 100644 --- a/docs/.vitepress/i18n.ts +++ b/docs/.vitepress/i18n.ts @@ -23,6 +23,7 @@ const en = { fileSystemWrappers: 'File System Wrappers', fileTree: 'File Tree', folder: '', + gdrive: 'Google Drive', home: 'Home', licenseMessage: 'All content licensed under the CC BY 4.0 License.', diff --git a/docs/src/pages/en/deep-dive/modules/gdrive.md b/docs/src/pages/en/deep-dive/modules/gdrive.md new file mode 100644 index 00000000..ed1a24e4 --- /dev/null +++ b/docs/src/pages/en/deep-dive/modules/gdrive.md @@ -0,0 +1,83 @@ +# Google Drive Module + +The Google Drive module connects Sync Engine to Google Drive. It registers the `gdrive` remote file system, authenticates with OAuth through Google's device flow, and stores the vault as regular files and folders that stay readable in the Drive web and mobile apps. + +## Requirements + +The module uses a Google Cloud OAuth client that you create in your own Google account, so no third-party server ever handles your tokens: + +1. In [Google Cloud Console](https://console.cloud.google.com/), create a project (or reuse one) and enable the **Google Drive API**. +2. Configure the OAuth consent screen. Add yourself as a test user, or publish the app to production — a consent screen left in testing mode expires refresh tokens after seven days, forcing weekly reconnects. +3. Create an OAuth client of type **TV and Limited Input devices** and note the client ID and client secret. + +The same client ID and secret are entered on every device that syncs; each device completes its own device-flow approval. + +## Settings and Configuration + +Install and enable the Google Drive module, then select **Google Drive** as the storage backend. Configure these module settings: + +| Setting | Description | +| ----------------------- | ----------------------------------------------------------------------------- | +| **OAuth client ID** | Client ID of your Google Cloud OAuth client. | +| **OAuth client secret** | Client secret of the same OAuth client. Stored in Obsidian's keychain. | +| **Google account** | Connect, reconnect, or disconnect the Google account through the device flow. | +| **Base directory** | Drive folder that holds the vault. Defaults to the vault name. | +| **Delete to trash** | Move remote deletions to the Drive trash instead of deleting permanently. | + +Connecting opens a dialog with a short code: visit the shown Google URL on any device, enter the code, and approve access. The dialog polls until Google confirms, then the module stores the refresh token and shows the connected account. + +## Credentials and Keychain + +The client secret and the refresh token are stored through Obsidian's secret storage, not as ordinary module settings. Access tokens are short-lived, kept only in memory, and refreshed automatically. Disconnecting clears the stored refresh token; access can also be revoked at [myaccount.google.com/permissions](https://myaccount.google.com/permissions). + +## Scope and Visibility + +The module requests only the `drive.file` scope, plus `email` to display the connected account. `drive.file` grants access exclusively to files this module created — it cannot read the rest of the Drive. The practical consequences: + +- Do not create the base directory manually in Drive; the module creates it on the first sync. A manually created folder is invisible to the module and leads to a duplicate. +- Files added to the vault folder through the Drive web or mobile apps are invisible to the module and never sync. Edit through Obsidian only; treat the Drive copy as read-only. +- Google shows an "unverified app" style consent step for personal OAuth clients. That is expected — the client is your own. + +## Base Directory + +The base directory is resolved to a Drive folder ID and used as the file-system root, so the folder path never appears in keys. It must be identical on every device syncing the same vault. Renaming or moving the vault folder in the Drive web interface changes nothing for sync (IDs stay stable) as long as the configured path still resolves; keep the setting and the actual folder in agreement. + +## Practical Behavior + +- Folders are real Drive folders and moves use Drive's native rename and re-parenting — no copy-and-delete. +- Deleting a missing file is treated as success. With **Delete to trash** enabled, deletions land in the Drive trash, which Google empties after 30 days. +- File UIDs use the Drive `md5Checksum`. The local modification time is written to Drive's `modifiedTime` on upload, so timestamps survive round trips between devices. +- Duplicate names in one folder (possible in Drive, not in a vault) resolve to the most recently modified file. +- [Asymmetric storage](../asymmetric-storage) can flatten and anchor remote keys. Use it only when remote files do not need to remain readable in their normal folder structure, and keep the setting consistent across devices. + +## Implementation + +### Unified File-System Mapping + +`GdriveFs` implements the SDK `RootFs` contract with unified keys. Drive is ID-based, so the module resolves path keys to file IDs segment by segment and caches the mapping for the lifetime of the instance. The basic operations map to Drive API v3 requests as follows: + +| File-system operation | Drive operation | +| --------------------- | ------------------------------------------------ | +| `read()` | `files.get` with `alt=media` | +| `write()` | Multipart upload (create or update) | +| `stat()` | `files.list` lookup by parent and name | +| `delete()` | `files.update` with `trashed`, or `files.delete` | +| `move()` | `files.update` with `addParents`/`removeParents` | +| `mkdir()` | `files.create` with the folder MIME type | +| `list()` | Paginated `files.list`, assembled into a tree | + +### Bearer Middleware + +When Google Drive is the selected backend, registered request middleware attaches a `Bearer` access token to every remote request. A shared token manager caches the access token, refreshes it through the stored refresh token shortly before expiry, deduplicates concurrent refreshes, and retries a request once after an authentication failure. A revoked refresh token surfaces as a clear reconnect prompt. + +### Flat Listing + +Because `drive.file` limits visibility to module-created files, `list()` fetches every visible file in one paginated query (1000 files per page) instead of one request per folder, then assembles the tree client-side from parent references. The walk honors the unified reporter verdicts, skipping excluded subtrees without visiting them. + +### Range Reads + +`readStream()` downloads media with ranged `GET` requests: 2 MiB chunks, at most eight in flight, emitted in file order. Empty files return an already-closed stream. + +### Resumable Uploads + +`writeStream()` buffers small files and sends them as one multipart upload. Files of 8 MiB or more use a Drive resumable upload session: metadata initiates the session, sequential `PUT` requests send 8 MiB chunks (Drive requires multiples of 256 KiB), and the final chunk closes the session. On failure the module attempts to cancel the session. diff --git a/modules.json b/modules.json index 44fc83e2..b70aa845 100644 --- a/modules.json +++ b/modules.json @@ -61,5 +61,14 @@ "icon": "combine", "main": "https://sync.consensia.cc/modules/smart-merge.js", "minPluginVersion": "3.1.0" + }, + { + "id": "gdrive", + "name": "Google Drive", + "version": "0.1.0", + "description": "Google Drive backend support.", + "icon": "hard-drive", + "main": "https://sync.consensia.cc/modules/gdrive.js", + "minPluginVersion": "3.1.0" } ] diff --git a/packages/gdrive/package.json b/packages/gdrive/package.json new file mode 100644 index 00000000..8c6c9e50 --- /dev/null +++ b/packages/gdrive/package.json @@ -0,0 +1,25 @@ +{ + "name": "gdrive", + "version": "0.0.1", + "private": true, + "license": "MIT", + "contributors": [ + { + "name": "Aaron", + "github": "Quzzar" + } + ], + "type": "module", + "scripts": { + "dev": "MODE=dev bun --bun tsdown", + "compile": "bun --bun tsdown", + "fix": "oxlint --silent --fix && oxfmt", + "check": "tsc && oxlint && oxfmt --check", + "tests": "bun --preload=./test/mocks test" + }, + "devDependencies": { + "@hesprs/sync-engine-sdk": "workspace:*", + "@repo/shared": "workspace:*", + "hash-wasm": "^4.12.0" + } +} diff --git a/packages/gdrive/src/gdrive/api.ts b/packages/gdrive/src/gdrive/api.ts new file mode 100644 index 00000000..05b39a44 --- /dev/null +++ b/packages/gdrive/src/gdrive/api.ts @@ -0,0 +1,73 @@ +import type { FileStat, RequestResponse } from '@hesprs/sync-engine-sdk'; + +export const DRIVE_API = 'https://www.googleapis.com/drive/v3'; +export const DRIVE_UPLOAD_API = 'https://www.googleapis.com/upload/drive/v3'; +export const OAUTH_DEVICE_CODE_URL = 'https://oauth2.googleapis.com/device/code'; +export const OAUTH_TOKEN_URL = 'https://oauth2.googleapis.com/token'; +export const OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive.file email'; +export const FOLDER_MIME = 'application/vnd.google-apps.folder'; +export const FILE_FIELDS = 'id,name,mimeType,md5Checksum,modifiedTime,size,parents'; + +export type DriveFile = { + id: string; + name: string; + mimeType: string; + md5Checksum?: string; + modifiedTime?: string; + size?: string; + parents?: Array; +}; + +export type DriveFileList = { + files?: Array; + nextPageToken?: string; +}; + +const mtimeMissing = new Error('Google Drive did not return the modified time for a file!'); + +/** Escapes a string literal used inside a Drive `q` search expression. */ +export function escapeQuery(value: string): string { + return value.replaceAll('\\', String.raw`\\`).replaceAll("'", String.raw`\'`); +} + +export function buildUrl(base: string, path: string, query: Record = {}): string { + const url = new URL(`${base}${path}`); + for (const [key, value] of Object.entries(query)) url.searchParams.set(key, value); + return url.toString(); +} + +export function getHeader( + headers: Record, + name: string, +): string | undefined { + const entry = Object.entries(headers).find( + ([headerName]) => headerName.toLowerCase() === name.toLowerCase(), + ); + return entry?.[1]; +} + +export function safeJson(response: RequestResponse): unknown { + try { + return response.json(); + } catch { + return {}; + } +} + +export function parseDriveError(response: RequestResponse): string | undefined { + const parsed = safeJson(response) as { + error?: { code?: number; message?: string } | string; + error_description?: string; + }; + if (typeof parsed.error === 'string') + return `Google Drive ${parsed.error}: ${parsed.error_description ?? ''}`; + if (parsed.error?.message) + return `Google Drive ${parsed.error.code ?? response.status}: ${parsed.error.message}`; +} + +export function toFileStat(key: string, file: DriveFile): FileStat { + if (!file.modifiedTime) throw mtimeMissing; + const mtime = new Date(file.modifiedTime).valueOf(); + const size = file.size === undefined ? 0 : Number.parseInt(file.size); + return { isDir: false, key, mtime, size, uid: file.md5Checksum ?? `${mtime}~${size}` }; +} diff --git a/packages/gdrive/src/gdrive/auth-http.ts b/packages/gdrive/src/gdrive/auth-http.ts new file mode 100644 index 00000000..0b3c7a92 --- /dev/null +++ b/packages/gdrive/src/gdrive/auth-http.ts @@ -0,0 +1,10 @@ +import { requestUrl } from 'obsidian'; +import type { AuthHttp } from './auth'; + +/** `AuthHttp` implementation backed by Obsidian's CORS-free `requestUrl`. */ +const requestUrlHttp: AuthHttp = async ({ url, method, body, contentType }) => { + const response = await requestUrl({ body, contentType, method, throw: false, url }); + return { json: () => response.json as unknown, status: response.status }; +}; + +export default requestUrlHttp; diff --git a/packages/gdrive/src/gdrive/auth.ts b/packages/gdrive/src/gdrive/auth.ts new file mode 100644 index 00000000..aef38e01 --- /dev/null +++ b/packages/gdrive/src/gdrive/auth.ts @@ -0,0 +1,263 @@ +import type { Request, RequestParam } from '@hesprs/sync-engine-sdk'; +import { OAUTH_DEVICE_CODE_URL, OAUTH_SCOPE, OAUTH_TOKEN_URL } from './api'; + +/** + * Minimal HTTP shape used for OAuth endpoints. Kept independent from the SDK + * `Request` so authentication can run from settings UI code (via Obsidian + * `requestUrl`) and from tests without a composed request chain. + */ +export type AuthHttp = (params: { + url: string; + method: 'GET' | 'POST'; + body?: string; + contentType?: string; +}) => Promise<{ status: number; json: () => unknown }>; + +export type AuthConfig = { + clientId: string; + clientSecret: string; + refreshToken: string; +}; + +export type DeviceAuthorization = { + deviceCode: string; + userCode: string; + verificationUrl: string; + expiresIn: number; + interval: number; +}; + +export type DeviceTokenResult = { + accessToken: string; + refreshToken: string; + expiresIn: number; + email?: string; +}; + +const FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded'; + +/** Secret storage id under which the Google refresh token is stored. */ +export const REFRESH_TOKEN_SECRET_ID = 'sync-engine-gdrive-refresh-token'; + +function formEncode(fields: Record): string { + return new URLSearchParams(fields).toString(); +} + +function describeAuthError(data: Record, status: number): string { + const description = + typeof data.error_description === 'string' ? data.error_description : undefined; + const code = typeof data.error === 'string' ? data.error : undefined; + return description ?? code ?? `HTTP ${status}`; +} + +function safeAuthJson(response: { json: () => unknown }): Record { + try { + const parsed = response.json(); + return typeof parsed === 'object' && parsed !== undefined && parsed !== null + ? (parsed as Record) + : {}; + } catch { + return {}; + } +} + +export function decodeIdTokenEmail(idToken: string): string | undefined { + try { + const payload = idToken.split('.')[1]; + if (!payload) return undefined; + const normalized = payload.replaceAll('-', '+').replaceAll('_', '/'); + const padded = normalized + '='.repeat((4 - (normalized.length % 4)) % 4); + const parsed = JSON.parse(atob(padded)) as { email?: unknown }; + return typeof parsed.email === 'string' ? parsed.email : undefined; + } catch { + return undefined; + } +} + +export async function startDeviceAuthorization( + http: AuthHttp, + clientId: string, +): Promise { + const response = await http({ + body: formEncode({ client_id: clientId, scope: OAUTH_SCOPE }), + contentType: FORM_CONTENT_TYPE, + method: 'POST', + url: OAUTH_DEVICE_CODE_URL, + }); + const data = safeAuthJson(response); + if (response.status < 200 || response.status >= 300) + throw new Error( + `Google device authorization failed: ${describeAuthError(data, response.status)}`, + ); + const deviceCode = data.device_code; + const userCode = data.user_code; + const verificationUrl = data.verification_url ?? data.verification_uri; + if ( + typeof deviceCode !== 'string' || + typeof userCode !== 'string' || + typeof verificationUrl !== 'string' + ) + throw new Error('Google device authorization returned an unexpected response.'); + return { + deviceCode, + expiresIn: typeof data.expires_in === 'number' ? data.expires_in : 1800, + interval: typeof data.interval === 'number' ? data.interval : 5, + userCode, + verificationUrl, + }; +} + +export async function pollDeviceToken( + http: AuthHttp, + options: { + clientId: string; + clientSecret: string; + authorization: DeviceAuthorization; + isCancelled?: () => boolean; + sleep?: (ms: number) => Promise; + now?: () => number; + }, +): Promise { + const sleep = + options.sleep ?? + ((ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + })); + const now = options.now ?? (() => Date.now()); + let interval = Math.max(options.authorization.interval, 1); + const deadline = now() + options.authorization.expiresIn * 1000; + while (true) { + await sleep(interval * 1000); + if (options.isCancelled?.()) throw new Error('Google Drive connection was cancelled.'); + if (now() > deadline) + throw new Error('The device code expired, please try connecting again.'); + const response = await http({ + body: formEncode({ + client_id: options.clientId, + client_secret: options.clientSecret, + device_code: options.authorization.deviceCode, + grant_type: 'urn:ietf:params:oauth:grant-type:device_code', + }), + contentType: FORM_CONTENT_TYPE, + method: 'POST', + url: OAUTH_TOKEN_URL, + }); + const data = safeAuthJson(response); + if ( + response.status >= 200 && + response.status < 300 && + typeof data.access_token === 'string' && + typeof data.refresh_token === 'string' + ) + return { + accessToken: data.access_token, + email: + typeof data.id_token === 'string' + ? decodeIdTokenEmail(data.id_token) + : undefined, + expiresIn: typeof data.expires_in === 'number' ? data.expires_in : 3600, + refreshToken: data.refresh_token, + }; + switch (data.error) { + case 'authorization_pending': { + continue; + } + case 'slow_down': { + interval += 5; + continue; + } + case 'access_denied': { + throw new Error('Google Drive access was denied.'); + } + case 'expired_token': { + throw new Error('The device code expired, please try connecting again.'); + } + default: { + throw new Error( + `Google Drive connection failed: ${describeAuthError(data, response.status)}`, + ); + } + } + } +} + +/** + * Caches the short-lived access token and refreshes it with the stored refresh + * token when needed. One instance is shared by the request middleware and the + * connection check so a token refresh happens at most once at a time. + */ +export class TokenManager { + private accessToken: string | undefined; + private expiresAt = 0; + private pending: Promise | undefined; + + constructor( + private readonly http: AuthHttp, + private readonly resolveAuth: () => AuthConfig, + private readonly now: () => number = () => Date.now(), + ) {} + + readonly getToken = (force = false): Promise => { + if (!force && this.accessToken !== undefined && this.now() < this.expiresAt - 60_000) + return Promise.resolve(this.accessToken); + this.pending ??= this.refresh().finally(() => { + this.pending = undefined; + }); + return this.pending; + }; + + readonly invalidate = (): void => { + this.accessToken = undefined; + this.expiresAt = 0; + }; + + private async refresh(): Promise { + const { clientId, clientSecret, refreshToken } = this.resolveAuth(); + const response = await this.http({ + body: formEncode({ + client_id: clientId, + client_secret: clientSecret, + grant_type: 'refresh_token', + refresh_token: refreshToken, + }), + contentType: FORM_CONTENT_TYPE, + method: 'POST', + url: OAUTH_TOKEN_URL, + }); + const data = safeAuthJson(response); + if ( + response.status >= 200 && + response.status < 300 && + typeof data.access_token === 'string' + ) { + this.accessToken = data.access_token; + this.expiresAt = + this.now() + (typeof data.expires_in === 'number' ? data.expires_in : 3600) * 1000; + return data.access_token; + } + this.invalidate(); + if (data.error === 'invalid_grant') + throw new Error( + 'Google Drive authorization expired or was revoked, please reconnect your Google account in the settings.', + ); + throw new Error( + `Google Drive token refresh failed: ${describeAuthError(data, response.status)}`, + ); + } +} + +/** Injects the bearer token into every remote request and retries once on 401. */ +export function bearerMiddleware(request: Request, manager: TokenManager): Request { + return async (params) => { + const base: RequestParam = typeof params === 'string' ? { url: params } : params; + const send = (token: string) => + request({ ...base, headers: { ...base.headers, Authorization: `Bearer ${token}` } }); + let response = await send(await manager.getToken()); + if (response.status === 401) { + manager.invalidate(); + response = await send(await manager.getToken(true)); + } + return response; + }; +} diff --git a/packages/gdrive/src/gdrive/check-connection.ts b/packages/gdrive/src/gdrive/check-connection.ts new file mode 100644 index 00000000..dede7685 --- /dev/null +++ b/packages/gdrive/src/gdrive/check-connection.ts @@ -0,0 +1,19 @@ +import type { CheckConnectionResult, Request } from '@hesprs/sync-engine-sdk'; +import { DRIVE_API, buildUrl, parseDriveError } from './api'; + +export default async function checkConnection(request: Request): Promise { + try { + const response = await request({ + method: 'GET', + url: buildUrl(DRIVE_API, '/about', { fields: 'user(emailAddress)' }), + }); + if (response.status >= 200 && response.status < 300) return { success: true } as const; + return { + reason: parseDriveError(response) ?? `HTTP ${response.status}`, + success: false, + } as const; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { reason: errorMessage, success: false } as const; + } +} diff --git a/packages/gdrive/src/gdrive/fs.ts b/packages/gdrive/src/gdrive/fs.ts new file mode 100644 index 00000000..c2838f02 --- /dev/null +++ b/packages/gdrive/src/gdrive/fs.ts @@ -0,0 +1,494 @@ +import type { + Binary, + FileStat, + ListReporter, + Request, + RequestParam, + RequestResponse, + RootFs, + Stat, +} from '@hesprs/sync-engine-sdk'; +import { textToUint8Array } from '@repo/shared/binary'; +import { getStatus } from '@repo/shared/get-status'; +import { basename, dirname, isFolder, normalizeBaseDir } from '@repo/shared/path'; +import type { DriveFile, DriveFileList } from './api'; +import { + DRIVE_API, + DRIVE_UPLOAD_API, + FILE_FIELDS, + FOLDER_MIME, + buildUrl, + escapeQuery, + parseDriveError, + safeJson, + toFileStat, +} from './api'; +import createRangeReadStream from './read-stream'; +import { RESUMABLE_CHUNK_SIZE, buildMultipartBody, guessMimeType, resumableUpload } from './upload'; + +export type GdriveFsOptions = { + account: string; + baseDirectory: string; + useTrash: boolean; + request: Request; +}; + +const READ_CHUNK_SIZE = 2 * 1024 * 1024; // 2 MiB +const READ_MAX_CONCURRENT = 8; +const PAGE_SIZE = 1000; +const WRITE_FIELDS = 'id,md5Checksum'; + +function notFoundError(key: string): Error { + const error = new Error(`Google Drive: ${key} does not exist.`); + (error as { status?: number }).status = 404; + return error; +} + +/** + * Google Drive stores files by immutable id inside real folders, while the + * sync engine speaks path keys. This class translates keys to ids on demand + * and caches the mapping for the lifetime of the instance. Only files created + * through this module are visible because of the `drive.file` OAuth scope. + */ +export default class GdriveFs implements RootFs { + private readonly request: Request; + private readonly baseDirectory: string; + private readonly useTrash: boolean; + /** Path key (`'/'`, `folder/`, `folder/note.md`) to Drive file id. */ + private readonly ids = new Map(); + + constructor(private readonly options: GdriveFsOptions) { + if (!options.request) throw new Error('Google Drive request is required.'); + this.request = options.request; + this.baseDirectory = normalizeBaseDir(options.baseDirectory); + this.useTrash = options.useTrash; + } + + getUid(): string { + return `gdrive~${this.options.account}~${this.baseDirectory}`; + } + + private async requestOrThrow(params: RequestParam): Promise { + const response = await this.request(params); + if (response.status >= 200 && response.status < 300) return response; + const error = new Error( + parseDriveError(response) ?? + `Google Drive request failed: ${response.status} ${params.method} ${params.url}`, + ); + (error as { status?: number }).status = response.status; + throw error; + } + + private async lookupChild( + parentId: string, + name: string, + folder: boolean, + ): Promise { + const mimeClause = folder + ? ` and mimeType = '${FOLDER_MIME}'` + : ` and mimeType != '${FOLDER_MIME}'`; + const url = buildUrl(DRIVE_API, '/files', { + fields: `files(${FILE_FIELDS})`, + orderBy: 'modifiedTime desc', + pageSize: '2', + q: `'${escapeQuery(parentId)}' in parents and name = '${escapeQuery(name)}' and trashed = false${mimeClause}`, + }); + const response = await this.requestOrThrow({ method: 'GET', url }); + return (safeJson(response) as DriveFileList).files?.[0]; + } + + private async createFolder(parentId: string, name: string): Promise { + const response = await this.requestOrThrow({ + body: textToUint8Array( + JSON.stringify({ mimeType: FOLDER_MIME, name, parents: [parentId] }), + ), + headers: { 'Content-Type': 'application/json; charset=UTF-8' }, + method: 'POST', + url: buildUrl(DRIVE_API, '/files', { fields: 'id' }), + }); + const created = safeJson(response) as DriveFile; + if (!created.id) throw new Error('Google Drive did not return an id for a created folder!'); + return created.id; + } + + /** + * Resolves the base directory to its folder id, creating missing folders + * when `create` is set. Always resolves the real root id (never the `root` + * alias) so listing can match ids returned in `parents`. + */ + private async ensureBase(create: boolean): Promise { + const cached = this.ids.get('/'); + if (cached !== undefined) return cached; + const rootResponse = await this.requestOrThrow({ + method: 'GET', + url: buildUrl(DRIVE_API, '/files/root', { fields: 'id' }), + }); + let currentId = (safeJson(rootResponse) as DriveFile).id; + if (!currentId) throw new Error('Google Drive did not return the root folder id!'); + for (const segment of this.baseDirectory.split('/').filter((part) => part !== '')) { + const existing = await this.lookupChild(currentId, segment, true); + if (existing) currentId = existing.id; + else if (create) currentId = await this.createFolder(currentId, segment); + else return undefined; + } + this.ids.set('/', currentId); + return currentId; + } + + /** Resolves a key to its id; `create` builds missing folders along the way. */ + private async resolveId(key: string, create: boolean): Promise { + if (key === '/') return this.ensureBase(create); + const cached = this.ids.get(key); + if (cached !== undefined) return cached; + const parentId = await this.resolveId(dirname(key), create); + if (parentId === undefined) return undefined; + const existing = await this.lookupChild(parentId, basename(key), isFolder(key)); + if (existing) { + this.ids.set(key, existing.id); + return existing.id; + } + if (!create || !isFolder(key)) return undefined; + const created = await this.createFolder(parentId, basename(key)); + this.ids.set(key, created); + return created; + } + + /** Fresh metadata lookup for a file key (also refreshes the id cache). */ + private async resolveEntry(key: string): Promise { + const parentId = await this.resolveId(dirname(key), false); + if (parentId === undefined) return undefined; + const entry = await this.lookupChild(parentId, basename(key), isFolder(key)); + if (entry) this.ids.set(key, entry.id); + return entry; + } + + private async requireId(key: string): Promise { + const cached = this.ids.get(key); + if (cached !== undefined) return cached; + const id = isFolder(key) + ? await this.resolveId(key, false) + : (await this.resolveEntry(key))?.id; + if (id === undefined) throw notFoundError(key); + return id; + } + + private cachedKeysUnder(folderKey: string): Array { + const keys: Array = []; + for (const cachedKey of this.ids.keys()) + if (cachedKey.startsWith(folderKey) && cachedKey !== folderKey) keys.push(cachedKey); + return keys; + } + + private dropCache(key: string): void { + this.ids.delete(key); + if (!isFolder(key)) return; + for (const cachedKey of this.cachedKeysUnder(key)) this.ids.delete(cachedKey); + } + + private remapCache(oldKey: string, newKey: string): void { + const id = this.ids.get(oldKey); + this.ids.delete(oldKey); + if (id !== undefined) this.ids.set(newKey, id); + if (!isFolder(oldKey)) return; + for (const cachedKey of this.cachedKeysUnder(oldKey)) { + const childId = this.ids.get(cachedKey); + this.ids.delete(cachedKey); + if (childId !== undefined) + this.ids.set(newKey + cachedKey.slice(oldKey.length), childId); + } + } + + async read(key: string): Promise { + const id = await this.requireId(key); + const response = await this.requestOrThrow({ + method: 'GET', + url: buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }), + }); + return response.bytes(); + } + + async readStream(key: string, { size }: FileStat): Promise> { + const id = await this.requireId(key); + const url = buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }); + return createRangeReadStream({ + chunkSize: READ_CHUNK_SIZE, + maxConcurrent: READ_MAX_CONCURRENT, + requestRange: async (start, endInclusive) => { + const response = await this.requestOrThrow({ + headers: { Range: `bytes=${start}-${endInclusive}` }, + method: 'GET', + url, + }); + return response.bytes(); + }, + size, + }); + } + + async write(key: string, value: Binary, stat: FileStat): Promise { + const existing = await this.resolveEntry(key); + const modifiedTime = new Date(stat.mtime).toISOString(); + const mimeType = guessMimeType(basename(key)); + let response: RequestResponse; + if (existing) { + const { body, contentType } = buildMultipartBody({ modifiedTime }, value, mimeType); + response = await this.requestOrThrow({ + body, + headers: { 'Content-Type': contentType }, + method: 'PATCH', + url: buildUrl(DRIVE_UPLOAD_API, `/files/${existing.id}`, { + fields: WRITE_FIELDS, + uploadType: 'multipart', + }), + }); + } else { + const parentId = await this.resolveId(dirname(key), true); + if (parentId === undefined) throw notFoundError(dirname(key)); + const { body, contentType } = buildMultipartBody( + { mimeType, modifiedTime, name: basename(key), parents: [parentId] }, + value, + mimeType, + ); + response = await this.requestOrThrow({ + body, + headers: { 'Content-Type': contentType }, + method: 'POST', + url: buildUrl(DRIVE_UPLOAD_API, '/files', { + fields: WRITE_FIELDS, + uploadType: 'multipart', + }), + }); + } + const file = safeJson(response) as DriveFile; + if (file.id) this.ids.set(key, file.id); + return file.md5Checksum ?? `${stat.mtime}~${stat.size}`; + } + + async writeStream(key: string, value: ReadableStream, stat: FileStat): Promise { + if (stat.size < RESUMABLE_CHUNK_SIZE) + return this.write(key, await collectStreamToBinary(value), stat); + const existing = await this.resolveEntry(key); + const modifiedTime = new Date(stat.mtime).toISOString(); + let file: DriveFile; + if (existing) + file = await resumableUpload( + { + initiateUrl: buildUrl(DRIVE_UPLOAD_API, `/files/${existing.id}`, { + fields: WRITE_FIELDS, + uploadType: 'resumable', + }), + metadata: { modifiedTime }, + method: 'PATCH', + request: this.request, + stat, + }, + value, + ); + else { + const parentId = await this.resolveId(dirname(key), true); + if (parentId === undefined) throw notFoundError(dirname(key)); + file = await resumableUpload( + { + initiateUrl: buildUrl(DRIVE_UPLOAD_API, '/files', { + fields: WRITE_FIELDS, + uploadType: 'resumable', + }), + metadata: { + mimeType: guessMimeType(basename(key)), + modifiedTime, + name: basename(key), + parents: [parentId], + }, + method: 'POST', + request: this.request, + stat, + }, + value, + ); + } + if (file.id) this.ids.set(key, file.id); + return file.md5Checksum ?? `${stat.mtime}~${stat.size}`; + } + + async delete(key: string): Promise { + let id: string; + try { + id = await this.requireId(key); + } catch (error) { + if (getStatus(error) === 404) return; + throw error; + } + try { + await this.requestOrThrow( + this.useTrash + ? { + body: textToUint8Array(JSON.stringify({ trashed: true })), + headers: { 'Content-Type': 'application/json; charset=UTF-8' }, + method: 'PATCH', + url: buildUrl(DRIVE_API, `/files/${id}`, { fields: 'id' }), + } + : { + method: 'DELETE', + url: buildUrl(DRIVE_API, `/files/${id}`), + }, + ); + } catch (error) { + if (getStatus(error) !== 404) throw error; + } + this.dropCache(key); + } + + async move(oldKey: string, newKey: string): Promise { + const id = await this.requireId(oldKey); + const oldParentId = await this.resolveId(dirname(oldKey), false); + const newParentId = await this.resolveId(dirname(newKey), true); + if (newParentId === undefined) throw notFoundError(dirname(newKey)); + const query: Record = { fields: 'id' }; + if (oldParentId !== undefined && oldParentId !== newParentId) { + query.addParents = newParentId; + query.removeParents = oldParentId; + } + await this.requestOrThrow({ + body: textToUint8Array(JSON.stringify({ name: basename(newKey) })), + headers: { 'Content-Type': 'application/json; charset=UTF-8' }, + method: 'PATCH', + url: buildUrl(DRIVE_API, `/files/${id}`, query), + }); + this.remapCache(oldKey, newKey); + } + + /** + * Drive folders always need an existing parent id, so missing parents are + * created regardless of `recursive`. + */ + async mkdir(key: string): Promise { + const id = await this.resolveId(key, true); + if (id === undefined) throw notFoundError(key); + } + + async stat(key: string): Promise { + if (key === '/') return { isDir: true, key }; + if (isFolder(key)) { + const id = await this.resolveId(key, false); + if (id === undefined) throw notFoundError(key); + return { isDir: true, key }; + } + const entry = await this.resolveEntry(key); + if (!entry) throw notFoundError(key); + return toFileStat(key, entry); + } + + async exists(key: string): Promise { + if (key === '/') return true; + try { + await this.stat(key); + return true; + } catch (error) { + if (getStatus(error) === 404) return false; + throw error; + } + } + + /** + * Fetches every visible file in one paginated query (the `drive.file` + * scope limits results to files this module created), then walks the tree + * under the requested key so the reporter can steer traversal. + */ + async list(key: string, reporter: ListReporter): Promise> { + const startId = + key === '/' ? await this.ensureBase(true) : await this.resolveId(key, false); + if (startId === undefined) throw notFoundError(key); + const all: Array = []; + let pageToken: string | undefined; + do { + const query: Record = { + fields: `nextPageToken,files(${FILE_FIELDS})`, + pageSize: String(PAGE_SIZE), + q: 'trashed = false', + }; + if (pageToken) query.pageToken = pageToken; + const response = await this.requestOrThrow({ + method: 'GET', + url: buildUrl(DRIVE_API, '/files', query), + }); + const parsed = safeJson(response) as DriveFileList; + all.push(...(parsed.files ?? [])); + pageToken = parsed.nextPageToken; + } while (pageToken); + + const childrenByParent = new Map>(); + for (const file of all) { + const parent = file.parents?.[0]; + if (!parent || file.name.includes('/')) continue; + const siblings = childrenByParent.get(parent); + if (siblings) siblings.push(file); + else childrenByParent.set(parent, [file]); + } + + const results: Array = []; + const total = all.length; + let completed = 0; + const walk = async (folderId: string, prefix: string): Promise => { + for (const entry of dedupeChildren(childrenByParent.get(folderId) ?? [])) { + const folder = entry.mimeType === FOLDER_MIME; + const childKey = `${prefix}${entry.name}${folder ? '/' : ''}`; + completed = Math.min(completed + 1, total); + const verdict = await reporter({ completed, current: childKey, total }); + if (verdict === 'exclude') continue; + this.ids.set(childKey, entry.id); + if (folder) { + results.push({ isDir: true, key: childKey }); + if (verdict === 'advance') await walk(entry.id, childKey); + } else results.push(toFileStat(childKey, entry)); + } + }; + await walk(startId, key === '/' ? '' : key); + return results; + } +} + +/** + * Drive allows duplicate names inside one folder; syncing needs one entry per + * key, so the most recently modified file (or the first folder) wins. + */ +function dedupeChildren(entries: Array): Array { + if (entries.length < 2) return entries; + const byName = new Map(); + for (const entry of entries) { + const nameKey = `${entry.mimeType === FOLDER_MIME ? 'd' : 'f'}~${entry.name}`; + const existing = byName.get(nameKey); + if (!existing) { + byName.set(nameKey, entry); + continue; + } + if ( + entry.mimeType !== FOLDER_MIME && + Date.parse(entry.modifiedTime ?? '') > Date.parse(existing.modifiedTime ?? '') + ) + byName.set(nameKey, entry); + } + return [...byName.values()]; +} + +async function collectStreamToBinary(source: ReadableStream): Promise { + const reader = source.getReader(); + const chunks: Array = []; + let total = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + total += value.byteLength; + } + } finally { + reader.releaseLock(); + } + const result = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} diff --git a/packages/gdrive/src/gdrive/read-stream.ts b/packages/gdrive/src/gdrive/read-stream.ts new file mode 100644 index 00000000..75f855f0 --- /dev/null +++ b/packages/gdrive/src/gdrive/read-stream.ts @@ -0,0 +1,111 @@ +import type { Binary } from '@hesprs/sync-engine-sdk'; + +export type CreateRangeReadStreamOptions = { + size: number; + chunkSize: number; + maxConcurrent: number; + requestRange: (start: number, endInclusive: number) => Promise; +}; + +/** Concurrent range-request read stream, mirroring the S3 module's implementation. */ +export default function createRangeReadStream({ + size, + chunkSize, + maxConcurrent, + requestRange, +}: CreateRangeReadStreamOptions): ReadableStream { + const totalChunks = size === 0 ? 0 : Math.ceil(size / chunkSize); + const maxBufferedBytes = chunkSize * maxConcurrent; + if (totalChunks === 0) + return new ReadableStream({ + start(controller) { + controller.close(); + }, + }); + + let controllerRef: ReadableStreamDefaultController | undefined; + let nextChunkIndex = 0; + let nextPendingIndex = 0; + let inFlight = 0; + let closed = false; + let consumerReady = false; + let pendingBytes = 0; + const pending = new Map(); + + const closeIfDone = () => { + if (closed || !controllerRef) return; + if (nextPendingIndex < totalChunks || inFlight > 0) return; + closed = true; + controllerRef.close(); + }; + + const flush = () => { + if (!controllerRef || closed) return; + while (consumerReady && pending.has(nextPendingIndex)) { + const chunk = pending.get(nextPendingIndex); + if (!chunk) break; + pending.delete(nextPendingIndex); + pendingBytes -= chunk.byteLength; + controllerRef.enqueue(chunk); + consumerReady = (controllerRef.desiredSize ?? 0) > 0; + nextPendingIndex++; + } + closeIfDone(); + }; + + const canScheduleNext = () => + controllerRef !== undefined && + !closed && + consumerReady && + inFlight < maxConcurrent && + nextChunkIndex < totalChunks && + pendingBytes < maxBufferedBytes; + + const requestChunk = (currentIndex: number) => { + inFlight++; + + const start = currentIndex * chunkSize; + const endInclusive = Math.min(start + chunkSize - 1, size - 1); + + void requestRange(start, endInclusive) + .then((buffer) => { + if (closed) return; + pending.set(currentIndex, buffer); + pendingBytes += buffer.byteLength; + inFlight--; + flush(); + schedule(); + }) + .catch((error: unknown) => { + if (closed) return; + closed = true; + controllerRef?.error(error); + }); + }; + + const schedule = () => { + while (canScheduleNext()) { + const currentIndex = nextChunkIndex; + nextChunkIndex++; + requestChunk(currentIndex); + } + }; + + return new ReadableStream( + { + cancel() { + closed = true; + }, + pull(controller) { + controllerRef = controller; + consumerReady = true; + flush(); + schedule(); + }, + start(controller) { + controllerRef = controller; + }, + }, + { highWaterMark: 0 }, + ); +} diff --git a/packages/gdrive/src/gdrive/upload.ts b/packages/gdrive/src/gdrive/upload.ts new file mode 100644 index 00000000..676e8e1f --- /dev/null +++ b/packages/gdrive/src/gdrive/upload.ts @@ -0,0 +1,142 @@ +import type { Binary, FileStat, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; +import { concatBinary, textToUint8Array } from '@repo/shared/binary'; +import type { DriveFile } from './api'; +import { getHeader, parseDriveError, safeJson } from './api'; + +/** Google Drive resumable uploads require chunk sizes in multiples of 256 KiB. */ +export const RESUMABLE_CHUNK_SIZE = 8 * 1024 * 1024; + +const MIME_BY_EXTENSION: Record = { + base: 'application/json', + canvas: 'application/json', + css: 'text/css', + gif: 'image/gif', + html: 'text/html', + jpeg: 'image/jpeg', + jpg: 'image/jpeg', + js: 'text/javascript', + json: 'application/json', + m4a: 'audio/mp4', + md: 'text/markdown', + mp3: 'audio/mpeg', + mp4: 'video/mp4', + pdf: 'application/pdf', + png: 'image/png', + svg: 'image/svg+xml', + txt: 'text/plain', + webm: 'video/webm', + webp: 'image/webp', +}; + +/** + * Content type declared for uploaded bytes so files keep useful previews in + * the Drive web interface. + */ +export function guessMimeType(name: string): string { + const dotIndex = name.lastIndexOf('.'); + if (dotIndex === -1) return 'application/octet-stream'; + const extension = name.slice(dotIndex + 1).toLowerCase(); + return MIME_BY_EXTENSION[extension] ?? 'application/octet-stream'; +} + +let boundaryCounter = 0; + +export function buildMultipartBody( + metadata: object, + content: Binary, + contentMimeType: string, +): { body: Binary; contentType: string } { + boundaryCounter++; + const boundary = `sync-engine-gdrive-${boundaryCounter.toString(36)}-${Math.random().toString(36).slice(2)}`; + const head = textToUint8Array( + `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify(metadata)}\r\n--${boundary}\r\nContent-Type: ${contentMimeType}\r\n\r\n`, + ); + const tail = textToUint8Array(`\r\n--${boundary}--`); + return { + body: concatBinary(head, content, tail), + contentType: `multipart/related; boundary=${boundary}`, + }; +} + +export type ResumableUploadOptions = { + initiateUrl: string; + method: 'PATCH' | 'POST'; + metadata: object; + stat: FileStat; + /** Raw composed request — resumable chunk responses use non-2xx status 308. */ + request: (params: RequestParam) => Promise; +}; + +export async function resumableUpload( + options: ResumableUploadOptions, + value: ReadableStream, +): Promise { + const initiate = await options.request({ + body: textToUint8Array(JSON.stringify(options.metadata)), + headers: { + 'Content-Type': 'application/json; charset=UTF-8', + 'X-Upload-Content-Length': String(options.stat.size), + }, + method: options.method, + url: options.initiateUrl, + }); + if (initiate.status < 200 || initiate.status >= 300) + throw new Error( + parseDriveError(initiate) ?? + `Google Drive resumable upload initiation failed: ${initiate.status}`, + ); + const location = getHeader(initiate.headers, 'location'); + if (!location) throw new Error('Google Drive did not return a resumable upload session URL!'); + + const total = options.stat.size; + let offset = 0; + let final: RequestResponse | undefined; + const putChunk = async ( + chunk: Binary, + isLast: boolean, + ): Promise => { + const start = offset; + const end = offset + chunk.byteLength - 1; + offset += chunk.byteLength; + const response = await options.request({ + body: chunk, + headers: { 'Content-Range': `bytes ${start}-${end}/${total}` }, + method: 'PUT', + url: location, + }); + if (response.status === 308) { + if (isLast) throw new Error('Google Drive resumable upload ended prematurely.'); + return undefined; + } + if (response.status >= 200 && response.status < 300) return response; + throw new Error( + parseDriveError(response) ?? `Google Drive resumable upload failed: ${response.status}`, + ); + }; + + const reader = value.getReader(); + let pending = new Uint8Array(0); + try { + while (final === undefined) { + const { done, value: chunk } = await reader.read(); + if (done) break; + pending = concatBinary(pending, chunk); + // Hold back at least one byte so the closing chunk is never empty. + while (pending.byteLength > RESUMABLE_CHUNK_SIZE && final === undefined) { + const part = pending.slice(0, RESUMABLE_CHUNK_SIZE); + pending = pending.slice(RESUMABLE_CHUNK_SIZE); + final = await putChunk(part, false); + } + } + final ??= await putChunk(pending, true); + } catch (error) { + // Best-effort session cancellation; Drive also expires sessions on its own. + await options.request({ method: 'DELETE', url: location }).catch(() => {}); + throw error; + } finally { + reader.releaseLock(); + } + if (final === undefined) + throw new Error('Google Drive resumable upload finished without a response.'); + return safeJson(final) as DriveFile; +} diff --git a/packages/gdrive/src/handle-input.ts b/packages/gdrive/src/handle-input.ts new file mode 100644 index 00000000..8724b9d2 --- /dev/null +++ b/packages/gdrive/src/handle-input.ts @@ -0,0 +1,30 @@ +import type { TextComponent } from 'obsidian'; +import { Notice } from 'obsidian'; + +export default function handleInput({ + text, + saveSettings, + processValue, + stringify = String, + key, + settings, + invalidValue, +}: { + text: TextComponent; + saveSettings: () => Promise; + processValue: (value: string) => T | false; + key: K; + settings: NoInfer>; + stringify?: (value: T) => string; + invalidValue: string; +}) { + text.inputEl.addEventListener('blur', () => { + const value = processValue(text.getValue()); + if (value === false) new Notice(invalidValue); + else if (settings[key] !== value) { + settings[key] = value; + void saveSettings(); + } + text.setValue(stringify(settings[key])); + }); +} diff --git a/packages/gdrive/src/i18n.ts b/packages/gdrive/src/i18n.ts new file mode 100644 index 00000000..2f4482e9 --- /dev/null +++ b/packages/gdrive/src/i18n.ts @@ -0,0 +1,37 @@ +import type { GdriveTranslations } from './setting'; + +const en: GdriveTranslations = { + account: 'Google account', + accountConnected: 'Connected as {{account}}.', + accountNotConnected: + 'Not connected. Enter the OAuth client credentials above, then connect your Google account.', + baseDirectory: 'Base directory', + baseDirectoryDescription: + 'Folder in Google Drive that holds this vault. Created automatically on the first sync — do not create it manually in Drive, files added outside this plugin stay invisible to it.', + baseDirectoryPlaceholder: 'my-vault/', + clientId: 'OAuth client ID', + clientIdDescription: + 'Client ID of your own Google Cloud OAuth client (application type "TV and Limited Input devices") with the Google Drive API enabled.', + clientIdPlaceholder: 'xxxxxxxx.apps.googleusercontent.com', + clientSecret: 'OAuth client secret', + clientSecretDescription: 'Client secret of the same OAuth client.', + codeCopied: 'Copied', + configureFirst: 'Enter the OAuth client ID and client secret first.', + connect: 'Connect', + connectSuccess: 'Connected to Google Drive as {{account}}.', + copyCode: 'Copy code', + deviceCodeInstruction: + 'On any device, visit {{url}} and enter the code below, then approve access.', + deviceCodeTitle: 'Connect Google Drive', + disconnect: 'Disconnect', + disconnected: 'Google Drive disconnected.', + gdrive: 'Google Drive', + openVerificationPage: 'Open Google', + reconnect: 'Reconnect', + useTrash: 'Delete to trash', + useTrashDescription: + 'Move remotely deleted files to the Google Drive trash instead of deleting them permanently. Drive clears its trash after 30 days.', + waitingApproval: 'Waiting for approval…', +}; + +export default en; diff --git a/packages/gdrive/src/index.ts b/packages/gdrive/src/index.ts new file mode 100644 index 00000000..b4eb6807 --- /dev/null +++ b/packages/gdrive/src/index.ts @@ -0,0 +1,130 @@ +import type { + Context, + ObsidianLanguageCode, + RemoteFsEntry, + RemoteRequestMiddlewareEntry, + SelectFromContext, + SettingEntry, + Settings, + Translate, + Translations, + TranslationResource, +} from '@hesprs/sync-engine-sdk'; +import type { App } from 'obsidian'; +import type { GdriveTranslations } from './setting'; +import { TokenManager, bearerMiddleware } from './gdrive/auth'; +import requestUrlHttp from './gdrive/auth-http'; +import checkConnection from './gdrive/check-connection'; +import GdriveFs from './gdrive/fs'; +import en from './i18n'; +import gdriveSetting from './setting'; + +export type GdriveSettings = { + account: string; + baseDirectory: string; + clientId: string; + clientSecret: string; + refreshToken: string; + useTrash: boolean; +}; + +export default class Gdrive { + private readonly cleanup: Array<() => void> = []; + private readonly tokenManager: TokenManager; + + constructor( + private readonly ctx: SelectFromContext<{ + translate: Translate; + registerRemoteFs: (id: string, entry: RemoteFsEntry) => () => void; + app: App; + registerRemoteRequestMiddleware: (entry: RemoteRequestMiddlewareEntry) => () => void; + registerSetting: (entry: SettingEntry) => () => void; + registerI18n: (lang: ObsidianLanguageCode, translations: TranslationResource) => void; + rerenderSettingTab: () => void; + }>, + ) { + if (!this.moduleSettings.baseDirectory) + this.moduleSettings.baseDirectory = `${ctx.app.vault.getName()}/`; + ctx.registerI18n('en', en); + this.tokenManager = new TokenManager(requestUrlHttp, () => this.resolveAuth()); + } + + readonly moduleSettings: GdriveSettings = { + account: '', + baseDirectory: '', + clientId: '', + clientSecret: '', + refreshToken: '', + useTrash: true, + }; + + declare settings: Settings; + + readonly start = () => { + const { translate, registerRemoteFs, registerRemoteRequestMiddleware, registerSetting } = + this.ctx; + this.cleanup.push( + registerRemoteFs('gdrive', { + checkConnection: (request) => { + try { + this.resolveConfig(); + } catch (error) { + return { + reason: error instanceof Error ? error.message : String(error), + success: false, + }; + } + return checkConnection(request); + }, + instantiate: (request) => { + const config = this.resolveConfig(); + return new GdriveFs({ + account: config.account, + baseDirectory: config.baseDirectory, + request, + useTrash: config.useTrash, + }); + }, + prettyName: () => translate('gdrive'), + }), + registerRemoteRequestMiddleware({ + apply: (request) => { + if (this.settings.remoteFs !== 'gdrive') return; + return bearerMiddleware(request, this.tokenManager); + }, + priority: 305, + }), + registerSetting({ + apply: gdriveSetting(this.ctx as Context, this.moduleSettings, this.tokenManager), + priority: 683, + }), + ); + }; + + private readonly resolveAuth = () => { + const { + clientId, + clientSecret: clientSecretId, + refreshToken: refreshTokenId, + } = this.moduleSettings; + const { secretStorage } = this.ctx.app; + const clientSecret = secretStorage.getSecret(clientSecretId); + if (!clientId || clientSecret === null || clientSecret === '') + throw new Error('Please configure the Google Drive OAuth client!'); + const refreshToken = secretStorage.getSecret(refreshTokenId); + if (refreshToken === null || refreshToken === '') + throw new Error('Please connect your Google account in the Sync Engine settings!'); + return { clientId, clientSecret, refreshToken }; + }; + + private readonly resolveConfig = () => { + this.resolveAuth(); + const { account, baseDirectory, useTrash } = this.moduleSettings; + return { account: account || 'unknown', baseDirectory, useTrash }; + }; + + readonly dispose = () => { + this.cleanup.forEach((fn) => fn()); + this.cleanup.length = 0; + }; +} diff --git a/packages/gdrive/src/setting.ts b/packages/gdrive/src/setting.ts new file mode 100644 index 00000000..0b1e1439 --- /dev/null +++ b/packages/gdrive/src/setting.ts @@ -0,0 +1,291 @@ +import type { GdriveSettings } from '@'; +import type { + CallableOrObjectTree, + LabelDefinition, + Translate, + Translations, +} from '@hesprs/sync-engine-sdk'; +import type { App, SettingGroupItem } from 'obsidian'; +import { s } from '@hesprs/sync-engine-sdk'; +import { normalizeBaseDir } from '@repo/shared/path'; +import { Modal, Notice, SecretComponent } from 'obsidian'; +import type { TokenManager } from './gdrive/auth'; +import { REFRESH_TOKEN_SECRET_ID, pollDeviceToken, startDeviceAuthorization } from './gdrive/auth'; +import requestUrlHttp from './gdrive/auth-http'; +import handleInput from './handle-input'; + +export type GdriveTranslations = { + gdrive: string; + clientId: string; + clientIdDescription: string; + clientIdPlaceholder: string; + clientSecret: string; + clientSecretDescription: string; + account: string; + accountConnected: string; + accountNotConnected: string; + connect: string; + reconnect: string; + disconnect: string; + disconnected: string; + configureFirst: string; + deviceCodeTitle: string; + deviceCodeInstruction: string; + copyCode: string; + codeCopied: string; + openVerificationPage: string; + waitingApproval: string; + connectSuccess: string; + baseDirectory: string; + baseDirectoryDescription: string; + baseDirectoryPlaceholder: string; + useTrash: string; + useTrashDescription: string; +}; + +type DeviceCodeModalOptions = { + title: string; + instruction: string; + userCode: string; + verificationUrl: string; + copyLabel: string; + copiedLabel: string; + openLabel: string; + waitingLabel: string; + onClose: () => void; +}; + +class DeviceCodeModal extends Modal { + constructor( + app: App, + private readonly options: DeviceCodeModalOptions, + ) { + super(app); + } + + override onOpen(): void { + const { contentEl, titleEl } = this; + titleEl.setText(this.options.title); + contentEl.createEl('p', { text: this.options.instruction }); + const codeEl = contentEl.createEl('div', { text: this.options.userCode }); + codeEl.setCssStyles({ + fontSize: '2em', + fontWeight: '700', + letterSpacing: '0.15em', + margin: '0.5em 0', + textAlign: 'center', + userSelect: 'text', + }); + const buttonRow = contentEl.createEl('div'); + buttonRow.setCssStyles({ + display: 'flex', + gap: '0.5em', + justifyContent: 'center', + marginBottom: '0.75em', + }); + const copyButton = buttonRow.createEl('button', { text: this.options.copyLabel }); + copyButton.addEventListener('click', () => { + void navigator.clipboard.writeText(this.options.userCode); + copyButton.setText(this.options.copiedLabel); + }); + const openButton = buttonRow.createEl('button', { + cls: 'mod-cta', + text: this.options.openLabel, + }); + openButton.addEventListener('click', () => { + window.open(this.options.verificationUrl); + }); + const statusEl = contentEl.createEl('p', { text: this.options.waitingLabel }); + statusEl.setCssStyles({ opacity: '0.7', textAlign: 'center' }); + } + + override onClose(): void { + this.options.onClose(); + this.contentEl.empty(); + } +} + +export default function gdriveSetting( + { + translate, + saveSettings, + app, + matchLabel, + rerenderSettingTab, + }: { + translate: Translate; + saveSettings: () => Promise; + app: App; + matchLabel: () => LabelDefinition; + rerenderSettingTab: () => void; + }, + settings: GdriveSettings, + tokenManager: TokenManager, +): CallableOrObjectTree { + const invalidValue = translate('invalidValue'); + + const connectGoogle = async () => { + const clientId = settings.clientId.trim(); + const clientSecret = app.secretStorage.getSecret(settings.clientSecret); + if (!clientId || clientSecret === null || clientSecret === '') { + new Notice(translate('configureFirst')); + return; + } + let cancelled = false; + try { + const authorization = await startDeviceAuthorization(requestUrlHttp, clientId); + let finished = false; + const modal = new DeviceCodeModal(app, { + copiedLabel: translate('codeCopied'), + copyLabel: translate('copyCode'), + instruction: translate('deviceCodeInstruction', { + url: authorization.verificationUrl, + }), + onClose: () => { + if (!finished) cancelled = true; + }, + openLabel: translate('openVerificationPage'), + title: translate('deviceCodeTitle'), + userCode: authorization.userCode, + verificationUrl: authorization.verificationUrl, + waitingLabel: translate('waitingApproval'), + }); + modal.open(); + try { + const token = await pollDeviceToken(requestUrlHttp, { + authorization, + clientId, + clientSecret, + isCancelled: () => cancelled, + }); + finished = true; + app.secretStorage.setSecret(REFRESH_TOKEN_SECRET_ID, token.refreshToken); + settings.refreshToken = REFRESH_TOKEN_SECRET_ID; + if (token.email) settings.account = token.email; + else if (!settings.account) settings.account = 'Google account'; + await saveSettings(); + tokenManager.invalidate(); + new Notice(translate('connectSuccess', { account: settings.account })); + } finally { + finished = true; + modal.close(); + } + rerenderSettingTab(); + } catch (error) { + if (!cancelled) new Notice(error instanceof Error ? error.message : String(error)); + } + }; + + return { + 683: s( + (self) => ({ + heading: translate('gdrive'), + items: Object.values(self).map((node) => node(node) as SettingGroupItem), + type: 'group', + }), + { + 1000: s(() => ({ + desc: translate('clientIdDescription'), + name: translate('clientId'), + render: (setting) => { + setting.addText((text) => { + text.setPlaceholder(translate('clientIdPlaceholder')).setValue( + settings.clientId, + ); + handleInput({ + invalidValue, + key: 'clientId', + processValue: (value) => value.trim(), + saveSettings, + settings, + text, + }); + }); + }, + })), + 2000: s(() => ({ + desc: translate('clientSecretDescription'), + name: translate('clientSecret'), + render: (setting) => { + setting.addComponent((element) => + new SecretComponent(app, element) + .setValue(settings.clientSecret) + .onChange((value) => { + settings.clientSecret = value ?? ''; + void saveSettings(); + }), + ); + }, + })), + 3000: s(() => ({ + desc: settings.refreshToken + ? translate('accountConnected', { account: settings.account }) + : translate('accountNotConnected'), + name: translate('account'), + render: (setting) => { + if (settings.refreshToken) + setting.addButton((button) => + button.setButtonText(translate('disconnect')).onClick(() => { + settings.refreshToken = ''; + settings.account = ''; + tokenManager.invalidate(); + void saveSettings(); + new Notice(translate('disconnected')); + rerenderSettingTab(); + }), + ); + setting.addButton((button) => + button + .setButtonText( + settings.refreshToken + ? translate('reconnect') + : translate('connect'), + ) + .setCta() + .onClick(async () => { + button.setDisabled(true); + try { + await connectGoogle(); + } finally { + button.setDisabled(false); + } + }), + ); + }, + })), + 4000: s(() => ({ + desc: translate('baseDirectoryDescription'), + labels: [matchLabel()], + name: translate('baseDirectory'), + render: (setting) => { + setting.addText((text) => { + text.setPlaceholder(translate('baseDirectoryPlaceholder')).setValue( + settings.baseDirectory, + ); + handleInput({ + invalidValue, + key: 'baseDirectory', + processValue: (original) => normalizeBaseDir(original.trim()), + saveSettings, + settings, + text, + }); + }); + }, + })), + 5000: s(() => ({ + desc: translate('useTrashDescription'), + name: translate('useTrash'), + render: (setting) => { + setting.addToggle((toggle) => + toggle.setValue(settings.useTrash).onChange((value) => { + settings.useTrash = value; + void saveSettings(); + }), + ); + }, + })), + }, + ), + }; +} diff --git a/packages/gdrive/test/auth.test.ts b/packages/gdrive/test/auth.test.ts new file mode 100644 index 00000000..dcb9c478 --- /dev/null +++ b/packages/gdrive/test/auth.test.ts @@ -0,0 +1,208 @@ +import type { Request, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; +import { expect, test } from 'bun:test'; +import type { AuthHttp } from '@/gdrive/auth'; +import { + TokenManager, + bearerMiddleware, + decodeIdTokenEmail, + pollDeviceToken, + startDeviceAuthorization, +} from '@/gdrive/auth'; + +type AuthResponse = { status: number; body: unknown }; + +function scriptedHttp(responses: Array) { + const calls: Array<{ url: string; body: string }> = []; + const http: AuthHttp = ({ url, body }) => { + calls.push({ body: body ?? '', url }); + const next = responses.shift(); + if (!next) throw new Error('scripted http exhausted'); + return Promise.resolve({ json: () => next.body, status: next.status }); + }; + return { calls, http }; +} + +function fakeIdToken(email: string): string { + const payload = btoa(JSON.stringify({ email })) + .replaceAll('+', '-') + .replaceAll('/', '_') + .replaceAll('=', ''); + return `header.${payload}.signature`; +} + +const AUTHORIZATION = { + deviceCode: 'device-1', + expiresIn: 1800, + interval: 5, + userCode: 'ABCD-EFGH', + verificationUrl: 'https://www.google.com/device', +}; + +test('startDeviceAuthorization parses the device code response', async () => { + const { calls, http } = scriptedHttp([ + { + body: { + device_code: 'device-1', + expires_in: 900, + interval: 7, + user_code: 'ABCD-EFGH', + verification_url: 'https://www.google.com/device', + }, + status: 200, + }, + ]); + const authorization = await startDeviceAuthorization(http, 'client-1'); + expect(authorization).toStrictEqual({ + deviceCode: 'device-1', + expiresIn: 900, + interval: 7, + userCode: 'ABCD-EFGH', + verificationUrl: 'https://www.google.com/device', + }); + expect(calls[0]?.body).toContain('client_id=client-1'); + expect(calls[0]?.body).toContain('drive.file'); +}); + +test('startDeviceAuthorization surfaces Google error descriptions', async () => { + const { http } = scriptedHttp([ + { body: { error: 'invalid_client', error_description: 'Unknown client.' }, status: 401 }, + ]); + await expect(startDeviceAuthorization(http, 'client-1')).rejects.toThrow('Unknown client.'); +}); + +test('pollDeviceToken waits through pending, honors slow_down, and resolves tokens', async () => { + const { calls, http } = scriptedHttp([ + { body: { error: 'authorization_pending' }, status: 428 }, + { body: { error: 'slow_down' }, status: 428 }, + { + body: { + access_token: 'access-1', + expires_in: 3599, + id_token: fakeIdToken('user@example.com'), + refresh_token: 'refresh-1', + }, + status: 200, + }, + ]); + const sleeps: Array = []; + const result = await pollDeviceToken(http, { + authorization: AUTHORIZATION, + clientId: 'client-1', + clientSecret: 'secret-1', + sleep: (ms) => { + sleeps.push(ms); + return Promise.resolve(); + }, + }); + expect(result).toStrictEqual({ + accessToken: 'access-1', + email: 'user@example.com', + expiresIn: 3599, + refreshToken: 'refresh-1', + }); + expect(sleeps).toStrictEqual([5000, 5000, 10_000]); + expect(calls[0]?.body).toContain( + 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code', + ); +}); + +test('pollDeviceToken maps denial, expiry, and cancellation to clear errors', async () => { + const denied = scriptedHttp([{ body: { error: 'access_denied' }, status: 403 }]); + await expect( + pollDeviceToken(denied.http, { + authorization: AUTHORIZATION, + clientId: 'c', + clientSecret: 's', + sleep: () => Promise.resolve(), + }), + ).rejects.toThrow('denied'); + + const expired = scriptedHttp([{ body: { error: 'expired_token' }, status: 428 }]); + await expect( + pollDeviceToken(expired.http, { + authorization: AUTHORIZATION, + clientId: 'c', + clientSecret: 's', + sleep: () => Promise.resolve(), + }), + ).rejects.toThrow('expired'); + + const cancelled = scriptedHttp([]); + await expect( + pollDeviceToken(cancelled.http, { + authorization: AUTHORIZATION, + clientId: 'c', + clientSecret: 's', + isCancelled: () => true, + sleep: () => Promise.resolve(), + }), + ).rejects.toThrow('cancelled'); +}); + +test('TokenManager caches tokens, refreshes on expiry, and dedupes concurrent refreshes', async () => { + let clock = 0; + const { calls, http } = scriptedHttp([ + { body: { access_token: 'token-a', expires_in: 3600 }, status: 200 }, + { body: { access_token: 'token-b', expires_in: 3600 }, status: 200 }, + ]); + const manager = new TokenManager( + http, + () => ({ clientId: 'c', clientSecret: 's', refreshToken: 'r' }), + () => clock, + ); + const [first, second] = await Promise.all([manager.getToken(), manager.getToken()]); + expect(first).toBe('token-a'); + expect(second).toBe('token-a'); + expect(calls.length).toBe(1); + expect(await manager.getToken()).toBe('token-a'); + clock = 3600 * 1000; // Past expiry minus the safety margin. + expect(await manager.getToken()).toBe('token-b'); + expect(calls.length).toBe(2); + expect(calls[0]?.body).toContain('grant_type=refresh_token'); +}); + +test('TokenManager reports revoked authorization clearly', async () => { + const { http } = scriptedHttp([{ body: { error: 'invalid_grant' }, status: 400 }]); + const manager = new TokenManager(http, () => ({ + clientId: 'c', + clientSecret: 's', + refreshToken: 'r', + })); + await expect(manager.getToken()).rejects.toThrow('reconnect'); +}); + +test('bearerMiddleware injects the token and retries once after a 401', async () => { + const tokenHttp = scriptedHttp([ + { body: { access_token: 'stale', expires_in: 3600 }, status: 200 }, + { body: { access_token: 'fresh', expires_in: 3600 }, status: 200 }, + ]); + const manager = new TokenManager(tokenHttp.http, () => ({ + clientId: 'c', + clientSecret: 's', + refreshToken: 'r', + })); + const seenAuth: Array = []; + const inner: Request = (params: RequestParam | string) => { + if (typeof params === 'string') throw new Error('unexpected string request'); + seenAuth.push(params.headers?.Authorization); + const status = seenAuth.length === 1 ? 401 : 200; + const response: RequestResponse = { + bytes: () => new Uint8Array(0), + headers: {}, + json: () => ({}), + status, + text: () => '', + }; + return Promise.resolve(response); + }; + const request = bearerMiddleware(inner, manager); + const response = await request({ method: 'GET', url: 'https://example.com' }); + expect(response.status).toBe(200); + expect(seenAuth).toStrictEqual(['Bearer stale', 'Bearer fresh']); +}); + +test('decodeIdTokenEmail tolerates malformed tokens', () => { + expect(decodeIdTokenEmail(fakeIdToken('a@b.c'))).toBe('a@b.c'); + expect(decodeIdTokenEmail('garbage')).toBeUndefined(); + expect(decodeIdTokenEmail('a.b.c')).toBeUndefined(); +}); diff --git a/packages/gdrive/test/fs-gdrive.test.ts b/packages/gdrive/test/fs-gdrive.test.ts new file mode 100644 index 00000000..682264ae --- /dev/null +++ b/packages/gdrive/test/fs-gdrive.test.ts @@ -0,0 +1,240 @@ +import type { Binary, ListReporter, RootFs, Stat } from '@hesprs/sync-engine-sdk'; +import { testKit } from '@hesprs/sync-engine-sdk/dev'; +import { expect, test } from 'bun:test'; +import { md5 } from 'hash-wasm'; +import checkConnection from '@/gdrive/check-connection'; +import GdriveFs from '@/gdrive/fs'; +import { RESUMABLE_CHUNK_SIZE } from '@/gdrive/upload'; +import { MockDrive, jsonResponse } from './mock-drive'; + +const { bytes, file, stream: createStream } = testKit; + +const includeAll: ListReporter = () => 'advance'; + +function createFs(options: { baseDirectory?: string; useTrash?: boolean } = {}) { + const drive = new MockDrive(); + const fs: RootFs = new GdriveFs({ + account: 'mock@example.com', + baseDirectory: options.baseDirectory ?? 'Vault/Notes/', + request: drive.request, + useTrash: options.useTrash ?? true, + }); + return { drive, fs }; +} + +async function collect(source: ReadableStream): Promise { + const reader = source.getReader(); + const chunks: Array = []; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + reader.releaseLock(); + let total = 0; + for (const chunk of chunks) total += chunk.byteLength; + const merged = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(merged); +} + +test('write creates the base directory chain and read round-trips content', async () => { + const { drive, fs } = createFs(); + const mtime = 1_700_000_000_000; + const uid = await fs.write('a.md', bytes('hello'), file('a.md', { mtime, size: 5 })); + expect(uid).toBe(await md5('hello')); + expect(drive.contentByPath('Vault/Notes/a.md')).toBe('hello'); + expect(drive.fileByPath('Vault/Notes/a.md')?.modifiedTime).toBe(new Date(mtime).toISOString()); + expect(new TextDecoder().decode(await fs.read('a.md', file('a.md')))).toBe('hello'); +}); + +test('write to an existing key updates the same Drive file in place', async () => { + const { drive, fs } = createFs(); + await fs.write('note.md', bytes('one'), file('note.md', { mtime: 1000, size: 3 })); + const firstId = drive.fileByPath('Vault/Notes/note.md')?.id; + await fs.write('note.md', bytes('two!'), file('note.md', { mtime: 2000, size: 4 })); + const updated = drive.fileByPath('Vault/Notes/note.md'); + expect(updated?.id).toBe(firstId ?? ''); + expect(drive.contentByPath('Vault/Notes/note.md')).toBe('two!'); + expect(updated?.modifiedTime).toBe(new Date(2000).toISOString()); +}); + +test('keys with apostrophes survive the query escaping round trip', async () => { + const { drive, fs } = createFs(); + await fs.write("it's ok.md", bytes('quoted'), file("it's ok.md", { size: 6 })); + expect(drive.contentByPath("Vault/Notes/it's ok.md")).toBe('quoted'); + expect(new TextDecoder().decode(await fs.read("it's ok.md", file("it's ok.md")))).toBe( + 'quoted', + ); +}); + +test('mkdir builds nested folders and stat and exists see them', async () => { + const { drive, fs } = createFs(); + await fs.mkdir('x/y/', true); + expect(drive.fileByPath('Vault/Notes/x/y')?.mimeType).toBe( + 'application/vnd.google-apps.folder', + ); + expect(await fs.stat('x/y/')).toStrictEqual({ isDir: true, key: 'x/y/' }); + expect(await fs.exists('x/')).toBe(true); + expect(await fs.exists('missing/')).toBe(false); + expect(await fs.exists('missing.md')).toBe(false); + await expect(fs.stat('missing.md')).rejects.toMatchObject({ status: 404 }); + await fs.mkdir('x/y/', true); // Idempotent + expect(await fs.exists('x/y/')).toBe(true); +}); + +test('stat returns md5 uid, size, and preserved mtime for files', async () => { + const { fs } = createFs(); + const mtime = 1_600_000_000_000; + await fs.write('s.md', bytes('stats'), file('s.md', { mtime, size: 5 })); + const stat = await fs.stat('s.md'); + expect(stat).toStrictEqual({ + isDir: false, + key: 's.md', + mtime, + size: 5, + uid: await md5('stats'), + }); +}); + +test('list walks the tree, honors reporter verdicts, and paginates', async () => { + const { fs } = createFs(); + await fs.write('a.md', bytes('a'), file('a.md', { size: 1 })); + await fs.write('excluded.md', bytes('x'), file('excluded.md', { size: 1 })); + await fs.write('sub/b.md', bytes('b'), file('sub/b.md', { size: 1 })); + await fs.write('skip/c.md', bytes('c'), file('skip/c.md', { size: 1 })); + const seen: Array = []; + const reporter: ListReporter = ({ current }) => { + seen.push(current); + if (current === 'excluded.md') return 'exclude'; + if (current === 'skip/') return 'include'; + if (current.endsWith('/')) return 'advance'; + return 'include'; + }; + const results = await fs.list('/', reporter); + const keys = results.map((stat: Stat) => stat.key).sort(); + expect(keys).toStrictEqual(['a.md', 'skip/', 'sub/', 'sub/b.md']); + expect(seen).toContain('excluded.md'); + expect(seen).not.toContain('skip/c.md'); + const fileStat = results.find((stat: Stat) => stat.key === 'sub/b.md'); + expect(fileStat?.isDir).toBe(false); + if (fileStat?.isDir === false) expect(fileStat.uid).toBe(await md5('b')); +}); + +test('list from a subfolder returns keys prefixed with that folder', async () => { + const { fs } = createFs(); + await fs.write('sub/deep/d.md', bytes('d'), file('sub/deep/d.md', { size: 1 })); + const results = await fs.list('sub/', includeAll); + const keys = results.map((stat: Stat) => stat.key).sort(); + expect(keys).toStrictEqual(['sub/deep/', 'sub/deep/d.md']); +}); + +test('move renames files, relocates them between folders, and moves folders whole', async () => { + const { drive, fs } = createFs(); + await fs.write('a.md', bytes('a'), file('a.md', { size: 1 })); + await fs.move('a.md', 'renamed.md'); + expect(drive.contentByPath('Vault/Notes/renamed.md')).toBe('a'); + expect(drive.fileByPath('Vault/Notes/a.md')).toBeUndefined(); + expect(await fs.exists('a.md')).toBe(false); + expect(new TextDecoder().decode(await fs.read('renamed.md', file('renamed.md')))).toBe('a'); + + await fs.move('renamed.md', 'other/renamed.md'); + expect(drive.contentByPath('Vault/Notes/other/renamed.md')).toBe('a'); + + await fs.write('sub/b.md', bytes('b'), file('sub/b.md', { size: 1 })); + await fs.move('sub/', 'moved/'); + expect(drive.contentByPath('Vault/Notes/moved/b.md')).toBe('b'); + expect(new TextDecoder().decode(await fs.read('moved/b.md', file('moved/b.md')))).toBe('b'); +}); + +test('delete trashes by default, is idempotent, and can delete permanently', async () => { + const trashing = createFs(); + await trashing.fs.write('t.md', bytes('t'), file('t.md', { size: 1 })); + await trashing.fs.delete('t.md'); + const trashed = [...trashing.drive.files.values()].find((entry) => entry.name === 't.md'); + expect(trashed?.trashed).toBe(true); + expect(await trashing.fs.exists('t.md')).toBe(false); + await trashing.fs.delete('t.md'); // Missing → silently succeeds + await trashing.fs.delete('never-existed.md'); + + const permanent = createFs({ useTrash: false }); + await permanent.fs.write('p.md', bytes('p'), file('p.md', { size: 1 })); + await permanent.fs.delete('p.md'); + expect( + [...permanent.drive.files.values()].find((entry) => entry.name === 'p.md'), + ).toBeUndefined(); +}); + +test('readStream assembles ranged chunks in order', async () => { + const { fs } = createFs(); + await fs.write('r.md', bytes('ranged content'), file('r.md', { size: 14 })); + const result = await fs.readStream('r.md', file('r.md', { size: 14 })); + expect(await collect(result)).toBe('ranged content'); +}); + +test('writeStream below the threshold uses one multipart upload', async () => { + const { drive, fs } = createFs(); + const uid = await fs.writeStream( + 'small.md', + createStream(['hello ', 'stream']), + file('small.md', { mtime: 3000, size: 12 }), + ); + expect(drive.contentByPath('Vault/Notes/small.md')).toBe('hello stream'); + expect(uid).toBe(await md5('hello stream')); + const uploadCalls = drive.requestLog.filter( + (params) => typeof params !== 'string' && params.url.includes('uploadType=multipart'), + ); + expect(uploadCalls.length).toBe(1); +}); + +test('writeStream at the threshold uses a chunked resumable session', async () => { + const { drive, fs } = createFs(); + const big = new Uint8Array(RESUMABLE_CHUNK_SIZE + 3).fill(97); + const uid = await fs.writeStream( + 'big.bin', + createStream([big]), + file('big.bin', { mtime: 4000, size: big.byteLength }), + ); + const stored = drive.fileByPath('Vault/Notes/big.bin'); + expect(stored?.content?.byteLength).toBe(big.byteLength); + expect(uid).toBe(await md5(big)); + const sessionPuts = drive.requestLog.filter( + (params) => + typeof params !== 'string' && + params.method === 'PUT' && + params.url.includes('/mock-session/'), + ); + expect(sessionPuts.length).toBe(2); +}); + +test('duplicate names in one folder resolve to the newest file', async () => { + const { drive, fs } = createFs(); + await fs.mkdir('/', true); + const base = drive.fileByPath('Vault/Notes'); + drive.addFile('dup.md', base?.id ?? '', 'old', new Date(1000).toISOString()); + drive.addFile('dup.md', base?.id ?? '', 'new', new Date(2000).toISOString()); + expect(new TextDecoder().decode(await fs.read('dup.md', file('dup.md')))).toBe('new'); + const results = await fs.list('/', includeAll); + expect(results.filter((stat: Stat) => stat.key === 'dup.md').length).toBe(1); +}); + +test('getUid identifies the account and base directory', () => { + const { fs } = createFs(); + expect(fs.getUid()).toBe('gdrive~mock@example.com~Vault/Notes/'); +}); + +test('checkConnection reports success and surfaces Drive errors', async () => { + const { drive } = createFs(); + expect(await checkConnection(drive.request)).toStrictEqual({ success: true }); + drive.failNext = jsonResponse(401, { + error: { code: 401, message: 'Invalid Credentials' }, + }); + expect(await checkConnection(drive.request)).toStrictEqual({ + reason: 'Google Drive 401: Invalid Credentials', + success: false, + }); +}); diff --git a/packages/gdrive/test/mock-drive.ts b/packages/gdrive/test/mock-drive.ts new file mode 100644 index 00000000..50597315 --- /dev/null +++ b/packages/gdrive/test/mock-drive.ts @@ -0,0 +1,386 @@ +import type { Binary, Request, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; +import { md5 } from 'hash-wasm'; + +export type MockFile = { + id: string; + name: string; + mimeType: string; + parents: Array; + content?: Uint8Array; + modifiedTime: string; + trashed: boolean; +}; + +const FOLDER_MIME = 'application/vnd.google-apps.folder'; +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); +const latin1 = new TextDecoder('latin1'); + +export function jsonResponse(status: number, value: unknown): RequestResponse { + const text = JSON.stringify(value); + return { + bytes: () => encoder.encode(text), + headers: { 'content-type': 'application/json' }, + json: () => JSON.parse(text) as unknown, + status, + text: () => text, + }; +} + +function bytesResponse(status: number, content: Uint8Array): RequestResponse { + return { + bytes: () => new Uint8Array(content), + headers: { 'content-type': 'application/octet-stream' }, + json: () => JSON.parse(decoder.decode(content)) as unknown, + status, + text: () => decoder.decode(content), + }; +} + +function emptyResponse(status: number, headers: Record = {}): RequestResponse { + return { + bytes: () => new Uint8Array(0), + headers, + json: () => ({}), + status, + text: () => '', + }; +} + +function notFound(): RequestResponse { + return jsonResponse(404, { error: { code: 404, message: 'File not found.' } }); +} + +function unescapeQueryLiteral(value: string): string { + return value.replaceAll(/\\(?['\\])/gu, '$'); +} + +function toBinary(body: RequestParam['body']): Uint8Array { + if (body === undefined) return new Uint8Array(0); + if (typeof body === 'string') return encoder.encode(body); + return body; +} + +type MultipartParts = { metadata: Record; content: Uint8Array }; + +/** Latin1 decoding maps one byte to one char, so string indexes equal byte offsets. */ +function parseMultipart(body: Uint8Array, contentType: string): MultipartParts { + const boundary = contentType.split('boundary=')[1]; + if (!boundary) throw new Error('mock: missing multipart boundary'); + const text = latin1.decode(body); + const delimiter = `--${boundary}`; + const firstHeaderEnd = text.indexOf('\r\n\r\n'); + const secondDelimiter = text.indexOf(delimiter, firstHeaderEnd); + const metadataText = text.slice(firstHeaderEnd + 4, secondDelimiter); + const secondHeaderEnd = text.indexOf('\r\n\r\n', secondDelimiter); + const closingDelimiter = text.lastIndexOf(`\r\n${delimiter}--`); + const metadata = JSON.parse(metadataText.trimEnd()) as Record; + const content = body.slice(secondHeaderEnd + 4, closingDelimiter); + return { content, metadata }; +} + +type ResumableSession = { + targetId?: string; + metadata: Record; + received: Array; +}; + +async function serialize(file: MockFile): Promise> { + return { + id: file.id, + md5Checksum: + file.mimeType === FOLDER_MIME || file.content === undefined + ? undefined + : await md5(file.content), + mimeType: file.mimeType, + modifiedTime: file.modifiedTime, + name: file.name, + parents: file.parents, + size: file.content === undefined ? undefined : String(file.content.byteLength), + }; +} + +/** + * In-memory Google Drive REST v3 covering the subset the module uses: files + * lookup/list queries, media downloads with ranges, multipart and resumable + * uploads, metadata patches, deletes, `files/root`, and `about`. + */ +export class MockDrive { + readonly files = new Map(); + readonly rootId = 'root-id-0001'; + requestLog: Array = []; + failNext: RequestResponse | undefined; + private idCounter = 0; + private readonly sessions = new Map(); + + readonly request: Request = (params) => { + const normalized: RequestParam = typeof params === 'string' ? { url: params } : params; + this.requestLog.push(normalized); + if (this.failNext) { + const response = this.failNext; + this.failNext = undefined; + return Promise.resolve(response); + } + return this.route(normalized); + }; + + addFolder(name: string, parentId: string): MockFile { + const folder: MockFile = { + id: this.nextId(), + mimeType: FOLDER_MIME, + modifiedTime: new Date(0).toISOString(), + name, + parents: [parentId], + trashed: false, + }; + this.files.set(folder.id, folder); + return folder; + } + + addFile( + name: string, + parentId: string, + content: string, + modifiedTime = new Date(0).toISOString(), + ): MockFile { + const file: MockFile = { + content: encoder.encode(content), + id: this.nextId(), + mimeType: 'text/plain', + modifiedTime, + name, + parents: [parentId], + trashed: false, + }; + this.files.set(file.id, file); + return file; + } + + fileByPath(path: string): MockFile | undefined { + let parentId = this.rootId; + const segments = path.split('/').filter((segment) => segment !== ''); + let current: MockFile | undefined; + for (const segment of segments) { + current = [...this.files.values()].find( + (file) => !file.trashed && file.parents[0] === parentId && file.name === segment, + ); + if (!current) return undefined; + parentId = current.id; + } + return current; + } + + contentByPath(path: string): string | undefined { + const file = this.fileByPath(path); + return file?.content === undefined ? undefined : decoder.decode(file.content); + } + + private nextId(): string { + this.idCounter++; + return `id-${this.idCounter.toString().padStart(4, '0')}`; + } + + private async route(params: RequestParam): Promise { + const url = new URL(params.url); + const method = params.method ?? 'GET'; + const path = url.pathname; + if (path === '/token' || url.host === 'oauth2.googleapis.com') + throw new Error(`mock: unexpected OAuth call ${params.url}`); + if (path.startsWith('/mock-session/')) return this.routeSession(params, url); + if (path === '/drive/v3/about') + return jsonResponse(200, { user: { emailAddress: 'mock@example.com' } }); + if (path === '/drive/v3/files/root') return jsonResponse(200, { id: this.rootId }); + if (path === '/drive/v3/files' && method === 'GET') return this.routeQuery(url); + if (path === '/drive/v3/files' && method === 'POST') + return this.routeCreateMetadata(params); + if (path === '/upload/drive/v3/files' && method === 'POST') + return this.routeUpload(params, url); + const uploadMatch = /^\/upload\/drive\/v3\/files\/(?[^/]+)$/u.exec(path); + if (uploadMatch && method === 'PATCH') + return this.routeUpload(params, url, uploadMatch.groups?.id); + const fileMatch = /^\/drive\/v3\/files\/(?[^/]+)$/u.exec(path); + if (fileMatch) return this.routeFile(params, url, fileMatch.groups?.id ?? '', method); + throw new Error(`mock: unhandled route ${method} ${params.url}`); + } + + private async routeQuery(url: URL): Promise { + const q = url.searchParams.get('q') ?? ''; + const lookup = + /^'(?.+)' in parents and name = '(?.*)' and trashed = false and mimeType (?=|!=) '(?.+)'$/u.exec( + q, + ); + if (lookup?.groups) { + const parent = unescapeQueryLiteral(lookup.groups.parent ?? ''); + const name = unescapeQueryLiteral(lookup.groups.name ?? ''); + const wantFolder = lookup.groups.op === '='; + const matches = [...this.files.values()] + .filter( + (file) => + !file.trashed && + file.parents[0] === (parent === 'root' ? this.rootId : parent) && + file.name === name && + (file.mimeType === FOLDER_MIME) === wantFolder, + ) + .sort((a, b) => Date.parse(b.modifiedTime) - Date.parse(a.modifiedTime)); + return jsonResponse(200, { + files: await Promise.all(matches.map((file) => serialize(file))), + }); + } + if (q === 'trashed = false') { + const pageSize = 2; // Force pagination so tests cover it. + const all = [...this.files.values()].filter((file) => !file.trashed); + const start = Number.parseInt(url.searchParams.get('pageToken') ?? '0'); + const page = all.slice(start, start + pageSize); + const nextIndex = start + pageSize; + return jsonResponse(200, { + files: await Promise.all(page.map((file) => serialize(file))), + nextPageToken: nextIndex < all.length ? String(nextIndex) : undefined, + }); + } + throw new Error(`mock: unhandled query ${q}`); + } + + private async routeCreateMetadata(params: RequestParam): Promise { + const metadata = JSON.parse(decoder.decode(toBinary(params.body))) as { + mimeType?: string; + name?: string; + parents?: Array; + }; + if (metadata.mimeType !== FOLDER_MIME) + throw new Error('mock: metadata-only create supports folders only'); + const folder = this.addFolder(metadata.name ?? '', metadata.parents?.[0] ?? this.rootId); + return jsonResponse(200, await serialize(folder)); + } + + private async routeUpload( + params: RequestParam, + url: URL, + targetId?: string, + ): Promise { + const uploadType = url.searchParams.get('uploadType'); + if (uploadType === 'multipart') { + const contentType = params.headers?.['Content-Type'] ?? ''; + const { metadata, content } = parseMultipart(toBinary(params.body), contentType); + return jsonResponse( + 200, + await serialize(this.applyUpload(targetId, metadata, content)), + ); + } + if (uploadType === 'resumable') { + const metadata = JSON.parse(decoder.decode(toBinary(params.body))) as Record< + string, + unknown + >; + const sessionId = `session-${this.sessions.size + 1}`; + this.sessions.set(sessionId, { metadata, received: [], targetId }); + return emptyResponse(200, { + location: `https://mock.googleapis.com/mock-session/${sessionId}`, + }); + } + throw new Error(`mock: unhandled upload type ${uploadType ?? 'none'}`); + } + + private async routeSession(params: RequestParam, url: URL): Promise { + const sessionId = url.pathname.split('/').pop() ?? ''; + const session = this.sessions.get(sessionId); + if (!session) return notFound(); + if (params.method === 'DELETE') { + this.sessions.delete(sessionId); + return emptyResponse(204); + } + const range = /^bytes (?\d+)-(?\d+)\/(?\d+|\*)$/u.exec( + params.headers?.['Content-Range'] ?? '', + ); + if (!range?.groups) throw new Error('mock: resumable PUT without Content-Range'); + session.received.push(toBinary(params.body)); + const receivedBytes = session.received.reduce((sum, chunk) => sum + chunk.byteLength, 0); + const end = Number.parseInt(range.groups.end ?? '0'); + if (receivedBytes < end + 1) throw new Error('mock: resumable chunks out of order'); + const total = range.groups.total; + if (total !== '*' && receivedBytes >= Number.parseInt(total)) { + const content = new Uint8Array(receivedBytes); + let offset = 0; + for (const chunk of session.received) { + content.set(chunk, offset); + offset += chunk.byteLength; + } + this.sessions.delete(sessionId); + return jsonResponse( + 200, + await serialize(this.applyUpload(session.targetId, session.metadata, content)), + ); + } + return emptyResponse(308); + } + + private applyUpload( + targetId: string | undefined, + metadata: Record, + content: Uint8Array, + ): MockFile { + if (targetId !== undefined) { + const existing = this.files.get(targetId); + if (!existing) throw new Error(`mock: upload to missing file ${targetId}`); + existing.content = content; + if (typeof metadata.modifiedTime === 'string') + existing.modifiedTime = metadata.modifiedTime; + return existing; + } + const file: MockFile = { + content, + id: this.nextId(), + mimeType: typeof metadata.mimeType === 'string' ? metadata.mimeType : 'text/plain', + modifiedTime: + typeof metadata.modifiedTime === 'string' + ? metadata.modifiedTime + : new Date(0).toISOString(), + name: typeof metadata.name === 'string' ? metadata.name : '', + parents: Array.isArray(metadata.parents) ? (metadata.parents as Array) : [], + trashed: false, + }; + this.files.set(file.id, file); + return file; + } + + private async routeFile( + params: RequestParam, + url: URL, + id: string, + method: string, + ): Promise { + const file = this.files.get(id); + if (!file || (file.trashed && method !== 'DELETE')) return notFound(); + if (method === 'GET' && url.searchParams.get('alt') === 'media') { + const content = file.content ?? new Uint8Array(0); + const range = /^bytes=(?\d+)-(?\d+)$/u.exec(params.headers?.Range ?? ''); + if (range?.groups) { + const start = Number.parseInt(range.groups.start ?? '0'); + const end = Number.parseInt(range.groups.end ?? '0'); + return bytesResponse(206, content.slice(start, end + 1)); + } + return bytesResponse(200, content); + } + if (method === 'GET') return jsonResponse(200, await serialize(file)); + if (method === 'DELETE') { + this.files.delete(id); + return emptyResponse(204); + } + if (method === 'PATCH') { + const metadata = JSON.parse(decoder.decode(toBinary(params.body))) as { + name?: string; + trashed?: boolean; + }; + if (typeof metadata.name === 'string') file.name = metadata.name; + if (metadata.trashed === true) file.trashed = true; + const addParents = url.searchParams.get('addParents'); + const removeParents = url.searchParams.get('removeParents'); + if (addParents) + file.parents = [ + addParents, + ...file.parents.filter((parent) => parent !== removeParents), + ]; + return jsonResponse(200, await serialize(file)); + } + throw new Error(`mock: unhandled file route ${method} ${params.url}`); + } +} diff --git a/packages/gdrive/test/mocks.ts b/packages/gdrive/test/mocks.ts new file mode 100644 index 00000000..ed25d7f1 --- /dev/null +++ b/packages/gdrive/test/mocks.ts @@ -0,0 +1,5 @@ +// oxlint-disable-next-line import/no-namespace +import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import { mock } from 'bun:test'; + +void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/gdrive/tsconfig.json b/packages/gdrive/tsconfig.json new file mode 100644 index 00000000..ea80c697 --- /dev/null +++ b/packages/gdrive/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "paths": { + "@/*": ["./src/*"], + "@": ["./src"] + } + }, + "include": ["src/**/*.ts", "test/**/*.ts", "tsdown.config.ts"] +} diff --git a/packages/gdrive/tsdown.config.ts b/packages/gdrive/tsdown.config.ts new file mode 100644 index 00000000..f174baad --- /dev/null +++ b/packages/gdrive/tsdown.config.ts @@ -0,0 +1,14 @@ +import { syncEngineTransform } from '@hesprs/sync-engine-sdk/dev'; +import { defineConfig } from 'tsdown'; + +const dev = process.env.MODE === 'dev'; + +export default defineConfig({ + clean: !dev, + dts: false, + entry: { gdrive: 'src/index.ts' }, + minify: true, + outExtensions: () => ({ js: '.js' }), + outputOptions: { codeSplitting: false }, + plugins: [syncEngineTransform()], +}); From d313e605c2529125961ca8a01784b9af858491bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?He=CC=84sperus?= Date: Sat, 22 Aug 2026 15:19:33 +0800 Subject: [PATCH 2/7] refactor(gdrive): de-slop source --- .gitignore | 1 + .oxlintrc.json | 5 +- AGENTS.md | 1 + packages/gdrive/package.json | 5 + packages/gdrive/src/gdrive/api.ts | 10 +- packages/gdrive/src/gdrive/auth-http.ts | 10 - packages/gdrive/src/gdrive/auth.ts | 243 ++++++-------- .../gdrive/src/gdrive/check-connection.ts | 3 +- packages/gdrive/src/gdrive/fs.ts | 309 ++++++------------ packages/gdrive/src/gdrive/upload.ts | 136 ++++---- packages/gdrive/src/i18n.ts | 2 +- packages/gdrive/src/index.ts | 77 ++--- packages/gdrive/src/setting.ts | 80 +---- packages/gdrive/tsdown.config.ts | 4 + packages/plugin/src/fs/middlewares/retry.ts | 1 - packages/plugin/src/utils/sleep.ts | 5 - packages/plugin/test/retry-middleware.test.ts | 4 +- packages/plugin/tsconfig.json | 9 +- 18 files changed, 341 insertions(+), 564 deletions(-) delete mode 100644 packages/gdrive/src/gdrive/auth-http.ts delete mode 100644 packages/plugin/src/utils/sleep.ts diff --git a/.gitignore b/.gitignore index b36980bd..476e46b5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,6 @@ Sync Engine Logs **/.turbo **/node_modules **/cache +**/.env !packages/plugin/dist packages/plugin/dist/**/*.js diff --git a/.oxlintrc.json b/.oxlintrc.json index e99d3935..a948fd0f 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -122,7 +122,7 @@ "unicorn/import-style": "off", "unicorn/no-process-exit": "off", "eslint/one-var": ["warn", "never"], - "unicorn/max-nested-calls": ["warn", { "max": 5 }], + "unicorn/max-nested-calls": ["warn", { "max": 5 }] }, "env": { "builtin": true, @@ -136,7 +136,8 @@ "activeWindow": "readonly", "createFragment": "readonly", "createEl": "readonly", - "createDiv": "readonly" + "createDiv": "readonly", + "sleep": "readonly" }, "overrides": [ { diff --git a/AGENTS.md b/AGENTS.md index 82e00e27..2efc8d73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,7 @@ This is the monorepo for an extensible Obsidian syncing plugin to sync vault fil - Plugin & module SDK: `packages/plugin/`, package name `@hesprs/sync-engine-sdk`, `dev` builds SDK. - WebDAV module: `packages/webdav/`, package name `webdav`. - S3 module: `packages/s3/`, package name `s3`. +- Google Drive module: `packages/gdrive/`, package name `gdrive`. - Encryption module: `packages/encryption/`, package name `encryption`. - Shared utils: `packages/shared/`, package name `@repo/shared`. - Documentation site: `docs/`, package name `docs`. diff --git a/packages/gdrive/package.json b/packages/gdrive/package.json index 8c6c9e50..f917f2cb 100644 --- a/packages/gdrive/package.json +++ b/packages/gdrive/package.json @@ -7,6 +7,11 @@ { "name": "Aaron", "github": "Quzzar" + }, + { + "name": "Hēsperus", + "email": "hesprs@outlook.com", + "github": "hesprs" } ], "type": "module", diff --git a/packages/gdrive/src/gdrive/api.ts b/packages/gdrive/src/gdrive/api.ts index 05b39a44..67ef0757 100644 --- a/packages/gdrive/src/gdrive/api.ts +++ b/packages/gdrive/src/gdrive/api.ts @@ -46,16 +46,8 @@ export function getHeader( return entry?.[1]; } -export function safeJson(response: RequestResponse): unknown { - try { - return response.json(); - } catch { - return {}; - } -} - export function parseDriveError(response: RequestResponse): string | undefined { - const parsed = safeJson(response) as { + const parsed = response as { error?: { code?: number; message?: string } | string; error_description?: string; }; diff --git a/packages/gdrive/src/gdrive/auth-http.ts b/packages/gdrive/src/gdrive/auth-http.ts deleted file mode 100644 index 0b3c7a92..00000000 --- a/packages/gdrive/src/gdrive/auth-http.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { requestUrl } from 'obsidian'; -import type { AuthHttp } from './auth'; - -/** `AuthHttp` implementation backed by Obsidian's CORS-free `requestUrl`. */ -const requestUrlHttp: AuthHttp = async ({ url, method, body, contentType }) => { - const response = await requestUrl({ body, contentType, method, throw: false, url }); - return { json: () => response.json as unknown, status: response.status }; -}; - -export default requestUrlHttp; diff --git a/packages/gdrive/src/gdrive/auth.ts b/packages/gdrive/src/gdrive/auth.ts index aef38e01..aa2ab2ec 100644 --- a/packages/gdrive/src/gdrive/auth.ts +++ b/packages/gdrive/src/gdrive/auth.ts @@ -1,22 +1,47 @@ import type { Request, RequestParam } from '@hesprs/sync-engine-sdk'; +import { getStatus } from '@repo/shared/get-status'; +import { requestUrl, SecretStorage } from 'obsidian'; import { OAUTH_DEVICE_CODE_URL, OAUTH_SCOPE, OAUTH_TOKEN_URL } from './api'; -/** - * Minimal HTTP shape used for OAuth endpoints. Kept independent from the SDK - * `Request` so authentication can run from settings UI code (via Obsidian - * `requestUrl`) and from tests without a composed request chain. - */ -export type AuthHttp = (params: { - url: string; - method: 'GET' | 'POST'; - body?: string; - contentType?: string; -}) => Promise<{ status: number; json: () => unknown }>; +export const CLIENT_ID = process.env.CLIENT_ID ?? ''; +export const CLIENT_SECRET = process.env.CLIENT_SECRET ?? ''; // Not really a secret +const KEYCHAIN_SECRET_ID = 'sync-engine-gdrive-refresh-token'; // Secret storage id under which the Google refresh token is stored. -export type AuthConfig = { - clientId: string; - clientSecret: string; - refreshToken: string; +type TokenResponse = { + access_token: string; + expires_in: number; + user_id?: string; + refresh_token?: string; +}; + +type TokenError = { + error: + | 'invalid_request' + | 'invalid_client' + | 'invalid_grant' + | 'unauthorized_client' + | 'unsupported_grant_type' + | 'authorization_pending' + | 'slow_down' + | 'expired_token' + | 'access_denied'; + error_description?: string; + error_uri?: string; +}; + +type DeviceCodeResponse = { + device_code: string; + user_code: string; + verification_uri: string; + verification_uri_complete?: string; + expires_in: number; + interval: number; +}; + +type DeviceCodeError = { + error: 'invalid_request' | 'invalid_client' | 'unsupported_grant_type' | 'unauthorized_client'; + error_description?: string; + error_uri?: string; }; export type DeviceAuthorization = { @@ -31,134 +56,74 @@ export type DeviceTokenResult = { accessToken: string; refreshToken: string; expiresIn: number; - email?: string; + userId: string; }; const FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded'; -/** Secret storage id under which the Google refresh token is stored. */ -export const REFRESH_TOKEN_SECRET_ID = 'sync-engine-gdrive-refresh-token'; - function formEncode(fields: Record): string { return new URLSearchParams(fields).toString(); } -function describeAuthError(data: Record, status: number): string { - const description = - typeof data.error_description === 'string' ? data.error_description : undefined; - const code = typeof data.error === 'string' ? data.error : undefined; - return description ?? code ?? `HTTP ${status}`; +function describeAuthError(data: TokenError, status: number): string { + return data.error_description ?? data.error ?? `HTTP ${status}`; } -function safeAuthJson(response: { json: () => unknown }): Record { - try { - const parsed = response.json(); - return typeof parsed === 'object' && parsed !== undefined && parsed !== null - ? (parsed as Record) - : {}; - } catch { - return {}; - } -} - -export function decodeIdTokenEmail(idToken: string): string | undefined { - try { - const payload = idToken.split('.')[1]; - if (!payload) return undefined; - const normalized = payload.replaceAll('-', '+').replaceAll('_', '/'); - const padded = normalized + '='.repeat((4 - (normalized.length % 4)) % 4); - const parsed = JSON.parse(atob(padded)) as { email?: unknown }; - return typeof parsed.email === 'string' ? parsed.email : undefined; - } catch { - return undefined; - } -} - -export async function startDeviceAuthorization( - http: AuthHttp, - clientId: string, -): Promise { - const response = await http({ - body: formEncode({ client_id: clientId, scope: OAUTH_SCOPE }), +export async function startDeviceAuthorization(): Promise { + const response = await requestUrl({ + body: formEncode({ client_id: CLIENT_ID, scope: OAUTH_SCOPE }), contentType: FORM_CONTENT_TYPE, method: 'POST', + throw: false, url: OAUTH_DEVICE_CODE_URL, }); - const data = safeAuthJson(response); - if (response.status < 200 || response.status >= 300) + const data = response.json as DeviceCodeResponse | DeviceCodeError; + if ('error' in data) throw new Error( `Google device authorization failed: ${describeAuthError(data, response.status)}`, ); - const deviceCode = data.device_code; - const userCode = data.user_code; - const verificationUrl = data.verification_url ?? data.verification_uri; - if ( - typeof deviceCode !== 'string' || - typeof userCode !== 'string' || - typeof verificationUrl !== 'string' - ) - throw new Error('Google device authorization returned an unexpected response.'); return { - deviceCode, - expiresIn: typeof data.expires_in === 'number' ? data.expires_in : 1800, - interval: typeof data.interval === 'number' ? data.interval : 5, - userCode, - verificationUrl, + deviceCode: data.device_code, + expiresIn: data.expires_in, + interval: data.interval, + userCode: data.user_code, + verificationUrl: data.verification_uri, }; } -export async function pollDeviceToken( - http: AuthHttp, - options: { - clientId: string; - clientSecret: string; - authorization: DeviceAuthorization; - isCancelled?: () => boolean; - sleep?: (ms: number) => Promise; - now?: () => number; - }, -): Promise { - const sleep = - options.sleep ?? - ((ms: number) => - new Promise((resolve) => { - setTimeout(resolve, ms); - })); - const now = options.now ?? (() => Date.now()); +export async function pollDeviceToken(options: { + authorization: DeviceAuthorization; + isCancelled: () => boolean; +}): Promise { let interval = Math.max(options.authorization.interval, 1); - const deadline = now() + options.authorization.expiresIn * 1000; + const deadline = Date.now() + options.authorization.expiresIn * 1000; while (true) { await sleep(interval * 1000); if (options.isCancelled?.()) throw new Error('Google Drive connection was cancelled.'); - if (now() > deadline) + if (Date.now() > deadline) throw new Error('The device code expired, please try connecting again.'); - const response = await http({ + const response = await requestUrl({ body: formEncode({ - client_id: options.clientId, - client_secret: options.clientSecret, + client_id: CLIENT_ID, + client_secret: CLIENT_SECRET, device_code: options.authorization.deviceCode, grant_type: 'urn:ietf:params:oauth:grant-type:device_code', }), contentType: FORM_CONTENT_TYPE, method: 'POST', + throw: false, url: OAUTH_TOKEN_URL, }); - const data = safeAuthJson(response); - if ( - response.status >= 200 && - response.status < 300 && - typeof data.access_token === 'string' && - typeof data.refresh_token === 'string' - ) - return { - accessToken: data.access_token, - email: - typeof data.id_token === 'string' - ? decodeIdTokenEmail(data.id_token) - : undefined, - expiresIn: typeof data.expires_in === 'number' ? data.expires_in : 3600, - refreshToken: data.refresh_token, - }; + const data = response.json as TokenResponse | TokenError; + if ('access_token' in data) + if (data.refresh_token && data.user_id) + return { + accessToken: data.access_token, + expiresIn: data.expires_in, + refreshToken: data.refresh_token, + userId: data.user_id, + }; + else throw new Error('Google authorization payload is malformed!'); switch (data.error) { case 'authorization_pending': { continue; @@ -188,52 +153,55 @@ export async function pollDeviceToken( * connection check so a token refresh happens at most once at a time. */ export class TokenManager { - private accessToken: string | undefined; + private accessToken?: string; private expiresAt = 0; - private pending: Promise | undefined; + private pending?: Promise; - constructor( - private readonly http: AuthHttp, - private readonly resolveAuth: () => AuthConfig, - private readonly now: () => number = () => Date.now(), - ) {} + constructor(private readonly secretStorage: SecretStorage) {} readonly getToken = (force = false): Promise => { - if (!force && this.accessToken !== undefined && this.now() < this.expiresAt - 60_000) + if (!force && this.accessToken && Date.now() < this.expiresAt - 60_000) return Promise.resolve(this.accessToken); - this.pending ??= this.refresh().finally(() => { - this.pending = undefined; - }); + this.pending ??= this.refresh().finally(() => (this.pending = undefined)); return this.pending; }; + readonly getRefreshToken = () => this.secretStorage.getSecret(KEYCHAIN_SECRET_ID); + + readonly setRefreshToken = (token: string) => + this.secretStorage.setSecret(KEYCHAIN_SECRET_ID, token); + + readonly deleteRefreshToken = () => this.secretStorage.deleteSecret(KEYCHAIN_SECRET_ID); + + readonly setToken = (token: string, expiresIn: number) => { + this.accessToken = token; + this.expiresAt = Date.now() + expiresIn * 1000; + }; + readonly invalidate = (): void => { this.accessToken = undefined; this.expiresAt = 0; }; private async refresh(): Promise { - const { clientId, clientSecret, refreshToken } = this.resolveAuth(); - const response = await this.http({ + const refresh_token = this.getRefreshToken(); + if (!refresh_token) throw new Error('Please authorize Google Account!'); + const response = await requestUrl({ body: formEncode({ - client_id: clientId, - client_secret: clientSecret, + client_id: CLIENT_ID, + client_secret: CLIENT_SECRET, grant_type: 'refresh_token', - refresh_token: refreshToken, + refresh_token, }), contentType: FORM_CONTENT_TYPE, method: 'POST', + throw: false, url: OAUTH_TOKEN_URL, }); - const data = safeAuthJson(response); - if ( - response.status >= 200 && - response.status < 300 && - typeof data.access_token === 'string' - ) { + const data = response.json as TokenResponse | TokenError; + if ('access_token' in data) { this.accessToken = data.access_token; - this.expiresAt = - this.now() + (typeof data.expires_in === 'number' ? data.expires_in : 3600) * 1000; + this.expiresAt = Date.now() + data.expires_in * 1000; return data.access_token; } this.invalidate(); @@ -253,11 +221,12 @@ export function bearerMiddleware(request: Request, manager: TokenManager): Reque const base: RequestParam = typeof params === 'string' ? { url: params } : params; const send = (token: string) => request({ ...base, headers: { ...base.headers, Authorization: `Bearer ${token}` } }); - let response = await send(await manager.getToken()); - if (response.status === 401) { + try { + return await send(await manager.getToken()); + } catch (error: unknown) { + if (getStatus(error) !== 401) throw error; manager.invalidate(); - response = await send(await manager.getToken(true)); + return send(await manager.getToken(true)); } - return response; }; } diff --git a/packages/gdrive/src/gdrive/check-connection.ts b/packages/gdrive/src/gdrive/check-connection.ts index dede7685..a279edf1 100644 --- a/packages/gdrive/src/gdrive/check-connection.ts +++ b/packages/gdrive/src/gdrive/check-connection.ts @@ -5,7 +5,8 @@ export default async function checkConnection(request: Request): Promise= 200 && response.status < 300) return { success: true } as const; return { diff --git a/packages/gdrive/src/gdrive/fs.ts b/packages/gdrive/src/gdrive/fs.ts index c2838f02..d6bf9e07 100644 --- a/packages/gdrive/src/gdrive/fs.ts +++ b/packages/gdrive/src/gdrive/fs.ts @@ -10,7 +10,7 @@ import type { } from '@hesprs/sync-engine-sdk'; import { textToUint8Array } from '@repo/shared/binary'; import { getStatus } from '@repo/shared/get-status'; -import { basename, dirname, isFolder, normalizeBaseDir } from '@repo/shared/path'; +import { basename, dirname, isFolder } from '@repo/shared/path'; import type { DriveFile, DriveFileList } from './api'; import { DRIVE_API, @@ -20,17 +20,14 @@ import { buildUrl, escapeQuery, parseDriveError, - safeJson, toFileStat, } from './api'; import createRangeReadStream from './read-stream'; -import { RESUMABLE_CHUNK_SIZE, buildMultipartBody, guessMimeType, resumableUpload } from './upload'; +import { guessMimeType, resumableUpload, singlePutUpload } from './upload'; export type GdriveFsOptions = { - account: string; - baseDirectory: string; + userId: string; useTrash: boolean; - request: Request; }; const READ_CHUNK_SIZE = 2 * 1024 * 1024; // 2 MiB @@ -51,25 +48,20 @@ function notFoundError(key: string): Error { * through this module are visible because of the `drive.file` OAuth scope. */ export default class GdriveFs implements RootFs { - private readonly request: Request; - private readonly baseDirectory: string; - private readonly useTrash: boolean; /** Path key (`'/'`, `folder/`, `folder/note.md`) to Drive file id. */ private readonly ids = new Map(); - constructor(private readonly options: GdriveFsOptions) { - if (!options.request) throw new Error('Google Drive request is required.'); - this.request = options.request; - this.baseDirectory = normalizeBaseDir(options.baseDirectory); - this.useTrash = options.useTrash; - } + constructor( + private readonly request: Request, + private readonly options: GdriveFsOptions, + ) {} getUid(): string { - return `gdrive~${this.options.account}~${this.baseDirectory}`; + return `gdrive~${this.options.userId}`; } private async requestOrThrow(params: RequestParam): Promise { - const response = await this.request(params); + const response = await this.request(Object.assign(params, { throw: false })); if (response.status >= 200 && response.status < 300) return response; const error = new Error( parseDriveError(response) ?? @@ -90,11 +82,11 @@ export default class GdriveFs implements RootFs { const url = buildUrl(DRIVE_API, '/files', { fields: `files(${FILE_FIELDS})`, orderBy: 'modifiedTime desc', - pageSize: '2', + pageSize: '1', q: `'${escapeQuery(parentId)}' in parents and name = '${escapeQuery(name)}' and trashed = false${mimeClause}`, }); const response = await this.requestOrThrow({ method: 'GET', url }); - return (safeJson(response) as DriveFileList).files?.[0]; + return (response.json() as DriveFileList).files?.[0]; } private async createFolder(parentId: string, name: string): Promise { @@ -106,48 +98,51 @@ export default class GdriveFs implements RootFs { method: 'POST', url: buildUrl(DRIVE_API, '/files', { fields: 'id' }), }); - const created = safeJson(response) as DriveFile; + const created = response.json() as DriveFile; if (!created.id) throw new Error('Google Drive did not return an id for a created folder!'); return created.id; } /** - * Resolves the base directory to its folder id, creating missing folders - * when `create` is set. Always resolves the real root id (never the `root` - * alias) so listing can match ids returned in `parents`. + * Resolves the real root id (never the `root` alias) so listing can match + * ids returned in `parents`. */ - private async ensureBase(create: boolean): Promise { + private async rootId(): Promise { const cached = this.ids.get('/'); if (cached !== undefined) return cached; - const rootResponse = await this.requestOrThrow({ + const response = await this.requestOrThrow({ method: 'GET', url: buildUrl(DRIVE_API, '/files/root', { fields: 'id' }), }); - let currentId = (safeJson(rootResponse) as DriveFile).id; - if (!currentId) throw new Error('Google Drive did not return the root folder id!'); - for (const segment of this.baseDirectory.split('/').filter((part) => part !== '')) { - const existing = await this.lookupChild(currentId, segment, true); - if (existing) currentId = existing.id; - else if (create) currentId = await this.createFolder(currentId, segment); - else return undefined; - } - this.ids.set('/', currentId); - return currentId; + const id = (response.json() as DriveFile).id; + if (!id) throw new Error('Google Drive did not return the root folder id!'); + this.ids.set('/', id); + return id; } - /** Resolves a key to its id; `create` builds missing folders along the way. */ - private async resolveId(key: string, create: boolean): Promise { - if (key === '/') return this.ensureBase(create); + /** Resolves a key to its id, or `undefined` when it does not exist. */ + private async resolveId(key: string): Promise { + if (key === '/') return this.rootId(); const cached = this.ids.get(key); if (cached !== undefined) return cached; - const parentId = await this.resolveId(dirname(key), create); + const parentId = await this.resolveId(dirname(key)); if (parentId === undefined) return undefined; const existing = await this.lookupChild(parentId, basename(key), isFolder(key)); + if (existing) this.ids.set(key, existing.id); + return existing?.id; + } + + /** Resolves a folder key to its id, creating the folder chain when missing. */ + private async ensureFolderId(key: string): Promise { + if (key === '/') return this.rootId(); + const cached = this.ids.get(key); + if (cached !== undefined) return cached; + const parentId = await this.ensureFolderId(dirname(key)); + const existing = await this.lookupChild(parentId, basename(key), true); if (existing) { this.ids.set(key, existing.id); return existing.id; } - if (!create || !isFolder(key)) return undefined; const created = await this.createFolder(parentId, basename(key)); this.ids.set(key, created); return created; @@ -155,51 +150,55 @@ export default class GdriveFs implements RootFs { /** Fresh metadata lookup for a file key (also refreshes the id cache). */ private async resolveEntry(key: string): Promise { - const parentId = await this.resolveId(dirname(key), false); + const parentId = await this.resolveId(dirname(key)); if (parentId === undefined) return undefined; - const entry = await this.lookupChild(parentId, basename(key), isFolder(key)); + const entry = await this.lookupChild(parentId, basename(key), false); if (entry) this.ids.set(key, entry.id); return entry; } - private async requireId(key: string): Promise { - const cached = this.ids.get(key); - if (cached !== undefined) return cached; - const id = isFolder(key) - ? await this.resolveId(key, false) - : (await this.resolveEntry(key))?.id; - if (id === undefined) throw notFoundError(key); - return id; - } - - private cachedKeysUnder(folderKey: string): Array { - const keys: Array = []; - for (const cachedKey of this.ids.keys()) - if (cachedKey.startsWith(folderKey) && cachedKey !== folderKey) keys.push(cachedKey); - return keys; - } - private dropCache(key: string): void { this.ids.delete(key); if (!isFolder(key)) return; - for (const cachedKey of this.cachedKeysUnder(key)) this.ids.delete(cachedKey); + for (const cachedKey of this.ids.keys()) + if (cachedKey.startsWith(key)) this.ids.delete(cachedKey); } - private remapCache(oldKey: string, newKey: string): void { - const id = this.ids.get(oldKey); - this.ids.delete(oldKey); - if (id !== undefined) this.ids.set(newKey, id); - if (!isFolder(oldKey)) return; - for (const cachedKey of this.cachedKeysUnder(oldKey)) { - const childId = this.ids.get(cachedKey); - this.ids.delete(cachedKey); - if (childId !== undefined) - this.ids.set(newKey + cachedKey.slice(oldKey.length), childId); - } + /** Session metadata for uploading `key`, updating the existing file when present. */ + private async sessionFor( + key: string, + stat: FileStat, + ): Promise<{ initiateUrl: string; method: 'PATCH' | 'POST'; metadata: object }> { + const modifiedTime = new Date(stat.mtime).toISOString(); + const existing = await this.resolveEntry(key); + if (existing) + return { + initiateUrl: buildUrl(DRIVE_UPLOAD_API, `/files/${existing.id}`, { + fields: WRITE_FIELDS, + uploadType: 'resumable', + }), + metadata: { modifiedTime }, + method: 'PATCH', + }; + const parentId = await this.ensureFolderId(dirname(key)); + return { + initiateUrl: buildUrl(DRIVE_UPLOAD_API, '/files', { + fields: WRITE_FIELDS, + uploadType: 'resumable', + }), + metadata: { + mimeType: guessMimeType(basename(key)), + modifiedTime, + name: basename(key), + parents: [parentId], + }, + method: 'POST', + }; } async read(key: string): Promise { - const id = await this.requireId(key); + const id = await this.resolveId(key); + if (id === undefined) throw notFoundError(key); const response = await this.requestOrThrow({ method: 'GET', url: buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }), @@ -208,7 +207,8 @@ export default class GdriveFs implements RootFs { } async readStream(key: string, { size }: FileStat): Promise> { - const id = await this.requireId(key); + const id = await this.resolveId(key); + if (id === undefined) throw notFoundError(key); const url = buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }); return createRangeReadStream({ chunkSize: READ_CHUNK_SIZE, @@ -226,101 +226,33 @@ export default class GdriveFs implements RootFs { } async write(key: string, value: Binary, stat: FileStat): Promise { - const existing = await this.resolveEntry(key); - const modifiedTime = new Date(stat.mtime).toISOString(); - const mimeType = guessMimeType(basename(key)); - let response: RequestResponse; - if (existing) { - const { body, contentType } = buildMultipartBody({ modifiedTime }, value, mimeType); - response = await this.requestOrThrow({ - body, - headers: { 'Content-Type': contentType }, - method: 'PATCH', - url: buildUrl(DRIVE_UPLOAD_API, `/files/${existing.id}`, { - fields: WRITE_FIELDS, - uploadType: 'multipart', - }), - }); - } else { - const parentId = await this.resolveId(dirname(key), true); - if (parentId === undefined) throw notFoundError(dirname(key)); - const { body, contentType } = buildMultipartBody( - { mimeType, modifiedTime, name: basename(key), parents: [parentId] }, - value, - mimeType, - ); - response = await this.requestOrThrow({ - body, - headers: { 'Content-Type': contentType }, - method: 'POST', - url: buildUrl(DRIVE_UPLOAD_API, '/files', { - fields: WRITE_FIELDS, - uploadType: 'multipart', - }), - }); - } - const file = safeJson(response) as DriveFile; + const file = await singlePutUpload( + { + ...(await this.sessionFor(key, stat)), + request: this.request, + size: value.byteLength, + }, + value, + ); if (file.id) this.ids.set(key, file.id); return file.md5Checksum ?? `${stat.mtime}~${stat.size}`; } async writeStream(key: string, value: ReadableStream, stat: FileStat): Promise { - if (stat.size < RESUMABLE_CHUNK_SIZE) - return this.write(key, await collectStreamToBinary(value), stat); - const existing = await this.resolveEntry(key); - const modifiedTime = new Date(stat.mtime).toISOString(); - let file: DriveFile; - if (existing) - file = await resumableUpload( - { - initiateUrl: buildUrl(DRIVE_UPLOAD_API, `/files/${existing.id}`, { - fields: WRITE_FIELDS, - uploadType: 'resumable', - }), - metadata: { modifiedTime }, - method: 'PATCH', - request: this.request, - stat, - }, - value, - ); - else { - const parentId = await this.resolveId(dirname(key), true); - if (parentId === undefined) throw notFoundError(dirname(key)); - file = await resumableUpload( - { - initiateUrl: buildUrl(DRIVE_UPLOAD_API, '/files', { - fields: WRITE_FIELDS, - uploadType: 'resumable', - }), - metadata: { - mimeType: guessMimeType(basename(key)), - modifiedTime, - name: basename(key), - parents: [parentId], - }, - method: 'POST', - request: this.request, - stat, - }, - value, - ); - } + const file = await resumableUpload( + { ...(await this.sessionFor(key, stat)), request: this.request, size: stat.size }, + value, + ); if (file.id) this.ids.set(key, file.id); return file.md5Checksum ?? `${stat.mtime}~${stat.size}`; } async delete(key: string): Promise { - let id: string; - try { - id = await this.requireId(key); - } catch (error) { - if (getStatus(error) === 404) return; - throw error; - } + const id = await this.resolveId(key); + if (id === undefined) return; try { await this.requestOrThrow( - this.useTrash + this.options.useTrash ? { body: textToUint8Array(JSON.stringify({ trashed: true })), headers: { 'Content-Type': 'application/json; charset=UTF-8' }, @@ -339,14 +271,14 @@ export default class GdriveFs implements RootFs { } async move(oldKey: string, newKey: string): Promise { - const id = await this.requireId(oldKey); - const oldParentId = await this.resolveId(dirname(oldKey), false); - const newParentId = await this.resolveId(dirname(newKey), true); - if (newParentId === undefined) throw notFoundError(dirname(newKey)); + const id = await this.resolveId(oldKey); + if (id === undefined) throw notFoundError(oldKey); + const oldParentId = await this.resolveId(dirname(oldKey)); + const newParentId = await this.ensureFolderId(dirname(newKey)); const query: Record = { fields: 'id' }; - if (oldParentId !== undefined && oldParentId !== newParentId) { + if (oldParentId !== newParentId) { query.addParents = newParentId; - query.removeParents = oldParentId; + if (oldParentId !== undefined) query.removeParents = oldParentId; } await this.requestOrThrow({ body: textToUint8Array(JSON.stringify({ name: basename(newKey) })), @@ -354,23 +286,17 @@ export default class GdriveFs implements RootFs { method: 'PATCH', url: buildUrl(DRIVE_API, `/files/${id}`, query), }); - this.remapCache(oldKey, newKey); + this.dropCache(oldKey); } - /** - * Drive folders always need an existing parent id, so missing parents are - * created regardless of `recursive`. - */ + /** Drive folders always need an existing parent id, so missing parents are created regardless of `recursive`. */ async mkdir(key: string): Promise { - const id = await this.resolveId(key, true); - if (id === undefined) throw notFoundError(key); + await this.ensureFolderId(key); } async stat(key: string): Promise { - if (key === '/') return { isDir: true, key }; if (isFolder(key)) { - const id = await this.resolveId(key, false); - if (id === undefined) throw notFoundError(key); + if ((await this.resolveId(key)) === undefined) throw notFoundError(key); return { isDir: true, key }; } const entry = await this.resolveEntry(key); @@ -379,14 +305,7 @@ export default class GdriveFs implements RootFs { } async exists(key: string): Promise { - if (key === '/') return true; - try { - await this.stat(key); - return true; - } catch (error) { - if (getStatus(error) === 404) return false; - throw error; - } + return key === '/' || (await this.resolveId(key)) !== undefined; } /** @@ -395,8 +314,7 @@ export default class GdriveFs implements RootFs { * under the requested key so the reporter can steer traversal. */ async list(key: string, reporter: ListReporter): Promise> { - const startId = - key === '/' ? await this.ensureBase(true) : await this.resolveId(key, false); + const startId = await this.resolveId(key); if (startId === undefined) throw notFoundError(key); const all: Array = []; let pageToken: string | undefined; @@ -411,7 +329,7 @@ export default class GdriveFs implements RootFs { method: 'GET', url: buildUrl(DRIVE_API, '/files', query), }); - const parsed = safeJson(response) as DriveFileList; + const parsed = response.json() as DriveFileList; all.push(...(parsed.files ?? [])); pageToken = parsed.nextPageToken; } while (pageToken); @@ -432,7 +350,7 @@ export default class GdriveFs implements RootFs { for (const entry of dedupeChildren(childrenByParent.get(folderId) ?? [])) { const folder = entry.mimeType === FOLDER_MIME; const childKey = `${prefix}${entry.name}${folder ? '/' : ''}`; - completed = Math.min(completed + 1, total); + completed++; const verdict = await reporter({ completed, current: childKey, total }); if (verdict === 'exclude') continue; this.ids.set(childKey, entry.id); @@ -469,26 +387,3 @@ function dedupeChildren(entries: Array): Array { } return [...byName.values()]; } - -async function collectStreamToBinary(source: ReadableStream): Promise { - const reader = source.getReader(); - const chunks: Array = []; - let total = 0; - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - total += value.byteLength; - } - } finally { - reader.releaseLock(); - } - const result = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - result.set(chunk, offset); - offset += chunk.byteLength; - } - return result; -} diff --git a/packages/gdrive/src/gdrive/upload.ts b/packages/gdrive/src/gdrive/upload.ts index 676e8e1f..f48a6099 100644 --- a/packages/gdrive/src/gdrive/upload.ts +++ b/packages/gdrive/src/gdrive/upload.ts @@ -1,10 +1,10 @@ -import type { Binary, FileStat, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; +import type { Binary, Request, RequestResponse } from '@hesprs/sync-engine-sdk'; import { concatBinary, textToUint8Array } from '@repo/shared/binary'; import type { DriveFile } from './api'; -import { getHeader, parseDriveError, safeJson } from './api'; +import { getHeader, parseDriveError } from './api'; /** Google Drive resumable uploads require chunk sizes in multiples of 256 KiB. */ -export const RESUMABLE_CHUNK_SIZE = 8 * 1024 * 1024; +export const RESUMABLE_CHUNK_SIZE = 5 * 1024 * 1024; const MIME_BY_EXTENSION: Record = { base: 'application/json', @@ -39,81 +39,77 @@ export function guessMimeType(name: string): string { return MIME_BY_EXTENSION[extension] ?? 'application/octet-stream'; } -let boundaryCounter = 0; - -export function buildMultipartBody( - metadata: object, - content: Binary, - contentMimeType: string, -): { body: Binary; contentType: string } { - boundaryCounter++; - const boundary = `sync-engine-gdrive-${boundaryCounter.toString(36)}-${Math.random().toString(36).slice(2)}`; - const head = textToUint8Array( - `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify(metadata)}\r\n--${boundary}\r\nContent-Type: ${contentMimeType}\r\n\r\n`, - ); - const tail = textToUint8Array(`\r\n--${boundary}--`); - return { - body: concatBinary(head, content, tail), - contentType: `multipart/related; boundary=${boundary}`, - }; -} - -export type ResumableUploadOptions = { +export type SessionOptions = { initiateUrl: string; method: 'PATCH' | 'POST'; metadata: object; - stat: FileStat; - /** Raw composed request — resumable chunk responses use non-2xx status 308. */ - request: (params: RequestParam) => Promise; + request: Request; + size: number; }; -export async function resumableUpload( - options: ResumableUploadOptions, - value: ReadableStream, -): Promise { - const initiate = await options.request({ - body: textToUint8Array(JSON.stringify(options.metadata)), +async function startSession({ + initiateUrl, + method, + metadata, + request, + size, +}: SessionOptions): Promise<{ request: Request; location: string }> { + const response = await request({ + body: textToUint8Array(JSON.stringify(metadata)), headers: { 'Content-Type': 'application/json; charset=UTF-8', - 'X-Upload-Content-Length': String(options.stat.size), + 'X-Upload-Content-Length': String(size), }, - method: options.method, - url: options.initiateUrl, + method, + url: initiateUrl, }); - if (initiate.status < 200 || initiate.status >= 300) + if (response.status < 200 || response.status >= 300) throw new Error( - parseDriveError(initiate) ?? - `Google Drive resumable upload initiation failed: ${initiate.status}`, + parseDriveError(response) ?? + `Google Drive upload session initiation failed: ${response.status}`, ); - const location = getHeader(initiate.headers, 'location'); - if (!location) throw new Error('Google Drive did not return a resumable upload session URL!'); + const location = getHeader(response.headers, 'location'); + if (!location) throw new Error('Google Drive did not return an upload session URL!'); + return { location, request }; +} - const total = options.stat.size; +/** Returns `undefined` when Drive answers 308 (chunk stored, upload incomplete). */ +async function putChunk( + { request, location }: { request: Request; location: string }, + chunk: Binary, + start: number, + total: number, +): Promise { + const end = start + chunk.byteLength - 1; + const response = await request({ + body: chunk, + headers: { + 'Content-Range': end < start ? `bytes */${total}` : `bytes ${start}-${end}/${total}`, + }, + method: 'PUT', + url: location, + }); + if (response.status === 308) return undefined; + if (response.status >= 200 && response.status < 300) return response; + throw new Error(parseDriveError(response) ?? `Google Drive upload failed: ${response.status}`); +} + +/** Uploads the whole value in a single PUT on a resumable session. */ +export async function singlePutUpload(options: SessionOptions, value: Binary): Promise { + const session = await startSession(options); + const response = await putChunk(session, value, 0, value.byteLength); + if (response === undefined) throw new Error('Google Drive upload ended prematurely.'); + return response.json() as DriveFile; +} + +export async function resumableUpload( + options: SessionOptions, + value: ReadableStream, +): Promise { + const session = await startSession(options); + const total = options.size; let offset = 0; let final: RequestResponse | undefined; - const putChunk = async ( - chunk: Binary, - isLast: boolean, - ): Promise => { - const start = offset; - const end = offset + chunk.byteLength - 1; - offset += chunk.byteLength; - const response = await options.request({ - body: chunk, - headers: { 'Content-Range': `bytes ${start}-${end}/${total}` }, - method: 'PUT', - url: location, - }); - if (response.status === 308) { - if (isLast) throw new Error('Google Drive resumable upload ended prematurely.'); - return undefined; - } - if (response.status >= 200 && response.status < 300) return response; - throw new Error( - parseDriveError(response) ?? `Google Drive resumable upload failed: ${response.status}`, - ); - }; - const reader = value.getReader(); let pending = new Uint8Array(0); try { @@ -125,18 +121,18 @@ export async function resumableUpload( while (pending.byteLength > RESUMABLE_CHUNK_SIZE && final === undefined) { const part = pending.slice(0, RESUMABLE_CHUNK_SIZE); pending = pending.slice(RESUMABLE_CHUNK_SIZE); - final = await putChunk(part, false); + final = await putChunk(session, part, offset, total); + offset += part.byteLength; } } - final ??= await putChunk(pending, true); + final ??= await putChunk(session, pending, offset, total); } catch (error) { // Best-effort session cancellation; Drive also expires sessions on its own. - await options.request({ method: 'DELETE', url: location }).catch(() => {}); + await options.request({ method: 'DELETE', url: session.location }).catch(() => {}); throw error; } finally { reader.releaseLock(); } - if (final === undefined) - throw new Error('Google Drive resumable upload finished without a response.'); - return safeJson(final) as DriveFile; + if (final === undefined) throw new Error('Google Drive upload finished incomplete.'); + return final.json() as DriveFile; } diff --git a/packages/gdrive/src/i18n.ts b/packages/gdrive/src/i18n.ts index 2f4482e9..db298d1a 100644 --- a/packages/gdrive/src/i18n.ts +++ b/packages/gdrive/src/i18n.ts @@ -18,7 +18,7 @@ const en: GdriveTranslations = { codeCopied: 'Copied', configureFirst: 'Enter the OAuth client ID and client secret first.', connect: 'Connect', - connectSuccess: 'Connected to Google Drive as {{account}}.', + connectSuccess: 'Connected to Google Drive.', copyCode: 'Copy code', deviceCodeInstruction: 'On any device, visit {{url}} and enter the code below, then approve access.', diff --git a/packages/gdrive/src/index.ts b/packages/gdrive/src/index.ts index b4eb6807..89250d56 100644 --- a/packages/gdrive/src/index.ts +++ b/packages/gdrive/src/index.ts @@ -1,5 +1,6 @@ import type { Context, + FsWrapperEntry, ObsidianLanguageCode, RemoteFsEntry, RemoteRequestMiddlewareEntry, @@ -11,21 +12,18 @@ import type { TranslationResource, } from '@hesprs/sync-engine-sdk'; import type { App } from 'obsidian'; +import { digOriginal, prefixWrapper } from '@hesprs/sync-engine-sdk'; import type { GdriveTranslations } from './setting'; import { TokenManager, bearerMiddleware } from './gdrive/auth'; -import requestUrlHttp from './gdrive/auth-http'; import checkConnection from './gdrive/check-connection'; import GdriveFs from './gdrive/fs'; import en from './i18n'; import gdriveSetting from './setting'; export type GdriveSettings = { - account: string; baseDirectory: string; - clientId: string; - clientSecret: string; - refreshToken: string; useTrash: boolean; + userId: string; }; export default class Gdrive { @@ -37,56 +35,47 @@ export default class Gdrive { translate: Translate; registerRemoteFs: (id: string, entry: RemoteFsEntry) => () => void; app: App; + registerRemoteFsWrapper: (entry: FsWrapperEntry) => () => void; registerRemoteRequestMiddleware: (entry: RemoteRequestMiddlewareEntry) => () => void; registerSetting: (entry: SettingEntry) => () => void; registerI18n: (lang: ObsidianLanguageCode, translations: TranslationResource) => void; - rerenderSettingTab: () => void; }>, ) { if (!this.moduleSettings.baseDirectory) this.moduleSettings.baseDirectory = `${ctx.app.vault.getName()}/`; ctx.registerI18n('en', en); - this.tokenManager = new TokenManager(requestUrlHttp, () => this.resolveAuth()); + this.tokenManager = new TokenManager(ctx.app.secretStorage); } readonly moduleSettings: GdriveSettings = { - account: '', baseDirectory: '', - clientId: '', - clientSecret: '', - refreshToken: '', useTrash: true, + userId: '', }; declare settings: Settings; readonly start = () => { - const { translate, registerRemoteFs, registerRemoteRequestMiddleware, registerSetting } = - this.ctx; + const { + translate, + registerRemoteFs, + registerRemoteFsWrapper, + registerRemoteRequestMiddleware, + registerSetting, + } = this.ctx; this.cleanup.push( registerRemoteFs('gdrive', { - checkConnection: (request) => { - try { - this.resolveConfig(); - } catch (error) { - return { - reason: error instanceof Error ? error.message : String(error), - success: false, - }; - } - return checkConnection(request); - }, - instantiate: (request) => { - const config = this.resolveConfig(); - return new GdriveFs({ - account: config.account, - baseDirectory: config.baseDirectory, - request, - useTrash: config.useTrash, - }); - }, + checkConnection, + instantiate: (request) => new GdriveFs(request, this.moduleSettings), prettyName: () => translate('gdrive'), }), + registerRemoteFsWrapper({ + apply: (fs) => { + if (digOriginal(fs) instanceof GdriveFs) + return prefixWrapper(fs, this.moduleSettings.baseDirectory); + }, + priority: 8308, + }), registerRemoteRequestMiddleware({ apply: (request) => { if (this.settings.remoteFs !== 'gdrive') return; @@ -101,28 +90,6 @@ export default class Gdrive { ); }; - private readonly resolveAuth = () => { - const { - clientId, - clientSecret: clientSecretId, - refreshToken: refreshTokenId, - } = this.moduleSettings; - const { secretStorage } = this.ctx.app; - const clientSecret = secretStorage.getSecret(clientSecretId); - if (!clientId || clientSecret === null || clientSecret === '') - throw new Error('Please configure the Google Drive OAuth client!'); - const refreshToken = secretStorage.getSecret(refreshTokenId); - if (refreshToken === null || refreshToken === '') - throw new Error('Please connect your Google account in the Sync Engine settings!'); - return { clientId, clientSecret, refreshToken }; - }; - - private readonly resolveConfig = () => { - this.resolveAuth(); - const { account, baseDirectory, useTrash } = this.moduleSettings; - return { account: account || 'unknown', baseDirectory, useTrash }; - }; - readonly dispose = () => { this.cleanup.forEach((fn) => fn()); this.cleanup.length = 0; diff --git a/packages/gdrive/src/setting.ts b/packages/gdrive/src/setting.ts index 0b1e1439..c87c50f0 100644 --- a/packages/gdrive/src/setting.ts +++ b/packages/gdrive/src/setting.ts @@ -8,10 +8,9 @@ import type { import type { App, SettingGroupItem } from 'obsidian'; import { s } from '@hesprs/sync-engine-sdk'; import { normalizeBaseDir } from '@repo/shared/path'; -import { Modal, Notice, SecretComponent } from 'obsidian'; +import { Modal, Notice } from 'obsidian'; import type { TokenManager } from './gdrive/auth'; -import { REFRESH_TOKEN_SECRET_ID, pollDeviceToken, startDeviceAuthorization } from './gdrive/auth'; -import requestUrlHttp from './gdrive/auth-http'; +import { pollDeviceToken, startDeviceAuthorization } from './gdrive/auth'; import handleInput from './handle-input'; export type GdriveTranslations = { @@ -123,17 +122,10 @@ export default function gdriveSetting( tokenManager: TokenManager, ): CallableOrObjectTree { const invalidValue = translate('invalidValue'); - const connectGoogle = async () => { - const clientId = settings.clientId.trim(); - const clientSecret = app.secretStorage.getSecret(settings.clientSecret); - if (!clientId || clientSecret === null || clientSecret === '') { - new Notice(translate('configureFirst')); - return; - } let cancelled = false; try { - const authorization = await startDeviceAuthorization(requestUrlHttp, clientId); + const authorization = await startDeviceAuthorization(); let finished = false; const modal = new DeviceCodeModal(app, { copiedLabel: translate('codeCopied'), @@ -152,20 +144,17 @@ export default function gdriveSetting( }); modal.open(); try { - const token = await pollDeviceToken(requestUrlHttp, { + const { refreshToken, userId, accessToken, expiresIn } = await pollDeviceToken({ authorization, - clientId, - clientSecret, isCancelled: () => cancelled, }); finished = true; - app.secretStorage.setSecret(REFRESH_TOKEN_SECRET_ID, token.refreshToken); - settings.refreshToken = REFRESH_TOKEN_SECRET_ID; - if (token.email) settings.account = token.email; - else if (!settings.account) settings.account = 'Google account'; + tokenManager.setRefreshToken(refreshToken); + settings.userId = userId; + tokenManager.setToken(accessToken, expiresIn); await saveSettings(); tokenManager.invalidate(); - new Notice(translate('connectSuccess', { account: settings.account })); + new Notice(translate('connectSuccess')); } finally { finished = true; modal.close(); @@ -175,9 +164,10 @@ export default function gdriveSetting( if (!cancelled) new Notice(error instanceof Error ? error.message : String(error)); } }; + const refreshToken = tokenManager.getRefreshToken(); return { - 683: s( + 551: s( (self) => ({ heading: translate('gdrive'), items: Object.values(self).map((node) => node(node) as SettingGroupItem), @@ -185,49 +175,15 @@ export default function gdriveSetting( }), { 1000: s(() => ({ - desc: translate('clientIdDescription'), - name: translate('clientId'), - render: (setting) => { - setting.addText((text) => { - text.setPlaceholder(translate('clientIdPlaceholder')).setValue( - settings.clientId, - ); - handleInput({ - invalidValue, - key: 'clientId', - processValue: (value) => value.trim(), - saveSettings, - settings, - text, - }); - }); - }, - })), - 2000: s(() => ({ - desc: translate('clientSecretDescription'), - name: translate('clientSecret'), - render: (setting) => { - setting.addComponent((element) => - new SecretComponent(app, element) - .setValue(settings.clientSecret) - .onChange((value) => { - settings.clientSecret = value ?? ''; - void saveSettings(); - }), - ); - }, - })), - 3000: s(() => ({ - desc: settings.refreshToken - ? translate('accountConnected', { account: settings.account }) + desc: refreshToken + ? translate('accountConnected') : translate('accountNotConnected'), name: translate('account'), render: (setting) => { - if (settings.refreshToken) + if (refreshToken) setting.addButton((button) => button.setButtonText(translate('disconnect')).onClick(() => { - settings.refreshToken = ''; - settings.account = ''; + tokenManager.deleteRefreshToken(); tokenManager.invalidate(); void saveSettings(); new Notice(translate('disconnected')); @@ -237,9 +193,7 @@ export default function gdriveSetting( setting.addButton((button) => button .setButtonText( - settings.refreshToken - ? translate('reconnect') - : translate('connect'), + refreshToken ? translate('reconnect') : translate('connect'), ) .setCta() .onClick(async () => { @@ -253,7 +207,7 @@ export default function gdriveSetting( ); }, })), - 4000: s(() => ({ + 2000: s(() => ({ desc: translate('baseDirectoryDescription'), labels: [matchLabel()], name: translate('baseDirectory'), @@ -273,7 +227,7 @@ export default function gdriveSetting( }); }, })), - 5000: s(() => ({ + 3000: s(() => ({ desc: translate('useTrashDescription'), name: translate('useTrash'), render: (setting) => { diff --git a/packages/gdrive/tsdown.config.ts b/packages/gdrive/tsdown.config.ts index f174baad..6abd540d 100644 --- a/packages/gdrive/tsdown.config.ts +++ b/packages/gdrive/tsdown.config.ts @@ -5,6 +5,10 @@ const dev = process.env.MODE === 'dev'; export default defineConfig({ clean: !dev, + define: { + 'process.env.CLIENT_ID': JSON.stringify(process.env.CLIENT_ID ?? ''), + 'process.env.CLIENT_SECRET': JSON.stringify(process.env.CLIENT_SECRET ?? ''), + }, dts: false, entry: { gdrive: 'src/index.ts' }, minify: true, diff --git a/packages/plugin/src/fs/middlewares/retry.ts b/packages/plugin/src/fs/middlewares/retry.ts index 7f5980d2..e4bb683b 100644 --- a/packages/plugin/src/fs/middlewares/retry.ts +++ b/packages/plugin/src/fs/middlewares/retry.ts @@ -1,7 +1,6 @@ import type { ErrorLike } from '@repo/shared/get-status'; import { getStatus } from '@repo/shared/get-status'; import type { Request } from '@/modules/Registrar'; -import sleep from '@/utils/sleep'; type RetryOptions = { maxRetry?: number; diff --git a/packages/plugin/src/utils/sleep.ts b/packages/plugin/src/utils/sleep.ts deleted file mode 100644 index cc803f8e..00000000 --- a/packages/plugin/src/utils/sleep.ts +++ /dev/null @@ -1,5 +0,0 @@ -export default function sleep(ms: number): Promise { - return new Promise((resolve) => { - window.setTimeout(resolve, ms); - }); -} diff --git a/packages/plugin/test/retry-middleware.test.ts b/packages/plugin/test/retry-middleware.test.ts index 9b80b45a..541020dc 100644 --- a/packages/plugin/test/retry-middleware.test.ts +++ b/packages/plugin/test/retry-middleware.test.ts @@ -1,8 +1,8 @@ import testKit from '$/test-kit'; +// oxlint-disable-next-line import/no-namespace +import * as sleepModule from '@repo/shared/sleep'; import { expect, spyOn, test } from 'bun:test'; import { retryMiddleware } from '@/fs'; -// oxlint-disable-next-line import/no-namespace -import * as sleepModule from '@/utils/sleep'; const { bytes, request } = testKit; const sleepSpy = spyOn(sleepModule, 'default').mockImplementation(() => Promise.resolve()); diff --git a/packages/plugin/tsconfig.json b/packages/plugin/tsconfig.json index 59c2dc62..7968ecd5 100644 --- a/packages/plugin/tsconfig.json +++ b/packages/plugin/tsconfig.json @@ -9,5 +9,12 @@ "$/*": ["./test/*"] } }, - "include": ["src/**/*.ts", "src/**/*.tsx", "test/**/*.ts", "tsdown.config.ts", "uno.config.ts"] + "include": [ + "src/**/*.ts", + "src/**/*.tsx", + "test/**/*.ts", + "tsdown.config.ts", + "uno.config.ts", + "../shared/src/sleep.ts" + ] } From 49bc4b4bd59e1baca376f1b175fb969e7b0096b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?He=CC=84sperus?= Date: Sat, 22 Aug 2026 18:05:43 +0800 Subject: [PATCH 3/7] refactor(gdrive): de-slop more --- .oxlintrc.json | 2 +- packages/gdrive/src/gdrive/fs.ts | 218 +++++++++++++++---------------- packages/gdrive/src/index.ts | 10 +- packages/gdrive/src/setting.ts | 59 ++++----- packages/gdrive/src/styles.css | 20 +++ packages/plugin/tsconfig.json | 9 +- 6 files changed, 155 insertions(+), 163 deletions(-) create mode 100644 packages/gdrive/src/styles.css diff --git a/.oxlintrc.json b/.oxlintrc.json index a948fd0f..859a77ac 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -65,7 +65,7 @@ "react/jsx-props-no-spreading": "off", "import/no-unassigned-import": [ "warn", - { "allow": ["**/*.css", "**/*.scss", "**/*.less"] } + { "allow": ["**/*.css", "**/*.scss", "**/*.less", "tsdown/client"] } ], "eslint/no-underscore-dangle": [ "warn", diff --git a/packages/gdrive/src/gdrive/fs.ts b/packages/gdrive/src/gdrive/fs.ts index d6bf9e07..3059a74f 100644 --- a/packages/gdrive/src/gdrive/fs.ts +++ b/packages/gdrive/src/gdrive/fs.ts @@ -1,5 +1,6 @@ import type { Binary, + DatabaseSync, FileStat, ListReporter, Request, @@ -7,6 +8,7 @@ import type { RequestResponse, RootFs, Stat, + StoreSync, } from '@hesprs/sync-engine-sdk'; import { textToUint8Array } from '@repo/shared/binary'; import { getStatus } from '@repo/shared/get-status'; @@ -30,10 +32,13 @@ export type GdriveFsOptions = { useTrash: boolean; }; +export type GdriveDB = DatabaseSync<{ gdriveIds: string }, { gdriveIdsMarker?: string }>; + const READ_CHUNK_SIZE = 2 * 1024 * 1024; // 2 MiB const READ_MAX_CONCURRENT = 8; const PAGE_SIZE = 1000; const WRITE_FIELDS = 'id,md5Checksum'; +const ROOT_ID = 'root'; function notFoundError(key: string): Error { const error = new Error(`Google Drive: ${key} does not exist.`); @@ -42,19 +47,28 @@ function notFoundError(key: string): Error { } /** - * Google Drive stores files by immutable id inside real folders, while the - * sync engine speaks path keys. This class translates keys to ids on demand - * and caches the mapping for the lifetime of the instance. Only files created - * through this module are visible because of the `drive.file` OAuth scope. + * Google Drive stores files by immutable id inside real folders, while + * Sync Engine speaks path keys. Fs translates keys to ids + * and caches the mapping. + * + * Limitation: cannot download a file with known key but not cached ID, this is + * fine in current implementation (impossible to happen). */ export default class GdriveFs implements RootFs { /** Path key (`'/'`, `folder/`, `folder/note.md`) to Drive file id. */ - private readonly ids = new Map(); + private readonly ids: StoreSync; constructor( private readonly request: Request, private readonly options: GdriveFsOptions, - ) {} + memoryDB: GdriveDB, + ) { + this.ids = memoryDB.getStore('gdriveIds'); + if (memoryDB.getMeta('gdriveIdsMarker') !== this.getUid()) { + this.ids.clear(); + memoryDB.setMeta('gdriveIdsMarker', this.getUid()); + } + } getUid(): string { return `gdrive~${this.options.userId}`; @@ -71,90 +85,9 @@ export default class GdriveFs implements RootFs { throw error; } - private async lookupChild( - parentId: string, - name: string, - folder: boolean, - ): Promise { - const mimeClause = folder - ? ` and mimeType = '${FOLDER_MIME}'` - : ` and mimeType != '${FOLDER_MIME}'`; - const url = buildUrl(DRIVE_API, '/files', { - fields: `files(${FILE_FIELDS})`, - orderBy: 'modifiedTime desc', - pageSize: '1', - q: `'${escapeQuery(parentId)}' in parents and name = '${escapeQuery(name)}' and trashed = false${mimeClause}`, - }); - const response = await this.requestOrThrow({ method: 'GET', url }); - return (response.json() as DriveFileList).files?.[0]; - } - - private async createFolder(parentId: string, name: string): Promise { - const response = await this.requestOrThrow({ - body: textToUint8Array( - JSON.stringify({ mimeType: FOLDER_MIME, name, parents: [parentId] }), - ), - headers: { 'Content-Type': 'application/json; charset=UTF-8' }, - method: 'POST', - url: buildUrl(DRIVE_API, '/files', { fields: 'id' }), - }); - const created = response.json() as DriveFile; - if (!created.id) throw new Error('Google Drive did not return an id for a created folder!'); - return created.id; - } - - /** - * Resolves the real root id (never the `root` alias) so listing can match - * ids returned in `parents`. - */ - private async rootId(): Promise { - const cached = this.ids.get('/'); - if (cached !== undefined) return cached; - const response = await this.requestOrThrow({ - method: 'GET', - url: buildUrl(DRIVE_API, '/files/root', { fields: 'id' }), - }); - const id = (response.json() as DriveFile).id; - if (!id) throw new Error('Google Drive did not return the root folder id!'); - this.ids.set('/', id); - return id; - } - - /** Resolves a key to its id, or `undefined` when it does not exist. */ - private async resolveId(key: string): Promise { - if (key === '/') return this.rootId(); - const cached = this.ids.get(key); - if (cached !== undefined) return cached; - const parentId = await this.resolveId(dirname(key)); - if (parentId === undefined) return undefined; - const existing = await this.lookupChild(parentId, basename(key), isFolder(key)); - if (existing) this.ids.set(key, existing.id); - return existing?.id; - } - - /** Resolves a folder key to its id, creating the folder chain when missing. */ - private async ensureFolderId(key: string): Promise { - if (key === '/') return this.rootId(); - const cached = this.ids.get(key); - if (cached !== undefined) return cached; - const parentId = await this.ensureFolderId(dirname(key)); - const existing = await this.lookupChild(parentId, basename(key), true); - if (existing) { - this.ids.set(key, existing.id); - return existing.id; - } - const created = await this.createFolder(parentId, basename(key)); - this.ids.set(key, created); - return created; - } - - /** Fresh metadata lookup for a file key (also refreshes the id cache). */ - private async resolveEntry(key: string): Promise { - const parentId = await this.resolveId(dirname(key)); - if (parentId === undefined) return undefined; - const entry = await this.lookupChild(parentId, basename(key), false); - if (entry) this.ids.set(key, entry.id); - return entry; + private resolveId(key: string): string | undefined { + if (key === '/') return ROOT_ID; + return this.ids.get(key); } private dropCache(key: string): void { @@ -164,23 +97,53 @@ export default class GdriveFs implements RootFs { if (cachedKey.startsWith(key)) this.ids.delete(cachedKey); } + /** + * Walks the path chain of `key` from root with fresh queries, ignoring and + * refreshing the cache along the way. + */ + private async resolveIdFresh(key: string): Promise { + if (key === '/') return ROOT_ID; + const segments = key.split('/').filter((segment) => segment !== ''); + let parentId = ROOT_ID; + let prefix = ''; + for (const [index, segment] of segments.entries()) { + const last = index === segments.length - 1; + const childKey = `${prefix}${segment}${last && !isFolder(key) ? '' : '/'}`; + const folder = !last || isFolder(key); + const response = await this.requestOrThrow({ + method: 'GET', + url: buildUrl(DRIVE_API, '/files', { + fields: 'files(id)', + pageSize: '1', + q: `'${parentId}' in parents and name = '${escapeQuery(segment)}' and mimeType ${folder ? '=' : '!='} '${FOLDER_MIME}' and trashed = false`, + }), + }); + const id = (response.json() as DriveFileList).files?.[0]?.id; + if (!id) return undefined; + this.ids.set(childKey, id); + parentId = id; + prefix = childKey; + } + return parentId; + } + /** Session metadata for uploading `key`, updating the existing file when present. */ - private async sessionFor( + private sessionFor( key: string, stat: FileStat, - ): Promise<{ initiateUrl: string; method: 'PATCH' | 'POST'; metadata: object }> { + ): { initiateUrl: string; method: 'PATCH' | 'POST'; metadata: object } { const modifiedTime = new Date(stat.mtime).toISOString(); - const existing = await this.resolveEntry(key); + const existing = this.resolveId(key); if (existing) return { - initiateUrl: buildUrl(DRIVE_UPLOAD_API, `/files/${existing.id}`, { + initiateUrl: buildUrl(DRIVE_UPLOAD_API, `/files/${existing}`, { fields: WRITE_FIELDS, uploadType: 'resumable', }), metadata: { modifiedTime }, method: 'PATCH', }; - const parentId = await this.ensureFolderId(dirname(key)); + const parentId = this.resolveId(dirname(key)); return { initiateUrl: buildUrl(DRIVE_UPLOAD_API, '/files', { fields: WRITE_FIELDS, @@ -197,7 +160,7 @@ export default class GdriveFs implements RootFs { } async read(key: string): Promise { - const id = await this.resolveId(key); + const id = this.resolveId(key); if (id === undefined) throw notFoundError(key); const response = await this.requestOrThrow({ method: 'GET', @@ -206,8 +169,8 @@ export default class GdriveFs implements RootFs { return response.bytes(); } - async readStream(key: string, { size }: FileStat): Promise> { - const id = await this.resolveId(key); + readStream(key: string, { size }: FileStat): ReadableStream { + const id = this.resolveId(key); if (id === undefined) throw notFoundError(key); const url = buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }); return createRangeReadStream({ @@ -228,7 +191,7 @@ export default class GdriveFs implements RootFs { async write(key: string, value: Binary, stat: FileStat): Promise { const file = await singlePutUpload( { - ...(await this.sessionFor(key, stat)), + ...this.sessionFor(key, stat), request: this.request, size: value.byteLength, }, @@ -240,7 +203,7 @@ export default class GdriveFs implements RootFs { async writeStream(key: string, value: ReadableStream, stat: FileStat): Promise { const file = await resumableUpload( - { ...(await this.sessionFor(key, stat)), request: this.request, size: stat.size }, + { ...this.sessionFor(key, stat), request: this.request, size: stat.size }, value, ); if (file.id) this.ids.set(key, file.id); @@ -248,7 +211,7 @@ export default class GdriveFs implements RootFs { } async delete(key: string): Promise { - const id = await this.resolveId(key); + const id = this.resolveId(key); if (id === undefined) return; try { await this.requestOrThrow( @@ -271,10 +234,11 @@ export default class GdriveFs implements RootFs { } async move(oldKey: string, newKey: string): Promise { - const id = await this.resolveId(oldKey); + const id = this.resolveId(oldKey); if (id === undefined) throw notFoundError(oldKey); - const oldParentId = await this.resolveId(dirname(oldKey)); - const newParentId = await this.ensureFolderId(dirname(newKey)); + const oldParentId = this.resolveId(dirname(oldKey)); + const newParentId = this.resolveId(dirname(newKey)); + if (!newParentId) throw new Error(`Parent not created when moving to "${newKey}"!`); const query: Record = { fields: 'id' }; if (oldParentId !== newParentId) { query.addParents = newParentId; @@ -290,31 +254,53 @@ export default class GdriveFs implements RootFs { } /** Drive folders always need an existing parent id, so missing parents are created regardless of `recursive`. */ - async mkdir(key: string): Promise { - await this.ensureFolderId(key); + async mkdir(key: string, recursive: boolean): Promise { + const parent = dirname(key); + let parentId = this.resolveId(parent); + if (!parent && recursive) { + await this.mkdir(parent, true); + parentId = this.resolveId(parent); + } + if (!parentId) throw new Error(`Parent is not created when creating "${key}"!`); + const response = await this.requestOrThrow({ + body: textToUint8Array( + JSON.stringify({ mimeType: FOLDER_MIME, name: basename(key), parents: [parentId] }), + ), + headers: { 'Content-Type': 'application/json; charset=UTF-8' }, + method: 'POST', + url: buildUrl(DRIVE_API, '/files', { fields: 'id' }), + }); + const created = response.json() as DriveFile; + if (!created.id) throw new Error('Google Drive did not return an id for a created folder!'); + this.ids.set(key, created.id); } async stat(key: string): Promise { - if (isFolder(key)) { - if ((await this.resolveId(key)) === undefined) throw notFoundError(key); - return { isDir: true, key }; - } - const entry = await this.resolveEntry(key); + const parentId = this.resolveId(dirname(key)); + if (!parentId) throw notFoundError(key); + const url = buildUrl(DRIVE_API, '/files', { + fields: `files(${FILE_FIELDS})`, + orderBy: 'modifiedTime desc', + pageSize: '1', + q: `'${parentId}' in parents and name = '${escapeQuery(basename(key))}' and trashed = false`, + }); + const response = await this.requestOrThrow({ method: 'GET', url }); + const entry = (response.json() as DriveFileList).files?.[0]; if (!entry) throw notFoundError(key); return toFileStat(key, entry); } + // When Sync Engine calls `exists()`, the only possibility is that something is unexpected, don't trust cache here async exists(key: string): Promise { - return key === '/' || (await this.resolveId(key)) !== undefined; + return (await this.resolveIdFresh(key)) !== undefined; } /** - * Fetches every visible file in one paginated query (the `drive.file` - * scope limits results to files this module created), then walks the tree + * Fetches every visible file in one paginated query, then walks the tree * under the requested key so the reporter can steer traversal. */ async list(key: string, reporter: ListReporter): Promise> { - const startId = await this.resolveId(key); + const startId = this.resolveId(key) ?? (await this.resolveIdFresh(key)); if (startId === undefined) throw notFoundError(key); const all: Array = []; let pageToken: string | undefined; @@ -337,7 +323,7 @@ export default class GdriveFs implements RootFs { const childrenByParent = new Map>(); for (const file of all) { const parent = file.parents?.[0]; - if (!parent || file.name.includes('/')) continue; + if (!parent) continue; const siblings = childrenByParent.get(parent); if (siblings) siblings.push(file); else childrenByParent.set(parent, [file]); diff --git a/packages/gdrive/src/index.ts b/packages/gdrive/src/index.ts index 89250d56..d50b4762 100644 --- a/packages/gdrive/src/index.ts +++ b/packages/gdrive/src/index.ts @@ -1,3 +1,4 @@ +import 'tsdown/client'; import type { Context, FsWrapperEntry, @@ -13,12 +14,14 @@ import type { } from '@hesprs/sync-engine-sdk'; import type { App } from 'obsidian'; import { digOriginal, prefixWrapper } from '@hesprs/sync-engine-sdk'; +import type { GdriveDB } from './gdrive/fs'; import type { GdriveTranslations } from './setting'; import { TokenManager, bearerMiddleware } from './gdrive/auth'; import checkConnection from './gdrive/check-connection'; import GdriveFs from './gdrive/fs'; import en from './i18n'; import gdriveSetting from './setting'; +import styles from './styles.css?inline'; export type GdriveSettings = { baseDirectory: string; @@ -35,10 +38,12 @@ export default class Gdrive { translate: Translate; registerRemoteFs: (id: string, entry: RemoteFsEntry) => () => void; app: App; + memoryDB: GdriveDB; registerRemoteFsWrapper: (entry: FsWrapperEntry) => () => void; registerRemoteRequestMiddleware: (entry: RemoteRequestMiddlewareEntry) => () => void; registerSetting: (entry: SettingEntry) => () => void; registerI18n: (lang: ObsidianLanguageCode, translations: TranslationResource) => void; + registerCss: (css: string) => () => void; }>, ) { if (!this.moduleSettings.baseDirectory) @@ -59,14 +64,17 @@ export default class Gdrive { const { translate, registerRemoteFs, + memoryDB, registerRemoteFsWrapper, registerRemoteRequestMiddleware, registerSetting, + registerCss, } = this.ctx; this.cleanup.push( + registerCss(styles), registerRemoteFs('gdrive', { checkConnection, - instantiate: (request) => new GdriveFs(request, this.moduleSettings), + instantiate: (request) => new GdriveFs(request, this.moduleSettings, memoryDB), prettyName: () => translate('gdrive'), }), registerRemoteFsWrapper({ diff --git a/packages/gdrive/src/setting.ts b/packages/gdrive/src/setting.ts index c87c50f0..ccd21a79 100644 --- a/packages/gdrive/src/setting.ts +++ b/packages/gdrive/src/setting.ts @@ -43,14 +43,9 @@ export type GdriveTranslations = { }; type DeviceCodeModalOptions = { - title: string; - instruction: string; + translate: Translate; userCode: string; verificationUrl: string; - copyLabel: string; - copiedLabel: string; - openLabel: string; - waitingLabel: string; onClose: () => void; }; @@ -63,39 +58,36 @@ class DeviceCodeModal extends Modal { } override onOpen(): void { - const { contentEl, titleEl } = this; - titleEl.setText(this.options.title); - contentEl.createEl('p', { text: this.options.instruction }); - const codeEl = contentEl.createEl('div', { text: this.options.userCode }); - codeEl.setCssStyles({ - fontSize: '2em', - fontWeight: '700', - letterSpacing: '0.15em', - margin: '0.5em 0', - textAlign: 'center', - userSelect: 'text', + const { + contentEl, + titleEl, + options: { translate, userCode, verificationUrl }, + } = this; + titleEl.setText(translate('deviceCodeTitle')); + contentEl.createEl('p', { + text: translate('deviceCodeInstruction', { url: verificationUrl }), }); - const buttonRow = contentEl.createEl('div'); - buttonRow.setCssStyles({ - display: 'flex', - gap: '0.5em', - justifyContent: 'center', - marginBottom: '0.75em', + contentEl.createEl('code', { + cls: 'gdrive-device-code', + text: userCode, }); - const copyButton = buttonRow.createEl('button', { text: this.options.copyLabel }); - copyButton.addEventListener('click', () => { + const buttonRow = contentEl.createEl('div', 'gdrive-device-code-buttons'); + const copyButton = buttonRow.createEl('button', { text: translate('copyCode') }); + copyButton.onClickEvent(() => { void navigator.clipboard.writeText(this.options.userCode); - copyButton.setText(this.options.copiedLabel); + copyButton.setText(translate('codeCopied')); }); const openButton = buttonRow.createEl('button', { cls: 'mod-cta', - text: this.options.openLabel, + text: translate('openVerificationPage'), }); openButton.addEventListener('click', () => { window.open(this.options.verificationUrl); }); - const statusEl = contentEl.createEl('p', { text: this.options.waitingLabel }); - statusEl.setCssStyles({ opacity: '0.7', textAlign: 'center' }); + contentEl.createEl('p', { + cls: 'gdrive-device-code-status', + text: translate('waitingApproval'), + }); } override onClose(): void { @@ -128,19 +120,12 @@ export default function gdriveSetting( const authorization = await startDeviceAuthorization(); let finished = false; const modal = new DeviceCodeModal(app, { - copiedLabel: translate('codeCopied'), - copyLabel: translate('copyCode'), - instruction: translate('deviceCodeInstruction', { - url: authorization.verificationUrl, - }), onClose: () => { if (!finished) cancelled = true; }, - openLabel: translate('openVerificationPage'), - title: translate('deviceCodeTitle'), + translate, userCode: authorization.userCode, verificationUrl: authorization.verificationUrl, - waitingLabel: translate('waitingApproval'), }); modal.open(); try { diff --git a/packages/gdrive/src/styles.css b/packages/gdrive/src/styles.css new file mode 100644 index 00000000..98787239 --- /dev/null +++ b/packages/gdrive/src/styles.css @@ -0,0 +1,20 @@ +.gdrive-device-code { + font-size: 2em; + font-weight: 700; + letter-spacing: 0.15em; + margin: 0.5em 0; + text-align: center; + user-select: text; +} + +.gdrive-device-code-buttons { + display: flex; + gap: 0.5em; + justify-content: center; + margin-bottom: 0.75em; +} + +.gdrive-device-code-status { + opacity: 0.7; + text-align: center; +} diff --git a/packages/plugin/tsconfig.json b/packages/plugin/tsconfig.json index 7968ecd5..59c2dc62 100644 --- a/packages/plugin/tsconfig.json +++ b/packages/plugin/tsconfig.json @@ -9,12 +9,5 @@ "$/*": ["./test/*"] } }, - "include": [ - "src/**/*.ts", - "src/**/*.tsx", - "test/**/*.ts", - "tsdown.config.ts", - "uno.config.ts", - "../shared/src/sleep.ts" - ] + "include": ["src/**/*.ts", "src/**/*.tsx", "test/**/*.ts", "tsdown.config.ts", "uno.config.ts"] } From ea5ad2688af70f8a3b5269f978625010c68368c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?He=CC=84sperus?= Date: Sun, 23 Aug 2026 00:59:49 +0800 Subject: [PATCH 4/7] fix(gdrive): make auth work --- packages/gdrive/src/gdrive/api.ts | 3 +- packages/gdrive/src/gdrive/auth.ts | 35 ++++-- packages/gdrive/src/gdrive/fs.ts | 2 +- packages/gdrive/src/i18n.ts | 32 +++--- packages/gdrive/src/setting.ts | 151 +++++++++++++------------ packages/gdrive/src/styles.css | 16 +-- packages/plugin/dist/index.spec.d.ts | 2 + packages/plugin/src/en.ts | 4 +- packages/plugin/src/modules/Setting.ts | 2 + 9 files changed, 130 insertions(+), 117 deletions(-) diff --git a/packages/gdrive/src/gdrive/api.ts b/packages/gdrive/src/gdrive/api.ts index 67ef0757..c5dff975 100644 --- a/packages/gdrive/src/gdrive/api.ts +++ b/packages/gdrive/src/gdrive/api.ts @@ -4,9 +4,10 @@ export const DRIVE_API = 'https://www.googleapis.com/drive/v3'; export const DRIVE_UPLOAD_API = 'https://www.googleapis.com/upload/drive/v3'; export const OAUTH_DEVICE_CODE_URL = 'https://oauth2.googleapis.com/device/code'; export const OAUTH_TOKEN_URL = 'https://oauth2.googleapis.com/token'; -export const OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive.file email'; +export const OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive.file openid'; export const FOLDER_MIME = 'application/vnd.google-apps.folder'; export const FILE_FIELDS = 'id,name,mimeType,md5Checksum,modifiedTime,size,parents'; +export const TOKEN_REVOKE_URL = 'https://oauth2.googleapis.com/revoke'; export type DriveFile = { id: string; diff --git a/packages/gdrive/src/gdrive/auth.ts b/packages/gdrive/src/gdrive/auth.ts index aa2ab2ec..edebfd8d 100644 --- a/packages/gdrive/src/gdrive/auth.ts +++ b/packages/gdrive/src/gdrive/auth.ts @@ -1,7 +1,13 @@ import type { Request, RequestParam } from '@hesprs/sync-engine-sdk'; import { getStatus } from '@repo/shared/get-status'; import { requestUrl, SecretStorage } from 'obsidian'; -import { OAUTH_DEVICE_CODE_URL, OAUTH_SCOPE, OAUTH_TOKEN_URL } from './api'; +import { + buildUrl, + OAUTH_DEVICE_CODE_URL, + OAUTH_SCOPE, + OAUTH_TOKEN_URL, + TOKEN_REVOKE_URL, +} from './api'; export const CLIENT_ID = process.env.CLIENT_ID ?? ''; export const CLIENT_SECRET = process.env.CLIENT_SECRET ?? ''; // Not really a secret @@ -10,8 +16,8 @@ const KEYCHAIN_SECRET_ID = 'sync-engine-gdrive-refresh-token'; // Secret storage type TokenResponse = { access_token: string; expires_in: number; - user_id?: string; refresh_token?: string; + id_token?: string; }; type TokenError = { @@ -26,14 +32,12 @@ type TokenError = { | 'expired_token' | 'access_denied'; error_description?: string; - error_uri?: string; }; type DeviceCodeResponse = { device_code: string; user_code: string; - verification_uri: string; - verification_uri_complete?: string; + verification_url: string; expires_in: number; interval: number; }; @@ -41,7 +45,6 @@ type DeviceCodeResponse = { type DeviceCodeError = { error: 'invalid_request' | 'invalid_client' | 'unsupported_grant_type' | 'unauthorized_client'; error_description?: string; - error_uri?: string; }; export type DeviceAuthorization = { @@ -87,7 +90,7 @@ export async function startDeviceAuthorization(): Promise { expiresIn: data.expires_in, interval: data.interval, userCode: data.user_code, - verificationUrl: data.verification_uri, + verificationUrl: data.verification_url, }; } @@ -116,12 +119,12 @@ export async function pollDeviceToken(options: { }); const data = response.json as TokenResponse | TokenError; if ('access_token' in data) - if (data.refresh_token && data.user_id) + if (data.refresh_token && data.id_token) return { accessToken: data.access_token, expiresIn: data.expires_in, refreshToken: data.refresh_token, - userId: data.user_id, + userId: extractSub(data.id_token), }; else throw new Error('Google authorization payload is malformed!'); switch (data.error) { @@ -146,6 +149,20 @@ export async function pollDeviceToken(options: { } } } +function extractSub(idToken: string): string { + const payload = JSON.parse(atob(idToken.split('.')[1])) as { sub: string }; + return payload.sub; +} + +// Fire-and-forget revocation +export function revokeToken(token: string) { + return requestUrl({ + contentType: FORM_CONTENT_TYPE, + method: 'POST', + throw: false, + url: buildUrl(TOKEN_REVOKE_URL, '', { token }), + }).catch(() => {}); +} /** * Caches the short-lived access token and refreshes it with the stored refresh diff --git a/packages/gdrive/src/gdrive/fs.ts b/packages/gdrive/src/gdrive/fs.ts index 3059a74f..3bed4052 100644 --- a/packages/gdrive/src/gdrive/fs.ts +++ b/packages/gdrive/src/gdrive/fs.ts @@ -319,7 +319,7 @@ export default class GdriveFs implements RootFs { all.push(...(parsed.files ?? [])); pageToken = parsed.nextPageToken; } while (pageToken); - + this.ids.clear(); const childrenByParent = new Map>(); for (const file of all) { const parent = file.parents?.[0]; diff --git a/packages/gdrive/src/i18n.ts b/packages/gdrive/src/i18n.ts index db298d1a..d31e5c4c 100644 --- a/packages/gdrive/src/i18n.ts +++ b/packages/gdrive/src/i18n.ts @@ -1,36 +1,30 @@ import type { GdriveTranslations } from './setting'; const en: GdriveTranslations = { - account: 'Google account', - accountConnected: 'Connected as {{account}}.', - accountNotConnected: - 'Not connected. Enter the OAuth client credentials above, then connect your Google account.', + accountConnected: 'Account connected', + accountConnectedDescription: 'Connected to Google Drive account.', + authorizationFailed: 'Authorization failed: {{reason}}', baseDirectory: 'Base directory', baseDirectoryDescription: - 'Folder in Google Drive that holds this vault. Created automatically on the first sync — do not create it manually in Drive, files added outside this plugin stay invisible to it.', + 'Set the folder in Google Drive that holds this vault. Created automatically on the first sync and do not create it manually in Drive, files added outside this plugin are invisible to sync.', baseDirectoryPlaceholder: 'my-vault/', - clientId: 'OAuth client ID', - clientIdDescription: - 'Client ID of your own Google Cloud OAuth client (application type "TV and Limited Input devices") with the Google Drive API enabled.', - clientIdPlaceholder: 'xxxxxxxx.apps.googleusercontent.com', - clientSecret: 'OAuth client secret', - clientSecretDescription: 'Client secret of the same OAuth client.', - codeCopied: 'Copied', configureFirst: 'Enter the OAuth client ID and client secret first.', connect: 'Connect', + connectAccount: 'Connect account', + connectAccountDescription: 'Click the button to connect to you Google Drive account.', connectSuccess: 'Connected to Google Drive.', - copyCode: 'Copy code', - deviceCodeInstruction: - 'On any device, visit {{url}} and enter the code below, then approve access.', + copyAndOpenGoogle: 'Copy and open Google', + deviceCodeInstruction: (frag, url) => { + frag.appendText('Please visit '); + frag.createEl('a', { attr: { href: url } }).createEl('code', { text: url }); + frag.appendText(' and enter the code below, then approve access.'); + }, deviceCodeTitle: 'Connect Google Drive', disconnect: 'Disconnect', - disconnected: 'Google Drive disconnected.', gdrive: 'Google Drive', - openVerificationPage: 'Open Google', - reconnect: 'Reconnect', useTrash: 'Delete to trash', useTrashDescription: - 'Move remotely deleted files to the Google Drive trash instead of deleting them permanently. Drive clears its trash after 30 days.', + 'Move deleted files to the Google Drive trash instead of deleting them permanently. Drive clears its trash after 30 days.', waitingApproval: 'Waiting for approval…', }; diff --git a/packages/gdrive/src/setting.ts b/packages/gdrive/src/setting.ts index ccd21a79..df3bee49 100644 --- a/packages/gdrive/src/setting.ts +++ b/packages/gdrive/src/setting.ts @@ -1,6 +1,7 @@ import type { GdriveSettings } from '@'; import type { CallableOrObjectTree, + Fragment, LabelDefinition, Translate, Translations, @@ -8,31 +9,23 @@ import type { import type { App, SettingGroupItem } from 'obsidian'; import { s } from '@hesprs/sync-engine-sdk'; import { normalizeBaseDir } from '@repo/shared/path'; -import { Modal, Notice } from 'obsidian'; +import { Modal, Notice, Setting } from 'obsidian'; import type { TokenManager } from './gdrive/auth'; -import { pollDeviceToken, startDeviceAuthorization } from './gdrive/auth'; +import { pollDeviceToken, revokeToken, startDeviceAuthorization } from './gdrive/auth'; import handleInput from './handle-input'; export type GdriveTranslations = { gdrive: string; - clientId: string; - clientIdDescription: string; - clientIdPlaceholder: string; - clientSecret: string; - clientSecretDescription: string; - account: string; + connectAccount: string; accountConnected: string; - accountNotConnected: string; + accountConnectedDescription: string; + connectAccountDescription: string; connect: string; - reconnect: string; disconnect: string; - disconnected: string; configureFirst: string; deviceCodeTitle: string; - deviceCodeInstruction: string; - copyCode: string; - codeCopied: string; - openVerificationPage: string; + deviceCodeInstruction: Fragment; + copyAndOpenGoogle: string; waitingApproval: string; connectSuccess: string; baseDirectory: string; @@ -40,10 +33,11 @@ export type GdriveTranslations = { baseDirectoryPlaceholder: string; useTrash: string; useTrashDescription: string; + authorizationFailed: string; }; type DeviceCodeModalOptions = { - translate: Translate; + translate: Translate; userCode: string; verificationUrl: string; onClose: () => void; @@ -57,42 +51,43 @@ class DeviceCodeModal extends Modal { super(app); } - override onOpen(): void { + onOpen(): void { const { contentEl, titleEl, options: { translate, userCode, verificationUrl }, } = this; titleEl.setText(translate('deviceCodeTitle')); + contentEl.addClass('markdown-rendered'); contentEl.createEl('p', { - text: translate('deviceCodeInstruction', { url: verificationUrl }), - }); - contentEl.createEl('code', { - cls: 'gdrive-device-code', - text: userCode, - }); - const buttonRow = contentEl.createEl('div', 'gdrive-device-code-buttons'); - const copyButton = buttonRow.createEl('button', { text: translate('copyCode') }); - copyButton.onClickEvent(() => { - void navigator.clipboard.writeText(this.options.userCode); - copyButton.setText(translate('codeCopied')); - }); - const openButton = buttonRow.createEl('button', { - cls: 'mod-cta', - text: translate('openVerificationPage'), - }); - openButton.addEventListener('click', () => { - window.open(this.options.verificationUrl); + text: translate('deviceCodeInstruction', verificationUrl), }); + contentEl.createEl('code', { cls: 'gdrive-device-code', text: userCode }); contentEl.createEl('p', { cls: 'gdrive-device-code-status', text: translate('waitingApproval'), }); + new Setting(contentEl) + .addButton((button) => + button + .setButtonText(translate('cancel')) + .setDestructive() + .onClick(() => this.close()), + ) + .addButton((button) => + button + .setCta() + .setButtonText(translate('copyAndOpenGoogle')) + .onClick(() => { + void navigator.clipboard.writeText(userCode); + window.open(verificationUrl); + button.setIcon('check'); + }), + ); } - - override onClose(): void { - this.options.onClose(); + onClose(): void { this.contentEl.empty(); + this.options.onClose(); } } @@ -102,26 +97,26 @@ export default function gdriveSetting( saveSettings, app, matchLabel, - rerenderSettingTab, + refreshSettingTab, }: { translate: Translate; saveSettings: () => Promise; app: App; matchLabel: () => LabelDefinition; - rerenderSettingTab: () => void; + refreshSettingTab: () => void; }, settings: GdriveSettings, tokenManager: TokenManager, ): CallableOrObjectTree { const invalidValue = translate('invalidValue'); - const connectGoogle = async () => { + const connectGoogle = async (resolve: () => void) => { let cancelled = false; try { const authorization = await startDeviceAuthorization(); - let finished = false; const modal = new DeviceCodeModal(app, { onClose: () => { - if (!finished) cancelled = true; + cancelled = true; + resolve(); }, translate, userCode: authorization.userCode, @@ -133,23 +128,26 @@ export default function gdriveSetting( authorization, isCancelled: () => cancelled, }); - finished = true; tokenManager.setRefreshToken(refreshToken); settings.userId = userId; tokenManager.setToken(accessToken, expiresIn); - await saveSettings(); - tokenManager.invalidate(); + void saveSettings(); new Notice(translate('connectSuccess')); + refreshSettingTab(); } finally { - finished = true; modal.close(); } - rerenderSettingTab(); } catch (error) { - if (!cancelled) new Notice(error instanceof Error ? error.message : String(error)); + if (!cancelled) + new Notice( + translate('authorizationFailed', { + reason: error instanceof Error ? error.message : String(error), + }), + ); + } finally { + resolve(); } }; - const refreshToken = tokenManager.getRefreshToken(); return { 551: s( @@ -160,37 +158,44 @@ export default function gdriveSetting( }), { 1000: s(() => ({ - desc: refreshToken - ? translate('accountConnected') - : translate('accountNotConnected'), - name: translate('account'), + desc: translate('connectAccountDescription'), + name: translate('connectAccount'), render: (setting) => { - if (refreshToken) - setting.addButton((button) => - button.setButtonText(translate('disconnect')).onClick(() => { - tokenManager.deleteRefreshToken(); - tokenManager.invalidate(); - void saveSettings(); - new Notice(translate('disconnected')); - rerenderSettingTab(); - }), - ); setting.addButton((button) => button - .setButtonText( - refreshToken ? translate('reconnect') : translate('connect'), - ) + .setButtonText(translate('connect')) .setCta() + .onClick( + () => + new Promise((resolve) => { + void connectGoogle(resolve); + }), + ), + ); + }, + visible: () => !tokenManager.getRefreshToken(), + })), + 1100: s(() => ({ + desc: translate('accountConnectedDescription'), + name: translate('accountConnected'), + render: (setting) => { + setting.addButton((button) => + button + .setButtonText(translate('disconnect')) + .setDestructive() .onClick(async () => { - button.setDisabled(true); - try { - await connectGoogle(); - } finally { - button.setDisabled(false); - } + const token = tokenManager.getRefreshToken(); + if (!token) return; + await revokeToken(token); + settings.userId = ''; + tokenManager.deleteRefreshToken(); + tokenManager.invalidate(); + void saveSettings(); + refreshSettingTab(); }), ); }, + visible: () => Boolean(tokenManager.getRefreshToken()), })), 2000: s(() => ({ desc: translate('baseDirectoryDescription'), diff --git a/packages/gdrive/src/styles.css b/packages/gdrive/src/styles.css index 98787239..c6ee3ad6 100644 --- a/packages/gdrive/src/styles.css +++ b/packages/gdrive/src/styles.css @@ -1,20 +1,14 @@ -.gdrive-device-code { +code.gdrive-device-code { + display: block; font-size: 2em; font-weight: 700; letter-spacing: 0.15em; - margin: 0.5em 0; - text-align: center; + margin: 0.5em auto; user-select: text; -} - -.gdrive-device-code-buttons { - display: flex; - gap: 0.5em; - justify-content: center; - margin-bottom: 0.75em; + width: fit-content; } .gdrive-device-code-status { - opacity: 0.7; + color: var(--text-muted); text-align: center; } diff --git a/packages/plugin/dist/index.spec.d.ts b/packages/plugin/dist/index.spec.d.ts index 957a7afb..5a60195c 100644 --- a/packages/plugin/dist/index.spec.d.ts +++ b/packages/plugin/dist/index.spec.d.ts @@ -382,12 +382,14 @@ declare class Setting$1 { private readonly speedLabel; private readonly addSettingTab; private readonly rerenderSettingTab; + private readonly refreshSettingTab; root: { addSettingTab: (plugin: Plugin) => void; matchLabel: () => { text: string; tooltip: string; }; + refreshSettingTab: () => void | undefined; registerSetting: (entry: SettingEntry) => () => boolean; rerenderSettingTab: () => void | undefined; speedLabel: () => { diff --git a/packages/plugin/src/en.ts b/packages/plugin/src/en.ts index c5b70508..907603b0 100644 --- a/packages/plugin/src/en.ts +++ b/packages/plugin/src/en.ts @@ -11,9 +11,7 @@ const en: Translations = { asymmetricStorageDescription: (frag) => { frag.appendText('Use '); frag.createEl('a', { - attr: { - href: 'https://sync.consensia.cc/deep-dive/asymmetric-storage', - }, + attr: { href: 'https://sync.consensia.cc/deep-dive/asymmetric-storage' }, text: 'asymmetric storage', }); frag.appendText(' to substantially accelerate syncing.'); diff --git a/packages/plugin/src/modules/Setting.ts b/packages/plugin/src/modules/Setting.ts index 868f9b95..393f30e5 100644 --- a/packages/plugin/src/modules/Setting.ts +++ b/packages/plugin/src/modules/Setting.ts @@ -77,10 +77,12 @@ export default class Setting { plugin.addSettingTab(this.settingTab); }; private readonly rerenderSettingTab = () => this.settingTab?.update(); + private readonly refreshSettingTab = () => this.settingTab?.refreshDomState(); root = { addSettingTab: this.addSettingTab, matchLabel: this.matchLabel, + refreshSettingTab: this.refreshSettingTab, registerSetting: setRegister(this.settingRegistry), rerenderSettingTab: this.rerenderSettingTab, speedLabel: this.speedLabel, From 034278a8492d28f78fa033e03055f1c9ebe30b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?He=CC=84sperus?= Date: Sun, 23 Aug 2026 11:01:15 +0800 Subject: [PATCH 5/7] fix(gdrive & core): kill bugs --- packages/gdrive/src/gdrive/fs.ts | 2 + .../plugin/src/fs/hierarchical-optimizer.ts | 31 +++-- packages/plugin/src/fs/vault/request.ts | 34 +++++- ...r.test.ts => hierarchal-optimizer.test.ts} | 111 ++++++++++++++++++ packages/plugin/test/retry-middleware.test.ts | 5 +- 5 files changed, 162 insertions(+), 21 deletions(-) rename packages/plugin/test/{fs-hierarchal-optimizer.test.ts => hierarchal-optimizer.test.ts} (66%) diff --git a/packages/gdrive/src/gdrive/fs.ts b/packages/gdrive/src/gdrive/fs.ts index 3bed4052..d1753130 100644 --- a/packages/gdrive/src/gdrive/fs.ts +++ b/packages/gdrive/src/gdrive/fs.ts @@ -319,7 +319,9 @@ export default class GdriveFs implements RootFs { all.push(...(parsed.files ?? [])); pageToken = parsed.nextPageToken; } while (pageToken); + this.ids.clear(); + this.ids.set(key, startId); const childrenByParent = new Map>(); for (const file of all) { const parent = file.parents?.[0]; diff --git a/packages/plugin/src/fs/hierarchical-optimizer.ts b/packages/plugin/src/fs/hierarchical-optimizer.ts index a60dbbe5..cbff475b 100644 --- a/packages/plugin/src/fs/hierarchical-optimizer.ts +++ b/packages/plugin/src/fs/hierarchical-optimizer.ts @@ -79,26 +79,33 @@ export default function hierarchicalOptimizer({ atoms, executeAtom }: OptimizerI const { write } = pathsOf(atom); if (write && (`${deletion.key}/` === write || `${write}/` === deletion.key)) dependencies.get(atom)?.add(deletion); + if (atom.type === 'move' && isSub(deletion.key, atom.oldKey)) + dependencies.get(deletion)?.add(atom); } } for (const atom of atoms) { const originalExecute = atom.execute; atom.execute = (async () => { - if (isSubsumable(atom)) { - const umbrella = umbrellas.get(atom); - if (umbrella) { - await executeAtom(umbrella); - atom.resolve(); - return; + try { + if (isSubsumable(atom)) { + const umbrella = umbrellas.get(atom); + if (umbrella) { + await executeAtom(umbrella); + atom.resolve(); + return; + } } + await Promise.all( + [...(dependencies.get(atom) as Set)].map((dependency) => + executeAtom(dependency), + ), + ); + return await originalExecute(); + } catch (error) { + atom.reject(error instanceof Error ? error : new Error(String(error))); + throw error; } - await Promise.all( - [...(dependencies.get(atom) as Set)].map((dependency) => - executeAtom(dependency), - ), - ); - return originalExecute(); }) as never; } diff --git a/packages/plugin/src/fs/vault/request.ts b/packages/plugin/src/fs/vault/request.ts index 3ca5f08f..0eb2879f 100644 --- a/packages/plugin/src/fs/vault/request.ts +++ b/packages/plugin/src/fs/vault/request.ts @@ -1,8 +1,8 @@ import type { Vault, Stat, ListedFiles, App } from 'obsidian'; import { toArrayBuffer, toUint8Array } from '@repo/shared/binary'; -import { isFolder, stripEndSlash } from '@repo/shared/path'; -import { TFile, TFolder } from 'obsidian'; -import type { Binary } from '@/types'; +import { basename, isFolder, stripEndSlash } from '@repo/shared/path'; +import { Platform, TFile, TFolder } from 'obsidian'; +import type { Binary, MaybePromise } from '@/types'; type VaultRequestParam = | { method: 'GET'; key: string } @@ -73,9 +73,13 @@ export default function createVaultRequest(app: App): VaultRequest { return response.body as never; } if (method === 'PUT') - return adapter.writeBinary(path, toArrayBuffer(params.value), params.headers) as never; + return withCheckChars(key, () => + adapter.writeBinary(path, toArrayBuffer(params.value), params.headers), + ) as never; if (method === 'APPEND') - return adapter.appendBinary(path, toArrayBuffer(params.value), params.headers) as never; + return withCheckChars(key, () => + adapter.appendBinary(path, toArrayBuffer(params.value), params.headers), + ) as never; if (method === 'DELETE') { const trashOption = getTrashOption(vault); if (trashOption === 'permanent' || params.headers?.permanent) @@ -86,7 +90,10 @@ export default function createVaultRequest(app: App): VaultRequest { } if (method === 'MOVE') return adapter.rename(path, toVaultPath(params.headers.destination)) as never; - if (method === 'MKDIR') return (key === '/' ? undefined : adapter.mkdir(path)) as never; + if (method === 'MKDIR') + return ( + key === '/' ? undefined : withCheckChars(key, () => adapter.mkdir(path)) + ) as never; if (method === 'EXISTS') { if (vault.getAbstractFileByPath(path)) return true as never; return adapter.exists(path, true) as never; @@ -124,3 +131,18 @@ export default function createVaultRequest(app: App): VaultRequest { return undefined as never; }; } + +async function withCheckChars(key: string, action: () => MaybePromise): Promise { + try { + return await action(); + } catch (error: unknown) { + if (Platform.isWin) { + const match = /[<>:"/\\|?*]/u.exec(basename(key)); + if (match) + throw new Error(`Windows forbids character "${match[0]}" in file names!`, { + cause: error, + }); + } + throw error; + } +} diff --git a/packages/plugin/test/fs-hierarchal-optimizer.test.ts b/packages/plugin/test/hierarchal-optimizer.test.ts similarity index 66% rename from packages/plugin/test/fs-hierarchal-optimizer.test.ts rename to packages/plugin/test/hierarchal-optimizer.test.ts index 5c684721..100bcfbe 100644 --- a/packages/plugin/test/fs-hierarchal-optimizer.test.ts +++ b/packages/plugin/test/hierarchal-optimizer.test.ts @@ -78,6 +78,34 @@ test('mkdir chain waits for ancestor mkdir', async () => { await pending; }); +test('dependent atom receives ancestor failure', async () => { + const parentError = new Error('parent mkdir failed'); + let childRejection: unknown; + const atoms: Array = [ + { + execute: () => Promise.reject(parentError), + key: 'folder/', + reject: () => {}, + resolve: () => {}, + type: 'mkdir', + }, + { + execute: () => 'write-uid', + key: 'folder/note.md', + reject: (error) => (childRejection = error), + resolve: () => {}, + type: 'write', + }, + ]; + const { executeAtom, optimized } = runOptimizer(atoms); + const results = await Promise.all( + optimized.map((atom) => executeAtom(atom).catch((error: unknown) => error)), + ); + + expect(results[1]).toBe(parentError); + expect(childRejection).toBe(parentError); +}); + test('move gates operations under destination', async () => { const logs: Array = []; const move = deferred(); @@ -117,6 +145,89 @@ test('move gates operations under destination', async () => { await pending; }); +test('Parent deletion waits for descendant moves', async () => { + const logs: Array = []; + const move = deferred(); + const atoms: Array = [ + { + execute: async () => { + logs.push('move:folder/src/->folder/dst/'); + await move.promise; + }, + newKey: 'folder/dst/', + oldKey: 'folder/src/', + reject: () => {}, + resolve: () => {}, + type: 'move', + }, + { + execute: () => { + logs.push('move:folder/src/a.md->folder/dst/a.md'); + }, + newKey: 'folder/dst/a.md', + oldKey: 'folder/src/a.md', + reject: () => {}, + resolve: () => {}, + type: 'move', + }, + { + execute: () => { + logs.push('delete:folder/'); + }, + key: 'folder/', + reject: () => {}, + resolve: () => {}, + type: 'delete', + }, + ]; + const { executeAtom, optimized } = runOptimizer(atoms); + const pending = Promise.all(optimized.map(executeAtom)); + + await flush(); + expect(logs).toStrictEqual(['move:folder/src/->folder/dst/']); + move.resolve(); + await flush(); + expect(logs).toStrictEqual(['move:folder/src/->folder/dst/', 'delete:folder/']); + + await pending; +}); + +test('Moves waits for destination creation', async () => { + const logs: Array = []; + const mkdir = deferred(); + const atoms: Array = [ + { + execute: () => { + logs.push('move:src/a.md->dst/a.md'); + }, + newKey: 'dst/a.md', + oldKey: 'src/a.md', + reject: () => {}, + resolve: () => {}, + type: 'move', + }, + { + execute: async () => { + logs.push('mkdir:dst/'); + await mkdir.promise; + }, + key: 'dst/', + reject: () => {}, + resolve: () => {}, + type: 'mkdir', + }, + ]; + const { executeAtom, optimized } = runOptimizer(atoms); + const pending = Promise.all(optimized.map(executeAtom)); + + await flush(); + expect(logs).toStrictEqual(['mkdir:dst/']); + mkdir.resolve(); + await flush(); + expect(logs).toStrictEqual(['mkdir:dst/', 'move:src/a.md->dst/a.md']); + await pending; +}); + test('folder deletion subsumes descendant deletions', async () => { const logs: Array = []; let childResolved = false; diff --git a/packages/plugin/test/retry-middleware.test.ts b/packages/plugin/test/retry-middleware.test.ts index 541020dc..1424bb7d 100644 --- a/packages/plugin/test/retry-middleware.test.ts +++ b/packages/plugin/test/retry-middleware.test.ts @@ -1,11 +1,10 @@ import testKit from '$/test-kit'; -// oxlint-disable-next-line import/no-namespace -import * as sleepModule from '@repo/shared/sleep'; import { expect, spyOn, test } from 'bun:test'; import { retryMiddleware } from '@/fs'; const { bytes, request } = testKit; -const sleepSpy = spyOn(sleepModule, 'default').mockImplementation(() => Promise.resolve()); +Object.assign(globalThis, { sleep: () => Promise.resolve() }); +const sleepSpy = spyOn(globalThis, 'sleep').mockImplementation(() => Promise.resolve()); test('retry middleware retries retryable request and waits between attempts', () => { sleepSpy.mockClear(); From b7cf44735d1e79a5fc6992e523a8b058bf4c0751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?He=CC=84sperus?= Date: Sun, 23 Aug 2026 11:24:21 +0800 Subject: [PATCH 6/7] refactor(gdrive): de-slop tests --- bun.lock | 1 + packages/gdrive/package.json | 3 +- packages/gdrive/test/auth.test.ts | 273 ++++--------- packages/gdrive/test/fs-gdrive.test.ts | 309 ++++---------- packages/gdrive/test/mock-drive.ts | 386 ------------------ packages/gdrive/test/mocks.ts | 2 +- packages/plugin/test/mocks.ts | 2 +- packages/plugin/test/retry-middleware.test.ts | 1 - packages/s3/test/mocks.ts | 2 +- .../shared/src/{obsidian-mock.ts => mocks.ts} | 7 + packages/smart-merge/test/mocks.ts | 2 +- packages/webdav/test/mocks.ts | 2 +- 12 files changed, 194 insertions(+), 796 deletions(-) delete mode 100644 packages/gdrive/test/mock-drive.ts rename packages/shared/src/{obsidian-mock.ts => mocks.ts} (87%) diff --git a/bun.lock b/bun.lock index cb721a66..257cc31e 100644 --- a/bun.lock +++ b/bun.lock @@ -50,6 +50,7 @@ "@hesprs/sync-engine-sdk": "workspace:*", "@repo/shared": "workspace:*", "hash-wasm": "^4.12.0", + "uni-kv": "../../uni-kv.tgz", }, }, "packages/i18n": { diff --git a/packages/gdrive/package.json b/packages/gdrive/package.json index f917f2cb..c3e4a1bf 100644 --- a/packages/gdrive/package.json +++ b/packages/gdrive/package.json @@ -25,6 +25,7 @@ "devDependencies": { "@hesprs/sync-engine-sdk": "workspace:*", "@repo/shared": "workspace:*", - "hash-wasm": "^4.12.0" + "hash-wasm": "^4.12.0", + "uni-kv": "../../uni-kv.tgz" } } diff --git a/packages/gdrive/test/auth.test.ts b/packages/gdrive/test/auth.test.ts index dcb9c478..dc9fb052 100644 --- a/packages/gdrive/test/auth.test.ts +++ b/packages/gdrive/test/auth.test.ts @@ -1,208 +1,115 @@ import type { Request, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; -import { expect, test } from 'bun:test'; -import type { AuthHttp } from '@/gdrive/auth'; -import { - TokenManager, - bearerMiddleware, - decodeIdTokenEmail, - pollDeviceToken, - startDeviceAuthorization, -} from '@/gdrive/auth'; +import type { SecretStorage } from 'obsidian'; +import * as ObsidianMock from '@repo/shared/mocks'; +import { expect, mock, test } from 'bun:test'; -type AuthResponse = { status: number; body: unknown }; +type HttpResponse = { json: unknown; status?: number }; +const requests: Array = []; +let responses: Array = []; -function scriptedHttp(responses: Array) { - const calls: Array<{ url: string; body: string }> = []; - const http: AuthHttp = ({ url, body }) => { - calls.push({ body: body ?? '', url }); - const next = responses.shift(); - if (!next) throw new Error('scripted http exhausted'); - return Promise.resolve({ json: () => next.body, status: next.status }); - }; - return { calls, http }; -} +void mock.module('obsidian', () => ({ + ...ObsidianMock, + requestUrl: (params: RequestParam) => { + requests.push(params); + const response = responses.shift(); + if (!response) throw new Error('Unexpected request'); + return Promise.resolve({ json: response.json, status: response.status ?? 200 }); + }, +})); -function fakeIdToken(email: string): string { - const payload = btoa(JSON.stringify({ email })) - .replaceAll('+', '-') - .replaceAll('/', '_') - .replaceAll('=', ''); - return `header.${payload}.signature`; -} +const { TokenManager, bearerMiddleware, pollDeviceToken, startDeviceAuthorization } = + await import('@/gdrive/auth'); -const AUTHORIZATION = { - deviceCode: 'device-1', - expiresIn: 1800, - interval: 5, - userCode: 'ABCD-EFGH', - verificationUrl: 'https://www.google.com/device', -}; +function reset(...next: Array) { + requests.length = 0; + responses = [...next]; +} -test('startDeviceAuthorization parses the device code response', async () => { - const { calls, http } = scriptedHttp([ - { - body: { - device_code: 'device-1', - expires_in: 900, - interval: 7, - user_code: 'ABCD-EFGH', - verification_url: 'https://www.google.com/device', - }, - status: 200, +test('starts device authorization from Google response', async () => { + reset({ + json: { + device_code: 'device', + expires_in: 900, + interval: 0, + user_code: 'ABCD', + verification_url: 'https://google.test/device', }, - ]); - const authorization = await startDeviceAuthorization(http, 'client-1'); - expect(authorization).toStrictEqual({ - deviceCode: 'device-1', - expiresIn: 900, - interval: 7, - userCode: 'ABCD-EFGH', - verificationUrl: 'https://www.google.com/device', }); - expect(calls[0]?.body).toContain('client_id=client-1'); - expect(calls[0]?.body).toContain('drive.file'); -}); -test('startDeviceAuthorization surfaces Google error descriptions', async () => { - const { http } = scriptedHttp([ - { body: { error: 'invalid_client', error_description: 'Unknown client.' }, status: 401 }, - ]); - await expect(startDeviceAuthorization(http, 'client-1')).rejects.toThrow('Unknown client.'); + expect(await startDeviceAuthorization()).toStrictEqual({ + deviceCode: 'device', + expiresIn: 900, + interval: 0, + userCode: 'ABCD', + verificationUrl: 'https://google.test/device', + }); + expect(requests[0]?.method).toBe('POST'); }); -test('pollDeviceToken waits through pending, honors slow_down, and resolves tokens', async () => { - const { calls, http } = scriptedHttp([ - { body: { error: 'authorization_pending' }, status: 428 }, - { body: { error: 'slow_down' }, status: 428 }, - { - body: { - access_token: 'access-1', - expires_in: 3599, - id_token: fakeIdToken('user@example.com'), - refresh_token: 'refresh-1', - }, - status: 200, - }, - ]); - const sleeps: Array = []; - const result = await pollDeviceToken(http, { - authorization: AUTHORIZATION, - clientId: 'client-1', - clientSecret: 'secret-1', - sleep: (ms) => { - sleeps.push(ms); - return Promise.resolve(); +test('polls device authorization and extracts user id from ID token', async () => { + const payload = btoa(JSON.stringify({ sub: 'google-user' })) + .replaceAll('+', '-') + .replaceAll('/', '_') + .replaceAll('=', ''); + reset({ + json: { + access_token: 'access', + expires_in: 3600, + id_token: `header.${payload}.signature`, + refresh_token: 'refresh', }, }); - expect(result).toStrictEqual({ - accessToken: 'access-1', - email: 'user@example.com', - expiresIn: 3599, - refreshToken: 'refresh-1', - }); - expect(sleeps).toStrictEqual([5000, 5000, 10_000]); - expect(calls[0]?.body).toContain( - 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code', - ); -}); -test('pollDeviceToken maps denial, expiry, and cancellation to clear errors', async () => { - const denied = scriptedHttp([{ body: { error: 'access_denied' }, status: 403 }]); - await expect( - pollDeviceToken(denied.http, { - authorization: AUTHORIZATION, - clientId: 'c', - clientSecret: 's', - sleep: () => Promise.resolve(), - }), - ).rejects.toThrow('denied'); - - const expired = scriptedHttp([{ body: { error: 'expired_token' }, status: 428 }]); - await expect( - pollDeviceToken(expired.http, { - authorization: AUTHORIZATION, - clientId: 'c', - clientSecret: 's', - sleep: () => Promise.resolve(), - }), - ).rejects.toThrow('expired'); - - const cancelled = scriptedHttp([]); - await expect( - pollDeviceToken(cancelled.http, { - authorization: AUTHORIZATION, - clientId: 'c', - clientSecret: 's', - isCancelled: () => true, - sleep: () => Promise.resolve(), + expect( + await pollDeviceToken({ + authorization: { + deviceCode: 'device', + expiresIn: 60, + interval: 0, + userCode: 'code', + verificationUrl: 'url', + }, + isCancelled: () => false, }), - ).rejects.toThrow('cancelled'); + ).toStrictEqual({ + accessToken: 'access', + expiresIn: 3600, + refreshToken: 'refresh', + userId: 'google-user', + }); }); -test('TokenManager caches tokens, refreshes on expiry, and dedupes concurrent refreshes', async () => { - let clock = 0; - const { calls, http } = scriptedHttp([ - { body: { access_token: 'token-a', expires_in: 3600 }, status: 200 }, - { body: { access_token: 'token-b', expires_in: 3600 }, status: 200 }, - ]); - const manager = new TokenManager( - http, - () => ({ clientId: 'c', clientSecret: 's', refreshToken: 'r' }), - () => clock, +test('caches tokens and retries bearer requests after a 401', async () => { + reset( + { json: { access_token: 'first', expires_in: 3600 } }, + { json: { access_token: 'second', expires_in: 3600 } }, ); - const [first, second] = await Promise.all([manager.getToken(), manager.getToken()]); - expect(first).toBe('token-a'); - expect(second).toBe('token-a'); - expect(calls.length).toBe(1); - expect(await manager.getToken()).toBe('token-a'); - clock = 3600 * 1000; // Past expiry minus the safety margin. - expect(await manager.getToken()).toBe('token-b'); - expect(calls.length).toBe(2); - expect(calls[0]?.body).toContain('grant_type=refresh_token'); -}); - -test('TokenManager reports revoked authorization clearly', async () => { - const { http } = scriptedHttp([{ body: { error: 'invalid_grant' }, status: 400 }]); - const manager = new TokenManager(http, () => ({ - clientId: 'c', - clientSecret: 's', - refreshToken: 'r', - })); - await expect(manager.getToken()).rejects.toThrow('reconnect'); -}); - -test('bearerMiddleware injects the token and retries once after a 401', async () => { - const tokenHttp = scriptedHttp([ - { body: { access_token: 'stale', expires_in: 3600 }, status: 200 }, - { body: { access_token: 'fresh', expires_in: 3600 }, status: 200 }, - ]); - const manager = new TokenManager(tokenHttp.http, () => ({ - clientId: 'c', - clientSecret: 's', - refreshToken: 'r', - })); - const seenAuth: Array = []; - const inner: Request = (params: RequestParam | string) => { - if (typeof params === 'string') throw new Error('unexpected string request'); - seenAuth.push(params.headers?.Authorization); - const status = seenAuth.length === 1 ? 401 : 200; - const response: RequestResponse = { + const secrets = new Map([['sync-engine-gdrive-refresh-token', 'refresh']]); + const storage = { + deleteSecret: (id: string) => void secrets.delete(id), + getSecret: (id: string) => secrets.get(id), + setSecret: (id: string, value: string) => void secrets.set(id, value), + }; + const manager = new TokenManager(storage as unknown as SecretStorage); + const seen: Array = []; + const request: Request = (params) => { + if (typeof params === 'string') throw new Error('Unexpected string request'); + seen.push(params.headers?.Authorization); + if (seen.length === 1) { + const error = new Error('Unauthorized') as Error & { status: number }; + error.status = 401; + return Promise.reject(error); + } + return Promise.resolve({ bytes: () => new Uint8Array(0), headers: {}, json: () => ({}), - status, + status: 200, text: () => '', - }; - return Promise.resolve(response); + } satisfies RequestResponse); }; - const request = bearerMiddleware(inner, manager); - const response = await request({ method: 'GET', url: 'https://example.com' }); - expect(response.status).toBe(200); - expect(seenAuth).toStrictEqual(['Bearer stale', 'Bearer fresh']); -}); -test('decodeIdTokenEmail tolerates malformed tokens', () => { - expect(decodeIdTokenEmail(fakeIdToken('a@b.c'))).toBe('a@b.c'); - expect(decodeIdTokenEmail('garbage')).toBeUndefined(); - expect(decodeIdTokenEmail('a.b.c')).toBeUndefined(); + const wrapped = bearerMiddleware(request, manager); + expect((await wrapped({ method: 'GET', url: 'https://drive.test' })).status).toBe(200); + expect(seen).toStrictEqual(['Bearer first', 'Bearer second']); }); diff --git a/packages/gdrive/test/fs-gdrive.test.ts b/packages/gdrive/test/fs-gdrive.test.ts index 682264ae..ff71d0cd 100644 --- a/packages/gdrive/test/fs-gdrive.test.ts +++ b/packages/gdrive/test/fs-gdrive.test.ts @@ -1,240 +1,109 @@ -import type { Binary, ListReporter, RootFs, Stat } from '@hesprs/sync-engine-sdk'; +import type { Binary, Request, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; import { testKit } from '@hesprs/sync-engine-sdk/dev'; -import { expect, test } from 'bun:test'; -import { md5 } from 'hash-wasm'; -import checkConnection from '@/gdrive/check-connection'; +import { beforeEach, expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { GdriveDB } from '@/gdrive/fs'; +import { DRIVE_API, DRIVE_UPLOAD_API, FOLDER_MIME } from '@/gdrive/api'; import GdriveFs from '@/gdrive/fs'; -import { RESUMABLE_CHUNK_SIZE } from '@/gdrive/upload'; -import { MockDrive, jsonResponse } from './mock-drive'; -const { bytes, file, stream: createStream } = testKit; - -const includeAll: ListReporter = () => 'advance'; - -function createFs(options: { baseDirectory?: string; useTrash?: boolean } = {}) { - const drive = new MockDrive(); - const fs: RootFs = new GdriveFs({ - account: 'mock@example.com', - baseDirectory: options.baseDirectory ?? 'Vault/Notes/', - request: drive.request, - useTrash: options.useTrash ?? true, - }); - return { drive, fs }; +const { bytes, file } = testKit; +const db: GdriveDB = openMemoryDB<{ gdriveIds: string }, { gdriveIdsMarker?: string }>( + 'gdrive-fs-test', +); + +function response( + value: unknown = {}, + status = 200, + headers: Record = {}, +): RequestResponse { + const body = new TextEncoder().encode(JSON.stringify(value)); + return { + bytes: () => body, + headers, + json: () => value, + status, + text: () => new TextDecoder().decode(body), + }; } -async function collect(source: ReadableStream): Promise { - const reader = source.getReader(); - const chunks: Array = []; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - } - reader.releaseLock(); - let total = 0; - for (const chunk of chunks) total += chunk.byteLength; - const merged = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - merged.set(chunk, offset); - offset += chunk.byteLength; - } - return new TextDecoder().decode(merged); +function binaryResponse(value: Binary, status = 200): RequestResponse { + return { ...response({}, status), bytes: () => value }; } -test('write creates the base directory chain and read round-trips content', async () => { - const { drive, fs } = createFs(); - const mtime = 1_700_000_000_000; - const uid = await fs.write('a.md', bytes('hello'), file('a.md', { mtime, size: 5 })); - expect(uid).toBe(await md5('hello')); - expect(drive.contentByPath('Vault/Notes/a.md')).toBe('hello'); - expect(drive.fileByPath('Vault/Notes/a.md')?.modifiedTime).toBe(new Date(mtime).toISOString()); - expect(new TextDecoder().decode(await fs.read('a.md', file('a.md')))).toBe('hello'); -}); - -test('write to an existing key updates the same Drive file in place', async () => { - const { drive, fs } = createFs(); - await fs.write('note.md', bytes('one'), file('note.md', { mtime: 1000, size: 3 })); - const firstId = drive.fileByPath('Vault/Notes/note.md')?.id; - await fs.write('note.md', bytes('two!'), file('note.md', { mtime: 2000, size: 4 })); - const updated = drive.fileByPath('Vault/Notes/note.md'); - expect(updated?.id).toBe(firstId ?? ''); - expect(drive.contentByPath('Vault/Notes/note.md')).toBe('two!'); - expect(updated?.modifiedTime).toBe(new Date(2000).toISOString()); -}); - -test('keys with apostrophes survive the query escaping round trip', async () => { - const { drive, fs } = createFs(); - await fs.write("it's ok.md", bytes('quoted'), file("it's ok.md", { size: 6 })); - expect(drive.contentByPath("Vault/Notes/it's ok.md")).toBe('quoted'); - expect(new TextDecoder().decode(await fs.read("it's ok.md", file("it's ok.md")))).toBe( - 'quoted', - ); -}); - -test('mkdir builds nested folders and stat and exists see them', async () => { - const { drive, fs } = createFs(); - await fs.mkdir('x/y/', true); - expect(drive.fileByPath('Vault/Notes/x/y')?.mimeType).toBe( - 'application/vnd.google-apps.folder', - ); - expect(await fs.stat('x/y/')).toStrictEqual({ isDir: true, key: 'x/y/' }); - expect(await fs.exists('x/')).toBe(true); - expect(await fs.exists('missing/')).toBe(false); - expect(await fs.exists('missing.md')).toBe(false); - await expect(fs.stat('missing.md')).rejects.toMatchObject({ status: 404 }); - await fs.mkdir('x/y/', true); // Idempotent - expect(await fs.exists('x/y/')).toBe(true); -}); - -test('stat returns md5 uid, size, and preserved mtime for files', async () => { - const { fs } = createFs(); - const mtime = 1_600_000_000_000; - await fs.write('s.md', bytes('stats'), file('s.md', { mtime, size: 5 })); - const stat = await fs.stat('s.md'); - expect(stat).toStrictEqual({ - isDir: false, - key: 's.md', - mtime, - size: 5, - uid: await md5('stats'), - }); -}); - -test('list walks the tree, honors reporter verdicts, and paginates', async () => { - const { fs } = createFs(); - await fs.write('a.md', bytes('a'), file('a.md', { size: 1 })); - await fs.write('excluded.md', bytes('x'), file('excluded.md', { size: 1 })); - await fs.write('sub/b.md', bytes('b'), file('sub/b.md', { size: 1 })); - await fs.write('skip/c.md', bytes('c'), file('skip/c.md', { size: 1 })); - const seen: Array = []; - const reporter: ListReporter = ({ current }) => { - seen.push(current); - if (current === 'excluded.md') return 'exclude'; - if (current === 'skip/') return 'include'; - if (current.endsWith('/')) return 'advance'; - return 'include'; +function createFs(handler: (params: RequestParam) => RequestResponse | Promise) { + const calls: Array = []; + const request: Request = (params) => { + if (typeof params === 'string') throw new Error('Unexpected string request'); + calls.push(params); + return Promise.resolve(handler(params)); }; - const results = await fs.list('/', reporter); - const keys = results.map((stat: Stat) => stat.key).sort(); - expect(keys).toStrictEqual(['a.md', 'skip/', 'sub/', 'sub/b.md']); - expect(seen).toContain('excluded.md'); - expect(seen).not.toContain('skip/c.md'); - const fileStat = results.find((stat: Stat) => stat.key === 'sub/b.md'); - expect(fileStat?.isDir).toBe(false); - if (fileStat?.isDir === false) expect(fileStat.uid).toBe(await md5('b')); -}); - -test('list from a subfolder returns keys prefixed with that folder', async () => { - const { fs } = createFs(); - await fs.write('sub/deep/d.md', bytes('d'), file('sub/deep/d.md', { size: 1 })); - const results = await fs.list('sub/', includeAll); - const keys = results.map((stat: Stat) => stat.key).sort(); - expect(keys).toStrictEqual(['sub/deep/', 'sub/deep/d.md']); -}); - -test('move renames files, relocates them between folders, and moves folders whole', async () => { - const { drive, fs } = createFs(); - await fs.write('a.md', bytes('a'), file('a.md', { size: 1 })); - await fs.move('a.md', 'renamed.md'); - expect(drive.contentByPath('Vault/Notes/renamed.md')).toBe('a'); - expect(drive.fileByPath('Vault/Notes/a.md')).toBeUndefined(); - expect(await fs.exists('a.md')).toBe(false); - expect(new TextDecoder().decode(await fs.read('renamed.md', file('renamed.md')))).toBe('a'); - - await fs.move('renamed.md', 'other/renamed.md'); - expect(drive.contentByPath('Vault/Notes/other/renamed.md')).toBe('a'); + return { calls, fs: new GdriveFs(request, { useTrash: true, userId: 'user-1' }, db) }; +} - await fs.write('sub/b.md', bytes('b'), file('sub/b.md', { size: 1 })); - await fs.move('sub/', 'moved/'); - expect(drive.contentByPath('Vault/Notes/moved/b.md')).toBe('b'); - expect(new TextDecoder().decode(await fs.read('moved/b.md', file('moved/b.md')))).toBe('b'); +beforeEach(() => { + db.clearStores(); + db.setMeta('gdriveIdsMarker', undefined); }); -test('delete trashes by default, is idempotent, and can delete permanently', async () => { - const trashing = createFs(); - await trashing.fs.write('t.md', bytes('t'), file('t.md', { size: 1 })); - await trashing.fs.delete('t.md'); - const trashed = [...trashing.drive.files.values()].find((entry) => entry.name === 't.md'); - expect(trashed?.trashed).toBe(true); - expect(await trashing.fs.exists('t.md')).toBe(false); - await trashing.fs.delete('t.md'); // Missing → silently succeeds - await trashing.fs.delete('never-existed.md'); - - const permanent = createFs({ useTrash: false }); - await permanent.fs.write('p.md', bytes('p'), file('p.md', { size: 1 })); - await permanent.fs.delete('p.md'); - expect( - [...permanent.drive.files.values()].find((entry) => entry.name === 'p.md'), - ).toBeUndefined(); -}); +test('writes and reads a file through Drive resumable upload', async () => { + const { calls, fs } = createFs((params) => { + if (params.url.startsWith(DRIVE_UPLOAD_API) && params.method === 'POST') + return response({}, 200, { Location: 'https://upload.example/session' }); + if (params.url === 'https://upload.example/session') + return response({ id: 'file-1', md5Checksum: 'drive-uid' }); + if (params.url === `${DRIVE_API}/files/file-1?alt=media`) + return binaryResponse(bytes('hello')); + throw new Error(`Unexpected request: ${params.method} ${params.url}`); + }); -test('readStream assembles ranged chunks in order', async () => { - const { fs } = createFs(); - await fs.write('r.md', bytes('ranged content'), file('r.md', { size: 14 })); - const result = await fs.readStream('r.md', file('r.md', { size: 14 })); - expect(await collect(result)).toBe('ranged content'); + const stat = file('note.md', { mtime: 1_700_000_000_000, size: 5 }); + expect(await fs.write('note.md', bytes('hello'), stat)).toBe('drive-uid'); + expect(await fs.read('note.md')).toStrictEqual(bytes('hello')); + expect(calls.map(({ method }) => method)).toStrictEqual(['POST', 'PUT', 'GET']); + expect(calls[0]?.url).toContain('uploadType=resumable'); }); -test('writeStream below the threshold uses one multipart upload', async () => { - const { drive, fs } = createFs(); - const uid = await fs.writeStream( - 'small.md', - createStream(['hello ', 'stream']), - file('small.md', { mtime: 3000, size: 12 }), - ); - expect(drive.contentByPath('Vault/Notes/small.md')).toBe('hello stream'); - expect(uid).toBe(await md5('hello stream')); - const uploadCalls = drive.requestLog.filter( - (params) => typeof params !== 'string' && params.url.includes('uploadType=multipart'), - ); - expect(uploadCalls.length).toBe(1); -}); +test('creates folders, lists visible descendants, and honors excluded subtrees', async () => { + const { calls, fs } = createFs((params) => { + if (params.method === 'POST' && params.url.startsWith(`${DRIVE_API}/files`)) + return response({ id: 'folder-1' }); + return response({ + files: [ + { id: 'folder-1', mimeType: FOLDER_MIME, name: 'notes', parents: ['root'] }, + { + id: 'file-1', + md5Checksum: 'uid', + mimeType: 'text/markdown', + modifiedTime: new Date(1000).toISOString(), + name: 'note.md', + parents: ['folder-1'], + size: '5', + }, + ], + }); + }); -test('writeStream at the threshold uses a chunked resumable session', async () => { - const { drive, fs } = createFs(); - const big = new Uint8Array(RESUMABLE_CHUNK_SIZE + 3).fill(97); - const uid = await fs.writeStream( - 'big.bin', - createStream([big]), - file('big.bin', { mtime: 4000, size: big.byteLength }), - ); - const stored = drive.fileByPath('Vault/Notes/big.bin'); - expect(stored?.content?.byteLength).toBe(big.byteLength); - expect(uid).toBe(await md5(big)); - const sessionPuts = drive.requestLog.filter( - (params) => - typeof params !== 'string' && - params.method === 'PUT' && - params.url.includes('/mock-session/'), + await fs.mkdir('notes/', true); + const result = await fs.list('/', ({ current }) => + current === 'notes/' ? 'include' : 'advance', ); - expect(sessionPuts.length).toBe(2); -}); - -test('duplicate names in one folder resolve to the newest file', async () => { - const { drive, fs } = createFs(); - await fs.mkdir('/', true); - const base = drive.fileByPath('Vault/Notes'); - drive.addFile('dup.md', base?.id ?? '', 'old', new Date(1000).toISOString()); - drive.addFile('dup.md', base?.id ?? '', 'new', new Date(2000).toISOString()); - expect(new TextDecoder().decode(await fs.read('dup.md', file('dup.md')))).toBe('new'); - const results = await fs.list('/', includeAll); - expect(results.filter((stat: Stat) => stat.key === 'dup.md').length).toBe(1); + expect(result).toStrictEqual([{ isDir: true, key: 'notes/' }]); + expect(calls[0]?.method).toBe('POST'); + expect(new TextDecoder().decode(calls[0]?.body as Binary)).toContain(FOLDER_MIME); }); -test('getUid identifies the account and base directory', () => { - const { fs } = createFs(); - expect(fs.getUid()).toBe('gdrive~mock@example.com~Vault/Notes/'); -}); - -test('checkConnection reports success and surfaces Drive errors', async () => { - const { drive } = createFs(); - expect(await checkConnection(drive.request)).toStrictEqual({ success: true }); - drive.failNext = jsonResponse(401, { - error: { code: 401, message: 'Invalid Credentials' }, - }); - expect(await checkConnection(drive.request)).toStrictEqual({ - reason: 'Google Drive 401: Invalid Credentials', - success: false, +test('moves a cached file with Drive native rename', async () => { + const { calls, fs } = createFs((params) => { + if (params.method === 'POST' && params.url.startsWith(DRIVE_UPLOAD_API)) + return response({}, 200, { location: 'https://upload.example/session' }); + if (params.url === 'https://upload.example/session') return response({ id: 'file-1' }); + if (params.method === 'PATCH') return response({ id: 'file-1' }); + throw new Error(`Unexpected request: ${params.method} ${params.url}`); }); + + await fs.write('old.md', bytes('x'), file('old.md', { size: 1 })); + await fs.move('old.md', 'new.md'); + const move = calls.find((call) => call.method === 'PATCH'); + expect(move?.url).toContain('/files/file-1'); + expect(new TextDecoder().decode(move?.body as Binary)).toBe('{"name":"new.md"}'); }); diff --git a/packages/gdrive/test/mock-drive.ts b/packages/gdrive/test/mock-drive.ts deleted file mode 100644 index 50597315..00000000 --- a/packages/gdrive/test/mock-drive.ts +++ /dev/null @@ -1,386 +0,0 @@ -import type { Binary, Request, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; -import { md5 } from 'hash-wasm'; - -export type MockFile = { - id: string; - name: string; - mimeType: string; - parents: Array; - content?: Uint8Array; - modifiedTime: string; - trashed: boolean; -}; - -const FOLDER_MIME = 'application/vnd.google-apps.folder'; -const encoder = new TextEncoder(); -const decoder = new TextDecoder(); -const latin1 = new TextDecoder('latin1'); - -export function jsonResponse(status: number, value: unknown): RequestResponse { - const text = JSON.stringify(value); - return { - bytes: () => encoder.encode(text), - headers: { 'content-type': 'application/json' }, - json: () => JSON.parse(text) as unknown, - status, - text: () => text, - }; -} - -function bytesResponse(status: number, content: Uint8Array): RequestResponse { - return { - bytes: () => new Uint8Array(content), - headers: { 'content-type': 'application/octet-stream' }, - json: () => JSON.parse(decoder.decode(content)) as unknown, - status, - text: () => decoder.decode(content), - }; -} - -function emptyResponse(status: number, headers: Record = {}): RequestResponse { - return { - bytes: () => new Uint8Array(0), - headers, - json: () => ({}), - status, - text: () => '', - }; -} - -function notFound(): RequestResponse { - return jsonResponse(404, { error: { code: 404, message: 'File not found.' } }); -} - -function unescapeQueryLiteral(value: string): string { - return value.replaceAll(/\\(?['\\])/gu, '$'); -} - -function toBinary(body: RequestParam['body']): Uint8Array { - if (body === undefined) return new Uint8Array(0); - if (typeof body === 'string') return encoder.encode(body); - return body; -} - -type MultipartParts = { metadata: Record; content: Uint8Array }; - -/** Latin1 decoding maps one byte to one char, so string indexes equal byte offsets. */ -function parseMultipart(body: Uint8Array, contentType: string): MultipartParts { - const boundary = contentType.split('boundary=')[1]; - if (!boundary) throw new Error('mock: missing multipart boundary'); - const text = latin1.decode(body); - const delimiter = `--${boundary}`; - const firstHeaderEnd = text.indexOf('\r\n\r\n'); - const secondDelimiter = text.indexOf(delimiter, firstHeaderEnd); - const metadataText = text.slice(firstHeaderEnd + 4, secondDelimiter); - const secondHeaderEnd = text.indexOf('\r\n\r\n', secondDelimiter); - const closingDelimiter = text.lastIndexOf(`\r\n${delimiter}--`); - const metadata = JSON.parse(metadataText.trimEnd()) as Record; - const content = body.slice(secondHeaderEnd + 4, closingDelimiter); - return { content, metadata }; -} - -type ResumableSession = { - targetId?: string; - metadata: Record; - received: Array; -}; - -async function serialize(file: MockFile): Promise> { - return { - id: file.id, - md5Checksum: - file.mimeType === FOLDER_MIME || file.content === undefined - ? undefined - : await md5(file.content), - mimeType: file.mimeType, - modifiedTime: file.modifiedTime, - name: file.name, - parents: file.parents, - size: file.content === undefined ? undefined : String(file.content.byteLength), - }; -} - -/** - * In-memory Google Drive REST v3 covering the subset the module uses: files - * lookup/list queries, media downloads with ranges, multipart and resumable - * uploads, metadata patches, deletes, `files/root`, and `about`. - */ -export class MockDrive { - readonly files = new Map(); - readonly rootId = 'root-id-0001'; - requestLog: Array = []; - failNext: RequestResponse | undefined; - private idCounter = 0; - private readonly sessions = new Map(); - - readonly request: Request = (params) => { - const normalized: RequestParam = typeof params === 'string' ? { url: params } : params; - this.requestLog.push(normalized); - if (this.failNext) { - const response = this.failNext; - this.failNext = undefined; - return Promise.resolve(response); - } - return this.route(normalized); - }; - - addFolder(name: string, parentId: string): MockFile { - const folder: MockFile = { - id: this.nextId(), - mimeType: FOLDER_MIME, - modifiedTime: new Date(0).toISOString(), - name, - parents: [parentId], - trashed: false, - }; - this.files.set(folder.id, folder); - return folder; - } - - addFile( - name: string, - parentId: string, - content: string, - modifiedTime = new Date(0).toISOString(), - ): MockFile { - const file: MockFile = { - content: encoder.encode(content), - id: this.nextId(), - mimeType: 'text/plain', - modifiedTime, - name, - parents: [parentId], - trashed: false, - }; - this.files.set(file.id, file); - return file; - } - - fileByPath(path: string): MockFile | undefined { - let parentId = this.rootId; - const segments = path.split('/').filter((segment) => segment !== ''); - let current: MockFile | undefined; - for (const segment of segments) { - current = [...this.files.values()].find( - (file) => !file.trashed && file.parents[0] === parentId && file.name === segment, - ); - if (!current) return undefined; - parentId = current.id; - } - return current; - } - - contentByPath(path: string): string | undefined { - const file = this.fileByPath(path); - return file?.content === undefined ? undefined : decoder.decode(file.content); - } - - private nextId(): string { - this.idCounter++; - return `id-${this.idCounter.toString().padStart(4, '0')}`; - } - - private async route(params: RequestParam): Promise { - const url = new URL(params.url); - const method = params.method ?? 'GET'; - const path = url.pathname; - if (path === '/token' || url.host === 'oauth2.googleapis.com') - throw new Error(`mock: unexpected OAuth call ${params.url}`); - if (path.startsWith('/mock-session/')) return this.routeSession(params, url); - if (path === '/drive/v3/about') - return jsonResponse(200, { user: { emailAddress: 'mock@example.com' } }); - if (path === '/drive/v3/files/root') return jsonResponse(200, { id: this.rootId }); - if (path === '/drive/v3/files' && method === 'GET') return this.routeQuery(url); - if (path === '/drive/v3/files' && method === 'POST') - return this.routeCreateMetadata(params); - if (path === '/upload/drive/v3/files' && method === 'POST') - return this.routeUpload(params, url); - const uploadMatch = /^\/upload\/drive\/v3\/files\/(?[^/]+)$/u.exec(path); - if (uploadMatch && method === 'PATCH') - return this.routeUpload(params, url, uploadMatch.groups?.id); - const fileMatch = /^\/drive\/v3\/files\/(?[^/]+)$/u.exec(path); - if (fileMatch) return this.routeFile(params, url, fileMatch.groups?.id ?? '', method); - throw new Error(`mock: unhandled route ${method} ${params.url}`); - } - - private async routeQuery(url: URL): Promise { - const q = url.searchParams.get('q') ?? ''; - const lookup = - /^'(?.+)' in parents and name = '(?.*)' and trashed = false and mimeType (?=|!=) '(?.+)'$/u.exec( - q, - ); - if (lookup?.groups) { - const parent = unescapeQueryLiteral(lookup.groups.parent ?? ''); - const name = unescapeQueryLiteral(lookup.groups.name ?? ''); - const wantFolder = lookup.groups.op === '='; - const matches = [...this.files.values()] - .filter( - (file) => - !file.trashed && - file.parents[0] === (parent === 'root' ? this.rootId : parent) && - file.name === name && - (file.mimeType === FOLDER_MIME) === wantFolder, - ) - .sort((a, b) => Date.parse(b.modifiedTime) - Date.parse(a.modifiedTime)); - return jsonResponse(200, { - files: await Promise.all(matches.map((file) => serialize(file))), - }); - } - if (q === 'trashed = false') { - const pageSize = 2; // Force pagination so tests cover it. - const all = [...this.files.values()].filter((file) => !file.trashed); - const start = Number.parseInt(url.searchParams.get('pageToken') ?? '0'); - const page = all.slice(start, start + pageSize); - const nextIndex = start + pageSize; - return jsonResponse(200, { - files: await Promise.all(page.map((file) => serialize(file))), - nextPageToken: nextIndex < all.length ? String(nextIndex) : undefined, - }); - } - throw new Error(`mock: unhandled query ${q}`); - } - - private async routeCreateMetadata(params: RequestParam): Promise { - const metadata = JSON.parse(decoder.decode(toBinary(params.body))) as { - mimeType?: string; - name?: string; - parents?: Array; - }; - if (metadata.mimeType !== FOLDER_MIME) - throw new Error('mock: metadata-only create supports folders only'); - const folder = this.addFolder(metadata.name ?? '', metadata.parents?.[0] ?? this.rootId); - return jsonResponse(200, await serialize(folder)); - } - - private async routeUpload( - params: RequestParam, - url: URL, - targetId?: string, - ): Promise { - const uploadType = url.searchParams.get('uploadType'); - if (uploadType === 'multipart') { - const contentType = params.headers?.['Content-Type'] ?? ''; - const { metadata, content } = parseMultipart(toBinary(params.body), contentType); - return jsonResponse( - 200, - await serialize(this.applyUpload(targetId, metadata, content)), - ); - } - if (uploadType === 'resumable') { - const metadata = JSON.parse(decoder.decode(toBinary(params.body))) as Record< - string, - unknown - >; - const sessionId = `session-${this.sessions.size + 1}`; - this.sessions.set(sessionId, { metadata, received: [], targetId }); - return emptyResponse(200, { - location: `https://mock.googleapis.com/mock-session/${sessionId}`, - }); - } - throw new Error(`mock: unhandled upload type ${uploadType ?? 'none'}`); - } - - private async routeSession(params: RequestParam, url: URL): Promise { - const sessionId = url.pathname.split('/').pop() ?? ''; - const session = this.sessions.get(sessionId); - if (!session) return notFound(); - if (params.method === 'DELETE') { - this.sessions.delete(sessionId); - return emptyResponse(204); - } - const range = /^bytes (?\d+)-(?\d+)\/(?\d+|\*)$/u.exec( - params.headers?.['Content-Range'] ?? '', - ); - if (!range?.groups) throw new Error('mock: resumable PUT without Content-Range'); - session.received.push(toBinary(params.body)); - const receivedBytes = session.received.reduce((sum, chunk) => sum + chunk.byteLength, 0); - const end = Number.parseInt(range.groups.end ?? '0'); - if (receivedBytes < end + 1) throw new Error('mock: resumable chunks out of order'); - const total = range.groups.total; - if (total !== '*' && receivedBytes >= Number.parseInt(total)) { - const content = new Uint8Array(receivedBytes); - let offset = 0; - for (const chunk of session.received) { - content.set(chunk, offset); - offset += chunk.byteLength; - } - this.sessions.delete(sessionId); - return jsonResponse( - 200, - await serialize(this.applyUpload(session.targetId, session.metadata, content)), - ); - } - return emptyResponse(308); - } - - private applyUpload( - targetId: string | undefined, - metadata: Record, - content: Uint8Array, - ): MockFile { - if (targetId !== undefined) { - const existing = this.files.get(targetId); - if (!existing) throw new Error(`mock: upload to missing file ${targetId}`); - existing.content = content; - if (typeof metadata.modifiedTime === 'string') - existing.modifiedTime = metadata.modifiedTime; - return existing; - } - const file: MockFile = { - content, - id: this.nextId(), - mimeType: typeof metadata.mimeType === 'string' ? metadata.mimeType : 'text/plain', - modifiedTime: - typeof metadata.modifiedTime === 'string' - ? metadata.modifiedTime - : new Date(0).toISOString(), - name: typeof metadata.name === 'string' ? metadata.name : '', - parents: Array.isArray(metadata.parents) ? (metadata.parents as Array) : [], - trashed: false, - }; - this.files.set(file.id, file); - return file; - } - - private async routeFile( - params: RequestParam, - url: URL, - id: string, - method: string, - ): Promise { - const file = this.files.get(id); - if (!file || (file.trashed && method !== 'DELETE')) return notFound(); - if (method === 'GET' && url.searchParams.get('alt') === 'media') { - const content = file.content ?? new Uint8Array(0); - const range = /^bytes=(?\d+)-(?\d+)$/u.exec(params.headers?.Range ?? ''); - if (range?.groups) { - const start = Number.parseInt(range.groups.start ?? '0'); - const end = Number.parseInt(range.groups.end ?? '0'); - return bytesResponse(206, content.slice(start, end + 1)); - } - return bytesResponse(200, content); - } - if (method === 'GET') return jsonResponse(200, await serialize(file)); - if (method === 'DELETE') { - this.files.delete(id); - return emptyResponse(204); - } - if (method === 'PATCH') { - const metadata = JSON.parse(decoder.decode(toBinary(params.body))) as { - name?: string; - trashed?: boolean; - }; - if (typeof metadata.name === 'string') file.name = metadata.name; - if (metadata.trashed === true) file.trashed = true; - const addParents = url.searchParams.get('addParents'); - const removeParents = url.searchParams.get('removeParents'); - if (addParents) - file.parents = [ - addParents, - ...file.parents.filter((parent) => parent !== removeParents), - ]; - return jsonResponse(200, await serialize(file)); - } - throw new Error(`mock: unhandled file route ${method} ${params.url}`); - } -} diff --git a/packages/gdrive/test/mocks.ts b/packages/gdrive/test/mocks.ts index ed25d7f1..d612befb 100644 --- a/packages/gdrive/test/mocks.ts +++ b/packages/gdrive/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/plugin/test/mocks.ts b/packages/plugin/test/mocks.ts index edec474c..bf6976b0 100644 --- a/packages/plugin/test/mocks.ts +++ b/packages/plugin/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; Object.assign(globalThis, { window: globalThis }); diff --git a/packages/plugin/test/retry-middleware.test.ts b/packages/plugin/test/retry-middleware.test.ts index 1424bb7d..a6f3f694 100644 --- a/packages/plugin/test/retry-middleware.test.ts +++ b/packages/plugin/test/retry-middleware.test.ts @@ -3,7 +3,6 @@ import { expect, spyOn, test } from 'bun:test'; import { retryMiddleware } from '@/fs'; const { bytes, request } = testKit; -Object.assign(globalThis, { sleep: () => Promise.resolve() }); const sleepSpy = spyOn(globalThis, 'sleep').mockImplementation(() => Promise.resolve()); test('retry middleware retries retryable request and waits between attempts', () => { diff --git a/packages/s3/test/mocks.ts b/packages/s3/test/mocks.ts index ed25d7f1..d612befb 100644 --- a/packages/s3/test/mocks.ts +++ b/packages/s3/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/shared/src/obsidian-mock.ts b/packages/shared/src/mocks.ts similarity index 87% rename from packages/shared/src/obsidian-mock.ts rename to packages/shared/src/mocks.ts index 215d6982..bf7b34cd 100644 --- a/packages/shared/src/obsidian-mock.ts +++ b/packages/shared/src/mocks.ts @@ -48,3 +48,10 @@ export function requireApiVersion() { } export const apiVersion = '1.12.7'; + +Object.assign(globalThis, { + sleep: (milliseconds: number) => + new Promise((resolve) => { + setTimeout(resolve, milliseconds); + }), +}); diff --git a/packages/smart-merge/test/mocks.ts b/packages/smart-merge/test/mocks.ts index ed25d7f1..d612befb 100644 --- a/packages/smart-merge/test/mocks.ts +++ b/packages/smart-merge/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/webdav/test/mocks.ts b/packages/webdav/test/mocks.ts index ed25d7f1..d612befb 100644 --- a/packages/webdav/test/mocks.ts +++ b/packages/webdav/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; void mock.module('obsidian', () => ObsidianMock); From a4136b53c2d28c8b2263f83e551184e9857ab207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?He=CC=84sperus?= Date: Sun, 23 Aug 2026 13:49:00 +0800 Subject: [PATCH 7/7] chore(docs): re-draft google drive docs --- .github/workflows/deploy.yml | 5 +- docs/src/pages/en/deep-dive/modules/gdrive.md | 205 +++++++++++++----- manifest.json | 2 +- modules.json | 9 - packages/gdrive/src/gdrive/auth.ts | 4 +- packages/gdrive/src/i18n.ts | 88 +++++++- packages/gdrive/src/index.ts | 5 +- packages/gdrive/test/mocks.ts | 3 + packages/gdrive/tsdown.config.ts | 5 +- packages/plugin/CHANGELOG.md | 24 ++ packages/plugin/package.json | 2 +- versions.json | 3 +- 12 files changed, 282 insertions(+), 73 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b9b367b9..1d08dfc0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,7 +12,7 @@ permissions: contents: write concurrency: - group: "pages" + group: 'pages' cancel-in-progress: true jobs: @@ -36,6 +36,9 @@ jobs: - name: Build modules run: bun compile + env: + GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} + GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} - name: Sync modules run: bun scripts/deploy-modules.ts diff --git a/docs/src/pages/en/deep-dive/modules/gdrive.md b/docs/src/pages/en/deep-dive/modules/gdrive.md index ed1a24e4..dd962259 100644 --- a/docs/src/pages/en/deep-dive/modules/gdrive.md +++ b/docs/src/pages/en/deep-dive/modules/gdrive.md @@ -1,83 +1,184 @@ # Google Drive Module -The Google Drive module connects Sync Engine to Google Drive. It registers the `gdrive` remote file system, authenticates with OAuth through Google's device flow, and stores the vault as regular files and folders that stay readable in the Drive web and mobile apps. +The Google Drive module registers the `gdrive` remote file system. It stores the vault in a Google Drive folder and performs file operations through the Google Drive public API. The module uses Google OAuth for authentication and does not use a third-party server or proxy. -## Requirements +## Settings -The module uses a Google Cloud OAuth client that you create in your own Google account, so no third-party server ever handles your tokens: +The module adds a **Google Drive** settings group. Configure these settings +before selecting Google Drive as the remote file system: -1. In [Google Cloud Console](https://console.cloud.google.com/), create a project (or reuse one) and enable the **Google Drive API**. -2. Configure the OAuth consent screen. Add yourself as a test user, or publish the app to production — a consent screen left in testing mode expires refresh tokens after seven days, forcing weekly reconnects. -3. Create an OAuth client of type **TV and Limited Input devices** and note the client ID and client secret. +### Connect account / Account connected -The same client ID and secret are entered on every device that syncs; each device completes its own device-flow approval. +This setting initiates the Google authorization process. Once authorized, the module stores the refresh token securely in Obsidian's secret storage and uses it to obtain short-lived access tokens as needed. The **Disconnect** option revokes the token with Google, removes it from secret storage, clears any cached access tokens, and deletes the stored account identifier. -## Settings and Configuration +### Base directory -Install and enable the Google Drive module, then select **Google Drive** as the storage backend. Configure these module settings: +This defines the folder in Google Drive that serves as the root directory for this vault. The value is normalized as a directory path. If left empty when the module starts, it defaults to `/`. The specified folder is created automatically during the first synchronization. -| Setting | Description | -| ----------------------- | ----------------------------------------------------------------------------- | -| **OAuth client ID** | Client ID of your Google Cloud OAuth client. | -| **OAuth client secret** | Client secret of the same OAuth client. Stored in Obsidian's keychain. | -| **Google account** | Connect, reconnect, or disconnect the Google account through the device flow. | -| **Base directory** | Drive folder that holds the vault. Defaults to the vault name. | -| **Delete to trash** | Move remote deletions to the Drive trash instead of deleting permanently. | +::: warning -Connecting opens a dialog with a short code: visit the shown Google URL on any device, enter the code, and approve access. The dialog polls until Google confirms, then the module stores the refresh token and shows the connected account. +The base directory is an application-managed namespace. The module can see and operate only on files that it created through its Google Drive integration. Files or folders created manually in Drive, or created by another application, are not visible to this module even when they are inside the configured base directory. They will not be imported, synchronized, or listed. -## Credentials and Keychain +Do not manually create, rename, move, or maintain vault files in this Drive folder. Set the base directory in the module settings, then let the first sync create it and let Sync Engine manage its contents. Manual Drive operations can leave files outside the module's visible set or cause conflicting changes. -The client secret and the refresh token are stored through Obsidian's secret storage, not as ordinary module settings. Access tokens are short-lived, kept only in memory, and refreshed automatically. Disconnecting clears the stored refresh token; access can also be revoked at [myaccount.google.com/permissions](https://myaccount.google.com/permissions). +This is a consequence of Google's `drive.file` scope, not a filtering option that can be disabled in Sync Engine. The module is intentionally not granted full Drive access. -## Scope and Visibility +::: -The module requests only the `drive.file` scope, plus `email` to display the connected account. `drive.file` grants access exclusively to files this module created — it cannot read the rest of the Drive. The practical consequences: +### Delete to trash -- Do not create the base directory manually in Drive; the module creates it on the first sync. A manually created folder is invisible to the module and leads to a duplicate. -- Files added to the vault folder through the Drive web or mobile apps are invisible to the module and never sync. Edit through Obsidian only; treat the Drive copy as read-only. -- Google shows an "unverified app" style consent step for personal OAuth clients. That is expected — the client is your own. +When enabled, files deleted remotely are moved to the Google Drive trash instead of being permanently removed. When disabled, deleted files are permanently erased immediately. Note that Google Drive typically clears items from the trash after 30 days. -## Base Directory +## Permissions And Scopes -The base directory is resolved to a Drive folder ID and used as the file-system root, so the folder path never appears in keys. It must be identical on every device syncing the same vault. Renaming or moving the vault folder in the Drive web interface changes nothing for sync (IDs stay stable) as long as the configured path still resolves; keep the setting and the actual folder in agreement. +The module requests exactly these OAuth scopes: -## Practical Behavior +### `drive.file` -- Folders are real Drive folders and moves use Drive's native rename and re-parenting — no copy-and-delete. -- Deleting a missing file is treated as success. With **Delete to trash** enabled, deletions land in the Drive trash, which Google empties after 30 days. -- File UIDs use the Drive `md5Checksum`. The local modification time is written to Drive's `modifiedTime` on upload, so timestamps survive round trips between devices. -- Duplicate names in one folder (possible in Drive, not in a vault) resolve to the most recently modified file. -- [Asymmetric storage](../asymmetric-storage) can flatten and anchor remote keys. Use it only when remote files do not need to remain readable in their normal folder structure, and keep the setting consistent across devices. +This scope permits the module to create and manage files that it creates in Google Drive. Sync requires this access to: -## Implementation +- Create the base directory and vault files +- List the module's files so it can discover remote changes +- Read file contents and metadata +- Upload new contents and update existing files +- Move or rename files +- Delete files, either permanently or by moving them to trash -### Unified File-System Mapping +The scope does **not** grant general access to the user's Drive. In particular, the module cannot discover or synchronize files created outside the plugin. Granting `drive.file` is a deliberate least-privilege choice: the module can manage its own sync data without receiving permission to read unrelated Drive files. -`GdriveFs` implements the SDK `RootFs` contract with unified keys. Drive is ID-based, so the module resolves path keys to file IDs segment by segment and caches the mapping for the lifetime of the instance. The basic operations map to Drive API v3 requests as follows: +### `openid` -| File-system operation | Drive operation | -| --------------------- | ------------------------------------------------ | -| `read()` | `files.get` with `alt=media` | -| `write()` | Multipart upload (create or update) | -| `stat()` | `files.list` lookup by parent and name | -| `delete()` | `files.update` with `trashed`, or `files.delete` | -| `move()` | `files.update` with `addParents`/`removeParents` | -| `mkdir()` | `files.create` with the folder MIME type | -| `list()` | Paginated `files.list`, assembled into a tree | +The `openid` scope makes Google return an OpenID Connect ID token during device authorization. The module reads the token's stable `sub` subject identifier and stores it as the connected account identifier. This lets Sync Engine identify which Google account is connected and distinguish multiple account connections, so that the sync record for different Google accounts don't interfere; it does not read the user's profile or request broad identity permissions. -### Bearer Middleware +## Authentication Flow -When Google Drive is the selected backend, registered request middleware attaches a `Bearer` access token to every remote request. A shared token manager caches the access token, refreshes it through the stored refresh token shortly before expiry, deduplicates concurrent refreshes, and retries a request once after an authentication failure. A revoked refresh token surfaces as a clear reconnect prompt. +The module uses [**Google OAuth 2.0 for TV and Limited-Input Device Applications**](https://developers.google.com/identity/protocols/oauth2/limited-input-device): -### Flat Listing +1. When you click **Connect**, the module requests a device code from Google's device authorization endpoint with the two scopes above. +2. Obsidian displays Google's verification URL and a one-time user code. The module can copy the code and open the URL in a browser. +3. You sign in to Google in that browser and approve the requested access. +4. While the dialog remains open, Obsidian polls Google's token endpoint. It waits when authorization is pending and backs off when Google requests a slower polling interval. +5. After approval, Google returns a short-lived access token, a refresh token, and an OpenID ID token. The module extracts the account subject from the ID token, stores the refresh token in Obsidian's secret storage, and caches the access token in memory. +6. Later Drive requests use the cached access token. When it is close to expiry, the module exchanges the refresh token for a new access token. A failed request with HTTP 401 causes one forced refresh and retry. -Because `drive.file` limits visibility to module-created files, `list()` fetches every visible file in one paginated query (1000 files per page) instead of one request per folder, then assembles the tree client-side from parent references. The walk honors the unified reporter verdicts, skipping excluded subtrees without visiting them. +Device authorization is used because Obsidian mobile cannot reliably provide the local browser redirect, localhost listener, or desktop-style custom URL callback required by common interactive OAuth flows. Device authorization keeps the OAuth interaction in a normal browser while the Obsidian app polls Google's endpoint, so the same connection flow works on desktop and mobile. -### Range Reads +## Privacy Policy -`readStream()` downloads media with ranged `GET` requests: 2 MiB chunks, at most eight in flight, emitted in file order. Empty files return an already-closed stream. +Last updated on **August 23, 2026**. -### Resumable Uploads +### Introduction -`writeStream()` buffers small files and sends them as one multipart upload. Files of 8 MiB or more use a Drive resumable upload session: metadata initiates the session, sequential `PUT` requests send 8 MiB chunks (Drive requires multiples of 256 KiB), and the final chunk closes the session. On failure the module attempts to cancel the session. +This Privacy Policy describes how the Sync Engine plugin for Obsidian with Google Drive module (“the Plugin”) handles your data when you connect a Google Drive account. The Plugin is open-source software licensed under the MIT License. + +### Data We Collect + +**We collect no data.** The Plugin has no telemetry, no analytics, no remote logging service, and no backend server. No information about you, your files, or your usage ever leaves your local device. + +### How Your Data Is Handled + +**Google Account Connection**: + +When you click “Connect,” the Plugin initiates a Google Device Authorization flow directly between your device and Google’s servers. The Plugin requests only these scopes: + +- `drive.file`: Access to files created by the Plugin. Files created outside the Plugin are not visible to it, even inside the configured base directory. +- `openid`: Supplies a stable Google account subject identifier so the Plugin can identify and deduplicate connections + +**Token Storage**: + +OAuth refresh tokens are stored exclusively in Electron’s encrypted secret storage on your local device. Tokens are never transmitted to any third party, never logged, and never included in crash reports or diagnostics. + +**File Operations**: + +All sync operations are triggered or scheduled manually by you. File reads and writes occur directly between your local Obsidian vault and Google Drive via Google’s API. No intermediary servers are involved. + +**Data Retention**: + +Your data exists only on your local device and in your own Google Drive account. When you click “Disconnect,” the Plugin: + +1. Revokes the OAuth token with Google +2. Deletes all stored tokens from Electron secret storage + +After disconnection, no trace of your Google Account connection remains on your device. + +### Third Parties + +The only third-party service involved is Google’s OAuth and Drive API, which you authorize directly. We have no relationship with Google beyond using their public APIs. We do not share, sell, or transfer any data to any entity. + +### Your Rights + +You have complete control: + +- All data is on your local device; inspect it anytime +- Click “Disconnect” to erase all local credentials instantly +- Revoke access anytime at `https://myaccount.google.com/permissions` +- Your files remain in your Google Drive regardless of Plugin status + +### Changes + +Updates to this policy will be published in the Plugin’s GitHub repository and `https://sync.consensia.cc`. Continued use after changes constitutes acceptance. + +### Contact + +Open an issue on our GitHub repository for privacy-related questions. + +## Terms of Service + +Last updated on **August 23, 2026**. + +### Acceptance + +By installing or using the Sync Engine plugin for Obsidian with Google Drive module (“the Plugin”), you agree to these Terms. If you disagree, uninstall the Plugin immediately. + +### License + +The Plugin is provided under the MIT License. You may use, modify, and distribute it freely per that license’s terms. + +### No Warranty + +THE PLUGIN IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. THE AUTHORS AND COPYRIGHT HOLDERS SHALL NOT BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY ARISING FROM USE OF THE PLUGIN, INCLUDING DATA LOSS, SYNC FAILURES, OR GOOGLE DRIVE API CHANGES. + +### User Responsibilities + +You are solely responsible for: + +- Maintaining the security of your Google Account credentials +- Understanding what data you choose to sync +- Backing up important files independently of the Plugin +- Complying with Google’s Terms of Service when connecting your Drive account +- Ensuring your local device’s Electron secret storage remains secure + +### Acceptable Use + +Do not use the Plugin to: + +- Violate Google’s API Terms of Service +- Access accounts you do not own or lack authorization for +- Circumvent Google Drive storage or rate limits +- Distribute malware or illegal content via synced files + +### Third-Party Services + +The Plugin interacts with Google’s OAuth and Drive APIs. These services are governed by Google’s own Terms of Service and Privacy Policy. We have no control over Google’s services and accept no liability for their availability, changes, or termination. + +### Disconnection & Termination + +You may terminate your use at any time by clicking “Disconnect” in the Plugin settings or uninstalling the Plugin. We reserve the right to discontinue development or distribution of the Plugin at any time without notice. + +### Limitation of Liability + +TO THE MAXIMUM EXTENT PERMITTED BY LAW, IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL, OR PUNITIVE DAMAGES, INCLUDING LOST PROFITS, DATA LOSS, OR BUSINESS INTERRUPTION, REGARDLESS OF THEORY OF LIABILITY. + +### Governing Law + +This Plugin is developed and maintained on a voluntary, non-commercial basis by contributors located in multiple jurisdictions worldwide. No single governing law applies. + +These Terms shall be interpreted in accordance with general principles of international law and the MIT License under which the Plugin is distributed. + +### Changes + +We may update these Terms at any time. Changes take effect upon publication in the GitHub repository. Continued use constitutes acceptance. + +### Contact + +Open an issue on our GitHub repository for questions regarding these Terms. diff --git a/manifest.json b/manifest.json index 6ea798ef..1f2e5d85 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "sync-engine", "name": "Sync Engine", - "version": "3.1.0", + "version": "3.1.1", "minAppVersion": "1.13.0", "authorUrl": "https://hesprs.github.io", "description": "The next-generation syncing plugin: Fast · Free · Extend with Modules. Supports WebDAV and S3.", diff --git a/modules.json b/modules.json index b70aa845..44fc83e2 100644 --- a/modules.json +++ b/modules.json @@ -61,14 +61,5 @@ "icon": "combine", "main": "https://sync.consensia.cc/modules/smart-merge.js", "minPluginVersion": "3.1.0" - }, - { - "id": "gdrive", - "name": "Google Drive", - "version": "0.1.0", - "description": "Google Drive backend support.", - "icon": "hard-drive", - "main": "https://sync.consensia.cc/modules/gdrive.js", - "minPluginVersion": "3.1.0" } ] diff --git a/packages/gdrive/src/gdrive/auth.ts b/packages/gdrive/src/gdrive/auth.ts index edebfd8d..f99013b4 100644 --- a/packages/gdrive/src/gdrive/auth.ts +++ b/packages/gdrive/src/gdrive/auth.ts @@ -9,8 +9,8 @@ import { TOKEN_REVOKE_URL, } from './api'; -export const CLIENT_ID = process.env.CLIENT_ID ?? ''; -export const CLIENT_SECRET = process.env.CLIENT_SECRET ?? ''; // Not really a secret +export const CLIENT_ID = atob(process.env.CLIENT_ID ?? ''); +export const CLIENT_SECRET = atob(process.env.CLIENT_SECRET ?? ''); // Not really a secret const KEYCHAIN_SECRET_ID = 'sync-engine-gdrive-refresh-token'; // Secret storage id under which the Google refresh token is stored. type TokenResponse = { diff --git a/packages/gdrive/src/i18n.ts b/packages/gdrive/src/i18n.ts index d31e5c4c..a5f53341 100644 --- a/packages/gdrive/src/i18n.ts +++ b/packages/gdrive/src/i18n.ts @@ -1,12 +1,12 @@ import type { GdriveTranslations } from './setting'; -const en: GdriveTranslations = { +export const en: GdriveTranslations = { accountConnected: 'Account connected', accountConnectedDescription: 'Connected to Google Drive account.', authorizationFailed: 'Authorization failed: {{reason}}', baseDirectory: 'Base directory', baseDirectoryDescription: - 'Set the folder in Google Drive that holds this vault. Created automatically on the first sync and do not create it manually in Drive, files added outside this plugin are invisible to sync.', + 'Set the folder in Google Drive that holds this vault. Created automatically on the first sync and do not create it manually in Drive. Files added outside this plugin are invisible to sync.', baseDirectoryPlaceholder: 'my-vault/', configureFirst: 'Enter the OAuth client ID and client secret first.', connect: 'Connect', @@ -28,4 +28,86 @@ const en: GdriveTranslations = { waitingApproval: 'Waiting for approval…', }; -export default en; +export const ru: GdriveTranslations = { + accountConnected: 'Аккаунт подключён', + accountConnectedDescription: 'Подключено к аккаунту Google Drive.', + authorizationFailed: 'Ошибка авторизации: {{reason}}', + baseDirectory: 'Базовый каталог', + baseDirectoryDescription: + 'Укажите папку в Google Drive, в которой будет храниться это хранилище. Она создаётся автоматически при первой синхронизации, не создавайте её вручную на Диске. Файлы, добавленные вне этого плагина, будут невидны для синхронизации.', + baseDirectoryPlaceholder: 'my-vault/', + configureFirst: 'Сначала введите OAuth client ID и client secret.', + connect: 'Подключить', + connectAccount: 'Подключить аккаунт', + connectAccountDescription: 'Нажмите кнопку, чтобы подключиться к вашему аккаунту Google Drive.', + connectSuccess: 'Успешно подключено к Google Drive.', + copyAndOpenGoogle: 'Скопировать и открыть Google', + deviceCodeInstruction: (frag, url) => { + frag.appendText('Пожалуйста, перейдите по ссылке '); + frag.createEl('a', { attr: { href: url } }).createEl('code', { text: url }); + frag.appendText(' и введите код ниже, после чего подтвердите доступ.'); + }, + deviceCodeTitle: 'Подключение Google Drive', + disconnect: 'Отключить', + gdrive: 'Google Drive', + useTrash: 'Удалять в корзину', + useTrashDescription: + 'Перемещать удалённые файлы в корзину Google Drive вместо их безвозвратного удаления. Диск автоматически очищает корзину через 30 дней.', + waitingApproval: 'Ожидание подтверждения…', +}; + +export const zhTW: GdriveTranslations = { + accountConnected: '帳號已連線', + accountConnectedDescription: '已成功連線至 Google Drive 帳號。', + authorizationFailed: '驗證失敗：{{reason}}', + baseDirectory: '基礎目錄', + baseDirectoryDescription: + '設定 Google Drive 中用來存放此儲存庫的資料夾。系統將於首次同步時自動建立，請勿手動在 Drive 中建立。在此外掛程式之外新增的檔案將無法被同步讀取。', + baseDirectoryPlaceholder: 'my-vault/', + configureFirst: '請先輸入 OAuth 用戶端 ID 與用戶端密鑰。', + connect: '連線', + connectAccount: '連結帳號', + connectAccountDescription: '點擊按鈕以連結您的 Google Drive 帳號。', + connectSuccess: '已成功連線至 Google Drive。', + copyAndOpenGoogle: '複製並前往 Google 頁面', + deviceCodeInstruction: (frag, url) => { + frag.appendText('請前往 '); + frag.createEl('a', { attr: { href: url } }).createEl('code', { text: url }); + frag.appendText(' 並輸入下方驗證碼，隨後核准存取權限。'); + }, + deviceCodeTitle: '連結 Google Drive', + disconnect: '中斷連線', + gdrive: 'Google Drive', + useTrash: '移至垃圾桶', + useTrashDescription: + '刪除檔案時將其移至 Google Drive 垃圾桶而非永久刪除。Drive 會在 30 天後自動清理垃圾桶。', + waitingApproval: '等待核准中…', +}; + +export const zh: GdriveTranslations = { + accountConnected: '账号已连接', + accountConnectedDescription: '已连接至 Google Drive 账号。', + authorizationFailed: '授权失败：{{reason}}', + baseDirectory: '基础目录', + baseDirectoryDescription: + '设置 Google Drive 中存放此仓库的文件夹。该目录会在首次同步时自动创建，请勿在 Drive 中手动创建。此插件之外添加的文件对同步不可见。', + baseDirectoryPlaceholder: 'my-vault/', + configureFirst: '请先输入 OAuth 客户端 ID 和客户端密钥。', + connect: '连接', + connectAccount: '连接账号', + connectAccountDescription: '点击按钮连接到您的 Google Drive 账号。', + connectSuccess: '已连接至 Google Drive。', + copyAndOpenGoogle: '复制并打开 Google', + deviceCodeInstruction: (frag, url) => { + frag.appendText('请访问 '); + frag.createEl('a', { attr: { href: url } }).createEl('code', { text: url }); + frag.appendText(' 并输入下方验证码，然后批准访问权限。'); + }, + deviceCodeTitle: '连接 Google Drive', + disconnect: '断开连接', + gdrive: 'Google Drive', + useTrash: '删除至回收站', + useTrashDescription: + '将删除的文件移动至 Google Drive 回收站，而非永久删除。Drive 会在 30 天后自动清空回收站。', + waitingApproval: '等待批准中…', +}; diff --git a/packages/gdrive/src/index.ts b/packages/gdrive/src/index.ts index d50b4762..2e0264d6 100644 --- a/packages/gdrive/src/index.ts +++ b/packages/gdrive/src/index.ts @@ -19,7 +19,7 @@ import type { GdriveTranslations } from './setting'; import { TokenManager, bearerMiddleware } from './gdrive/auth'; import checkConnection from './gdrive/check-connection'; import GdriveFs from './gdrive/fs'; -import en from './i18n'; +import { ru, en, zh, zhTW } from './i18n'; import gdriveSetting from './setting'; import styles from './styles.css?inline'; @@ -49,6 +49,9 @@ export default class Gdrive { if (!this.moduleSettings.baseDirectory) this.moduleSettings.baseDirectory = `${ctx.app.vault.getName()}/`; ctx.registerI18n('en', en); + ctx.registerI18n('zh', zh); + ctx.registerI18n('zh-TW', zhTW); + ctx.registerI18n('ru', ru); this.tokenManager = new TokenManager(ctx.app.secretStorage); } diff --git a/packages/gdrive/test/mocks.ts b/packages/gdrive/test/mocks.ts index d612befb..45f6b609 100644 --- a/packages/gdrive/test/mocks.ts +++ b/packages/gdrive/test/mocks.ts @@ -2,4 +2,7 @@ import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; +process.env.CLIENT_ID = btoa(process.env.GDRIVE_CLIENT_ID ?? ''); +process.env.CLIENT_SECRET = btoa(process.env.GDRIVE_CLIENT_SECRET ?? ''); + void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/gdrive/tsdown.config.ts b/packages/gdrive/tsdown.config.ts index 6abd540d..92593f19 100644 --- a/packages/gdrive/tsdown.config.ts +++ b/packages/gdrive/tsdown.config.ts @@ -5,9 +5,10 @@ const dev = process.env.MODE === 'dev'; export default defineConfig({ clean: !dev, + css: { minify: true }, define: { - 'process.env.CLIENT_ID': JSON.stringify(process.env.CLIENT_ID ?? ''), - 'process.env.CLIENT_SECRET': JSON.stringify(process.env.CLIENT_SECRET ?? ''), + 'process.env.CLIENT_ID': JSON.stringify(btoa(process.env.GDRIVE_CLIENT_ID ?? '')), + 'process.env.CLIENT_SECRET': JSON.stringify(btoa(process.env.GDRIVE_CLIENT_SECRET ?? '')), }, dts: false, entry: { gdrive: 'src/index.ts' }, diff --git a/packages/plugin/CHANGELOG.md b/packages/plugin/CHANGELOG.md index fb781937..2f87ccf3 100644 --- a/packages/plugin/CHANGELOG.md +++ b/packages/plugin/CHANGELOG.md @@ -2,6 +2,30 @@ All notable changes to this project will be documented in this file. +## Sync Engine v3.1.1 - 2026-08-23 + +### Core + +- Fixed sync failure caused by deleting a parent folder whose children are moved to other folders. +- Added clearer failure message for file names containing Windows forbidden characters. + +### S3 Module + +- Supported optional `session_token` in S3 authorization. +- Optimized sync speed by caching SigV4 signing key. + +### Google Drive Module + +- Experimental Google Drive backend. Internal testing while waiting Google App review. + +### WebDAV Module + +- Eliminated minor discrepancy on Etag handling between `PUT` and `PROPFIND` responses. + +### Contributors + +@Quzzar, @xx025, @hesprs + ## Sync Engine v3.1.0 - 2026-08-21 ### UI Modernization diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 601a0865..8926a84d 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@hesprs/sync-engine-sdk", - "version": "3.1.0", + "version": "3.1.1", "description": "Official SDK for developing modules targeting Sync Engine, the extensible Obsidian syncing plugin.", "keywords": [ "obsidian-plugin", diff --git a/versions.json b/versions.json index fdd96fef..f3a6e382 100644 --- a/versions.json +++ b/versions.json @@ -6,5 +6,6 @@ "3.0.4": "1.12.3", "3.0.5": "1.12.3", "3.0.6": "1.12.3", - "3.1.0": "1.13.0" + "3.1.0": "1.13.0", + "3.1.1": "1.13.0" }