fix(firestore)!: decode missing keys for Optional and server-stamped fields

example/app/src/routes/firestore.tsx

@@ -87,8 +87,6 @@ function PostForm({
             title: value.title,
             content: value.content,
             author: AuthorId.make('1'),
-            createdAt: undefined,
-            updatedAt: undefined,
             checked: false,
             optional: Option.none(),
             list: [],

packages/effect-firebase/AGENTS.md

@@ -77,6 +77,7 @@ export class PostModel extends Model.Class<PostModel>('PostModel')({
   likes: Firestore.Number, // accepts Firestore.increment(n) in update
   tags: Firestore.Array(Schema.String), // accepts arrayUnion/arrayRemove in update
   summary: Firestore.OptionalDeletable(Schema.String), // Option in app; Firestore.delete() removes it
+  subtitle: Schema.optionalKey(Schema.String), // plain `string | absent`; prefer over Schema.optional
   checked: Schema.Boolean.pipe(
     Schema.withDecodingDefault(Effect.succeed(false)),
   ),
@@ -91,20 +92,20 @@ Variants: `PostModel` (alias `.select`, what reads decode to), `.insert`,
 
 Field helpers (all under `Firestore.` unless noted):
 
-| Helper                                                   | Notes                                                                                                                                      |
-| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
-| `Model.GeneratedByDb(s)` / `Model.GeneratedByApp(s)`     | From `effect/unstable/schema`. DB-generated ids vs app-generated ids.                                                                      |
-| `DateTimeInsert`, `DateTimeUpdate`                       | Auto server timestamps. App type is `DateTime.Utc`.                                                                                        |
-| `DateTime`, `ServerDateTime`                             | Plain timestamp; `ServerDateTime` writes server time when given `undefined`.                                                               |
-| `WithServerTimestamp(field)`                             | Lets insert/update accept `Firestore.serverTimestamp()` explicitly.                                                                        |
-| `Reference(id, path)`, `ReferenceOptional(id, path)`     | Typed reference exposed as branded id.                                                                                                     |
-| `ReferenceAsInstance(id, path)`, `ReferencePath(path)`   | Expose `FirestoreSchema.Reference` instance / full path string.                                                                            |
-| `AnyIdReference`, `AnyPathReference`                     | Untyped references.                                                                                                                        |
-| `Optional(s)`, `OptionalNull(s)`, `OptionalDeletable(s)` | `Option` in app. `Optional` accepts null/undefined, `OptionalNull` only null, `OptionalDeletable` supports `Firestore.delete()` in update. |
-| `Array(s)`, `WithArrayFields(field)`                     | `Firestore.arrayUnion([...])` / `arrayRemove([...])` in update.                                                                            |
-| `Number`, `WithIncrementField(field)`                    | `Firestore.increment(n)` in update.                                                                                                        |
-| `GeoPoint`                                               | `FirestoreSchema.GeoPoint` instance in app, `{ latitude, longitude }` in JSON.                                                             |
-| `Model.Field({ select, insert, update, json, ... })`     | Fully custom per-variant schemas (from `effect/unstable/schema`).                                                                          |
+| Helper                                                   | Notes                                                                                                                                                                                      |
+| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `Model.GeneratedByDb(s)` / `Model.GeneratedByApp(s)`     | From `effect/unstable/schema`. DB-generated ids vs app-generated ids.                                                                                                                      |
+| `DateTimeInsert`, `DateTimeUpdate`                       | Auto server timestamps. App type is `DateTime.Utc`.                                                                                                                                        |
+| `DateTime`, `ServerDateTime`                             | Plain timestamp; `ServerDateTime` writes server time when the key is omitted or `undefined`.                                                                                               |
+| `WithServerTimestamp(field)`                             | Lets insert/update accept `Firestore.serverTimestamp()` explicitly.                                                                                                                        |
+| `Reference(id, path)`, `ReferenceOptional(id, path)`     | Typed reference exposed as branded id.                                                                                                                                                     |
+| `ReferenceAsInstance(id, path)`, `ReferencePath(path)`   | Expose `FirestoreSchema.Reference` instance / full path string.                                                                                                                            |
+| `AnyIdReference`, `AnyPathReference`                     | Untyped references.                                                                                                                                                                        |
+| `Optional(s)`, `OptionalNull(s)`, `OptionalDeletable(s)` | `Option` in app. `Optional` reads a missing key/null/undefined and writes `null`; `OptionalNull` only null; `OptionalDeletable` omits the key and supports `Firestore.delete()` in update. |
+| `Array(s)`, `WithArrayFields(field)`                     | `Firestore.arrayUnion([...])` / `arrayRemove([...])` in update.                                                                                                                            |
+| `Number`, `WithIncrementField(field)`                    | `Firestore.increment(n)` in update.                                                                                                                                                        |
+| `GeoPoint`                                               | `FirestoreSchema.GeoPoint` instance in app, `{ latitude, longitude }` in JSON.                                                                                                             |
+| `Model.Field({ select, insert, update, json, ... })`     | Fully custom per-variant schemas (from `effect/unstable/schema`).                                                                                                                          |
 
 ## Create a repository
 
@@ -450,6 +451,12 @@ root: https://github.com/fwal/effect-firebase/blob/main/REACT.md.
     nested partial; a plain `{ a: { b: v } }` replaces the whole `a` map.
     Undeclared keys fail with `SchemaError`; `update` never silently drops
     them.
+11. Optional fields: use `Firestore.Optional*` when the app wants an
+    `Option`, and `Schema.optionalKey(s)` for a plain `T | absent` field.
+    Avoid `Schema.optional(s)`: it lets `{ field: undefined }` through the
+    schema, and the Firebase SDKs reject `undefined` as a value at write
+    time. `optionalKey` fails with `SchemaError` instead. For a default,
+    `Schema.optionalKey(s).pipe(Schema.withDecodingDefault(Effect.succeed(v)))`.
 
 ## Where to look
 

packages/effect-firebase/MIGRATION.md

@@ -191,10 +191,39 @@ const Custom = Model.Field({ select: A, insert: B, update: C, json: D });
 `Model.ServerDateTime` was wrapped in `VariantSchema.Overrideable` and
 required `Model.Override(value)` to write an explicit timestamp. In v1.0
 `Firestore.ServerDateTime` is a plain field: pass a `DateTime.Utc` to write
-that instant, or `undefined` to write the server timestamp. For fields where
-you want to opt into the server timestamp explicitly, wrap any timestamp
-field in `Firestore.WithServerTimestamp(...)` and pass
-`Firestore.serverTimestamp()`.
+that instant, or omit the key (or pass `undefined`) to write the server
+timestamp. The same applies to the `insert`/`update` variants of
+`DateTimeInsert` and `DateTimeUpdate`, so `repo.add({ title })` no longer
+needs `createdAt: undefined`. For fields where you want to opt into the
+server timestamp explicitly, wrap any timestamp field in
+`Firestore.WithServerTimestamp(...)` and pass `Firestore.serverTimestamp()`.
+
+**`Optional` encoded type.** The database variants (`select`/`insert`/
+`update`) of `Firestore.Optional(s)` decode a missing key, `null` and
+`undefined` to `Option.none()`, as they did in v0.x (Effect v4 no longer
+tolerates a missing key on a required property, so v1.0 betas before beta.7
+failed reads of documents without the field with `SchemaError: Missing key`).
+`Option.none()` is still encoded as `null`, so write payloads are unchanged.
+The encoded field type moved from `Schema.NullishOr<S>` to
+`Schema.optional(Schema.NullOr<S>)`, which makes the key optional in
+`Model.Encoded` / `Model.insert.Encoded` / `Model.update.Encoded`. Code that
+spelled these fields out as `Schema.OptionFromNullishOr<...>` must use
+`Schema.OptionFromOptionalNullOr<...>` instead. `OptionalNull` and
+`ReferenceOptional` are unchanged and still require the key to be present.
+
+**Plain optional fields.** For a field that should be `T | absent` in the
+app (no `Option`), declare it with `Schema.optionalKey(s)` rather than
+`Schema.optional(s)`. `Schema.optional` also admits `undefined` as a value,
+which passes the schema but is rejected by the Firebase SDKs on write;
+`optionalKey` rejects it with a `SchemaError` up front. Firestore documents
+never contain `undefined`, so nothing is lost on read.
+
+**`OptionalDeletable` update variant.** `Option.none()` in an `update`
+payload is now encoded as a missing key instead of `undefined` (which the
+Firebase SDKs reject as a value), and a document without the key decodes
+through `Model.update`. A merge update whose only field is `Option.none()`
+therefore fails with `FirestoreError` code `invalid-argument` (empty
+payload); use `Option.some(Firestore.delete())` to remove the field.
 
 ### 5. Update `FirestoreField` import (converter usage)
 
@@ -379,16 +408,21 @@ the mock's simulated states live.
 
 Additional Effect v4 breaking changes you may encounter in your own code:
 
-| v3                                                | v4                                                             |
-| ------------------------------------------------- | -------------------------------------------------------------- |
-| `Effect.catchAll(f)`                              | `Effect.catch(f)`                                              |
-| `Effect.catchAllDefect(f)`                        | `Effect.catchDefect(f)`                                        |
-| `Effect.catchAllCause(f)`                         | `Effect.catchCause(f)`                                         |
-| `Schema.Union(a, b, ...)`                         | `Schema.Union([a, b, ...])`                                    |
-| `struct.pick('field')`                            | `struct.mapFields(Struct.pick(['field']))`                     |
-| `ParseResult.ArrayFormatter.formatErrorSync(e)`   | `e.message` (use `Schema.isSchemaError(e)` to narrow)          |
-| `import { ParseError } from 'effect/ParseResult'` | `import { Schema } from 'effect'` → use `Schema.isSchemaError` |
-| `Context.Tag('id')<Self, Shape>()`                | `Context.Service<Self, Shape>()('id')`                         |
+| v3                                                | v4                                                                          |
+| ------------------------------------------------- | --------------------------------------------------------------------------- |
+| `Effect.catchAll(f)`                              | `Effect.catch(f)`                                                           |
+| `Effect.catchAllDefect(f)`                        | `Effect.catchDefect(f)`                                                     |
+| `Effect.catchAllCause(f)`                         | `Effect.catchCause(f)`                                                      |
+| `Schema.Union(a, b, ...)`                         | `Schema.Union([a, b, ...])`                                                 |
+| `Schema.optionalWith(s, { default: () => v })`    | `Schema.optionalKey(s).pipe(Schema.withDecodingDefault(Effect.succeed(v)))` |
+| `struct.pick('field')`                            | `struct.mapFields(Struct.pick(['field']))`                                  |
+| `ParseResult.ArrayFormatter.formatErrorSync(e)`   | `e.message` (use `Schema.isSchemaError(e)` to narrow)                       |
+| `import { ParseError } from 'effect/ParseResult'` | `import { Schema } from 'effect'` → use `Schema.isSchemaError`              |
+| `Context.Tag('id')<Self, Shape>()`                | `Context.Service<Self, Shape>()('id')`                                      |
+
+Fields spread from a struct with a decoding default (the
+`Schema.optionalWith(..., { default })` pattern) work in `Model.Class` again;
+v0.x rejected them with `Unsupported schema`.
 
 For a complete list of Effect v4 breaking changes beyond what's covered here,
 see the [Effect migration guide](https://effect.website/docs/migration-guide).

packages/effect-firebase/src/lib/firestore/model/datetime.spec.ts

@@ -5,6 +5,7 @@ import {
   DateTime,
   DateTimeInsert,
   DateTimeUpdate,
+  ServerDateTime,
   WithServerTimestamp,
 } from './datetime.js';
 import { Timestamp } from '../schema/timestamp.js';
@@ -130,13 +131,33 @@ describe('Model.DateTimeInsert', () => {
   });
 
   describe('insert variant', () => {
+    it('should encode a missing key to ServerTimestamp', () => {
+      const encode = Schema.encodeSync(TestModel.insert);
+      const result = encode({});
+
+      expect(result.createdAt).toBeInstanceOf(FirestoreSchema.ServerTimestamp);
+    });
+
     it('should encode undefined to ServerTimestamp', () => {
       const encode = Schema.encodeSync(TestModel.insert);
       const result = encode({ createdAt: undefined });
 
       expect(result.createdAt).toBeInstanceOf(FirestoreSchema.ServerTimestamp);
     });
 
+    it('should decode Timestamp to DateTime.Utc', () => {
+      const decode = Schema.decodeUnknownSync(TestModel.insert);
+      const result = decode({ createdAt: Timestamp.fromMillis(1705315800000) });
+
+      expect(EffectDateTime.isDateTime(result.createdAt)).toBe(true);
+    });
+
+    it('should reject decoding a ServerTimestamp', () => {
+      const decode = Schema.decodeUnknownSync(TestModel.insert);
+
+      expect(() => decode({ createdAt: serverTimestamp() })).toThrow();
+    });
+
     it('should encode a DateTime.Utc value to Timestamp', () => {
       const encode = Schema.encodeSync(TestModel.insert);
       const millis = 1705315800000;
@@ -293,5 +314,77 @@ describe('Model.DateTimeUpdate', () => {
 
       expect(result.updatedAt).toBeInstanceOf(FirestoreSchema.ServerTimestamp);
     });
+
+    it('should encode a missing key to ServerTimestamp', () => {
+      const encode = Schema.encodeSync(TestModel.update);
+      const result = encode({});
+
+      expect(result.updatedAt).toBeInstanceOf(FirestoreSchema.ServerTimestamp);
+    });
+
+    it('should encode a DateTime.Utc value to Timestamp', () => {
+      const encode = Schema.encodeSync(TestModel.update);
+      const result = encode({
+        updatedAt: EffectDateTime.makeUnsafe(1705315800000),
+      });
+
+      expect(result.updatedAt).toEqual({ seconds: 1705315800, nanoseconds: 0 });
+    });
+  });
+
+  describe('insert variant', () => {
+    it('should encode a missing key to ServerTimestamp', () => {
+      const encode = Schema.encodeSync(TestModel.insert);
+      const result = encode({});
+
+      expect(result.updatedAt).toBeInstanceOf(FirestoreSchema.ServerTimestamp);
+    });
+  });
+});
+
+describe('Model.ServerDateTime', () => {
+  class TestModel extends Model.Class<TestModel>('TestModel')({
+    id: Model.GeneratedByDb(Schema.String),
+    seenAt: ServerDateTime,
+  }) {}
+
+  describe('get variant', () => {
+    it('should decode Timestamp to DateTime.Utc', () => {
+      const decode = Schema.decodeUnknownSync(TestModel);
+      const result = decode({
+        id: 'post-1',
+        seenAt: Timestamp.fromMillis(1705315800000),
+      });
+
+      expect(EffectDateTime.isDateTime(result.seenAt)).toBe(true);
+    });
+  });
+
+  describe.each([
+    ['insert', () => TestModel.insert],
+    ['update', () => TestModel.update],
+  ] as const)('%s variant', (_, variant) => {
+    it('should encode a missing key to ServerTimestamp', () => {
+      const encode = Schema.encodeUnknownSync(variant());
+      const result = encode({});
+
+      expect(result.seenAt).toBeInstanceOf(FirestoreSchema.ServerTimestamp);
+    });
+
+    it('should encode undefined to ServerTimestamp', () => {
+      const encode = Schema.encodeUnknownSync(variant());
+      const result = encode({ seenAt: undefined });
+
+      expect(result.seenAt).toBeInstanceOf(FirestoreSchema.ServerTimestamp);
+    });
+
+    it('should encode a DateTime.Utc value to Timestamp', () => {
+      const encode = Schema.encodeUnknownSync(variant());
+      const result = encode({
+        seenAt: EffectDateTime.makeUnsafe(1705315800000),
+      });
+
+      expect(result.seenAt).toEqual({ seconds: 1705315800, nanoseconds: 0 });
+    });
   });
 });

packages/effect-firebase/src/lib/firestore/model/datetime.ts

@@ -1,6 +1,7 @@
 import {
   DateTime as EffectDateTime,
   Effect,
+  Option,
   Schema,
   SchemaGetter,
   SchemaIssue,
@@ -23,15 +24,18 @@ export const DateTime: DateTime = Model.Field({
 });
 
 /**
- * Schema for add/update variants that:
+ * Schema for the insert/update variants that:
  * - Decodes: Timestamp → DateTime.Utc (ServerTimestamp decode fails)
- * - Encodes: DateTime.Utc → Timestamp, undefined → ServerTimestamp
+ * - Encodes: DateTime.Utc → Timestamp, missing key or undefined → ServerTimestamp
+ *
+ * The key is optional on the decoded side, so callers can omit the field to
+ * request the server timestamp.
  */
 const ServerDateTimeSchema = Schema.Union([
   FirestoreSchema.TimestampInstance,
   FirestoreSchema.ServerTimestampInstance,
 ]).pipe(
-  Schema.decodeTo(Schema.UndefinedOr(Schema.DateTimeUtc), {
+  Schema.decodeTo(Schema.optional(Schema.DateTimeUtc), {
     decode: SchemaGetter.transformOrFail(
       (input: FirestoreSchema.Timestamp | FirestoreSchema.ServerTimestamp) => {
         if (input instanceof FirestoreSchema.Timestamp) {
@@ -47,13 +51,17 @@ const ServerDateTimeSchema = Schema.Union([
         );
       },
     ),
-    encode: SchemaGetter.transform(
+    encode: SchemaGetter.transformOptional(
       (
-        dt: EffectDateTime.Utc | undefined,
-      ): FirestoreSchema.Timestamp | FirestoreSchema.ServerTimestamp =>
-        dt !== undefined
-          ? FirestoreSchema.Timestamp.fromDateTime(dt)
-          : new FirestoreSchema.ServerTimestamp(),
+        dt: Option.Option<EffectDateTime.Utc | undefined>,
+      ): Option.Option<
+        FirestoreSchema.Timestamp | FirestoreSchema.ServerTimestamp
+      > =>
+        Option.some(
+          Option.isSome(dt) && dt.value !== undefined
+            ? FirestoreSchema.Timestamp.fromDateTime(dt.value)
+            : new FirestoreSchema.ServerTimestamp(),
+        ),
     ),
   }),
 );