jit: hot-path wins, [jit] in a C++ AOT host, and unresolved AOT-object address globals

daslib/aot_cpp.das

@@ -3798,7 +3798,7 @@ class public CppAot : AstVisitor {
             }
             return empty(bif.cppName);
         }
-        if (func.flags.noAot || func.flags.aotHybrid) return true;
+        if (func.flags.noAot || func.flags.aotHybrid || func.moreFlags.requestJit) return true;
         return func._module != program.getThisModule;
     }
     def needsArgPassType(argType : TypeDeclPtr) {

doc/reflections/das2rst.das

@@ -207,7 +207,7 @@ get_value|insert_clone|emplace_new|insert_default|emplace_default|get_with_defau
         group_by_regex("Bit operations", mod, %regex~(popcnt|clz|ctz|mul128|__bit_set)$%%),
         group_by_regex("Intervals", mod, %regex~(interval)$%%),
         group_by_regex("RTTI", mod, %regex~(class_rtti_size)$%%),
-        hide_group(group_by_regex("Jit", mod,%regex~(invoke_code|.*jit.*)$%%)),
+        hide_group(group_by_regex("Jit", mod,%regex~(invoke_code|is_aot_function|.*jit.*)$%%)),
         group_by_regex("Initialization and finalization", mod, %regex~(using|clone|finalize)$%%),
         group_by_regex("Algorithms", mod, %regex~(swap|iter_range|count|ucount|long_iter_range)$%%),
         group_by_regex("Memset", mod, %regex~(memset.*)$%%),

include/daScript/simulate/ARCHITECTURE.md

@@ -16,6 +16,14 @@ the hot set; its cost is judged against the allocate/copy/rehash it rides.
 The ledger the checklist's hot-path rule routes to. Each entry: what was added, where, why
 correctness required it, and the alternative that was rejected.
 
+- **The `adBySid` memo** (`simulate.h`: `adBySidMemo`, `Context::AdMemo`) - jitted code resolves a
+  block's annotation data by sid on every call, once per tick for an entity system; it measured
+  212 cycles a call on enlisted's act stage. A 16-slot direct-mapped memo answers the repeats.
+  Cost: one load and one predictable branch comparing `adMemo.owner` against `tabAdLookup.get()`,
+  plus 256 bytes per `Context`, placed last so no offset the emitter bakes moves. Rejected
+  alternative: clearing the memo at every site that assigns `tabAdLookup` - cheaper to read, but
+  an invariant spread across four writers, one of which had already been missed.
+
 - **CRT scalar transcendentals** (`sim_policy.h`) - the scalar float arms of `Exp`, `Exp2`,
   `Log2` and `Pow` call the CRT; the `vec4f` arms stay on the vecmath polynomials, where
   four lanes amortize the setup. The lane trick's `v_set_x`/`v_extract_x` round-trip is a

include/daScript/simulate/aot_builtin_jit.h

@@ -18,6 +18,7 @@ namespace das {
 
     float4 das_invoke_code ( void * pfun, vec4f anything, void * cmres, Context * context );
     bool das_is_jit_function ( const Func func );
+    bool das_is_aot_function ( const Func func );
     bool das_has_jit_fastpath ( const Func func );
     bool das_remove_jit ( const Func func );
     bool das_instrument_jit ( void * pfun, const Func func, const LineInfo & info, Context & context );

include/daScript/simulate/simulate.h

@@ -537,6 +537,24 @@ namespace das
             DAS_ASSERT(it!=tabAdLookup->end());
             return it->second;
         }
+        // Jitted code resolves a block's annotation data by sid on every call - once per ES tick -
+        // and the map misses cache on each distinct sid. It is immutable after buildADLookup and a
+        // Context is single-threaded, so a direct-mapped memo answers the repeat lookups.
+        __forceinline uint64_t adBySidMemo ( uint64_t sid ) {
+            if ( !sid || !tabAdLookup ) return 0;
+            if ( adMemo.owner != tabAdLookup.get() ) {
+                memset(adMemo.sid, 0, sizeof(adMemo.sid));
+                memset(adMemo.val, 0, sizeof(adMemo.val));
+                adMemo.owner = tabAdLookup.get();
+            }
+            uint32_t slot = uint32_t((sid * 0x9E3779B97F4A7C15ull) >> (64u - AdMemo::BITS));
+            if ( adMemo.sid[slot]==sid ) return adMemo.val[slot];
+            auto it = tabAdLookup->find(sid);
+            uint64_t ad = it!=tabAdLookup->end() ? it->second : 0;
+            adMemo.sid[slot] = sid;
+            adMemo.val[slot] = ad;
+            return ad;
+        }
         __forceinline SimFunction * fnByMangledName ( uint64_t mnh ) {
             if ( mnh==0 ) return nullptr;
             auto it = tabMnLookup->find(mnh);
@@ -922,6 +940,14 @@ namespace das
             void *llvm_context;
         };
         JitContext deleteJITOnFinish = {};
+        // last on purpose: the emitter bakes the offsets above into every jitted function
+        struct AdMemo {
+            static constexpr uint32_t BITS = 4;
+            const void *    owner = nullptr;
+            uint64_t        sid[1u<<BITS] = {};
+            uint64_t        val[1u<<BITS] = {};
+        };
+        AdMemo          adMemo;
         vector<FileInfo*>  deleteUponFinish;
     };
 

modules/dasLLVM/daslib/llvm_aot.das

@@ -10,6 +10,8 @@ require llvm/daslib/llvm_exe
 require llvm/daslib/llvm_dll_utils
 require daslib/ast_boost
 require daslib/rtti
+require daslib/safe_addr
+require strings
 
 //! LLVM-AOT registration pass. After the JIT visitor has generated + optimized the module (as for a
 //! normal exe), this adds the offline-AOT-object registration layer so a statically-linked .o binds
@@ -101,3 +103,37 @@ def public build_llvm_aot_ctors(ctx : LLVMContextRef; mod : LLVMOpaqueModule?; v
     let glob_ctor = emit_aot_object_globinit(ctx, mod, types, funcs, uids)
     return emit_aot_register_ctor_dtor(types, funcs, uids, prog, glob_ctor)
 }
+
+//! Names, for the caller to panic on, the DllName.glob() address globals nothing fills at load: LLVM
+//! folds their loads to null, proves the body UB and drops it to a zero-length `unreachable` that
+//! shares an address with the next function, so das_aot_register binds that hash to alien code.
+def public collect_unresolved_address_globals(mod : LLVMOpaqueModule?) : array<string> {
+    var bad : array<string>
+    var glob = LLVMGetFirstGlobal(mod)
+    while (glob != null) {
+        let init = LLVMGetInitializer(glob)
+        if (init != null && LLVMIsNull(init) != 0) {
+            var name_len = 0ul
+            let name = LLVMGetValueName2(glob, safe_addr(name_len))
+            if (name |> ends_with(" glob")) {
+                var loaded = false
+                var escaped = false
+                var use = LLVMGetFirstUse(glob)
+                while (use != null) {
+                    let user = LLVMGetUser(use)
+                    if (LLVMIsALoadInst(user) != null) {
+                        loaded = true
+                    } else {
+                        escaped = true   // stored into, or the address itself is passed on
+                    }
+                    use = LLVMGetNextUse(use)
+                }
+                if (loaded && !escaped) {
+                    bad |> push(clone_string(name))
+                }
+            }
+        }
+        glob = LLVMGetNextGlobal(glob)
+    }
+    return <- bad
+}

modules/dasLLVM/daslib/llvm_boost.das

@@ -183,27 +183,6 @@ def StructType(types : PrimitiveTypes?; var fields : array<LLVMOpaqueType?>) {
 }
 
 
-def LLVMBuildSRemInt32(builder : LLVMOpaqueBuilder?; var types : PrimitiveTypes?; lhs, rhs : LLVMOpaqueValue?; name : string) {
-    // convert lhs % rhs to lhs - int(double(lhs)/double(rhs))*rhs
-    let l = LLVMBuildSIToFP(builder, lhs, types.t_double, "")
-    let r = LLVMBuildSIToFP(builder, rhs, types.t_double, "")
-    let d = LLVMBuildFDiv(builder, l, r, "")
-    let i = LLVMBuildFPToSI(builder, d, types.t_int32, "")
-    let m = LLVMBuildMul(builder, i, rhs, "")
-    return LLVMBuildSub(builder, lhs, m, name)
-}
-
-
-def LLVMBuildURemUInt32(builder : LLVMOpaqueBuilder?; var types : PrimitiveTypes?; lhs, rhs : LLVMOpaqueValue?; name : string) {
-    // convert lhs % rhs to lhs - uint(double(lhs)/double(rhs))*rhs
-    let l = LLVMBuildUIToFP(builder, lhs, types.t_double, "")
-    let r = LLVMBuildUIToFP(builder, rhs, types.t_double, "")
-    let d = LLVMBuildFDiv(builder, l, r, "")
-    let i = LLVMBuildFPToUI(builder, d, types.t_int32, "")
-    let m = LLVMBuildMul(builder, i, rhs, "")
-    return LLVMBuildSub(builder, lhs, m, name)
-}
-
 def LLVMIsVector3(typ : LLVMOpaqueType?) : bool {
     return (LLVMGetTypeKind(typ) == LLVMTypeKind.LLVMVectorTypeKind) && (LLVMGetVectorSize(typ) == 3u)
 }

modules/dasLLVM/daslib/llvm_exe.das

@@ -413,6 +413,18 @@ class public CollectExternVisitor : AstVisitor {
         }
     }
 
+    // `new [[Handle...]]` allocates through the annotation the same way ExprNew does, and the
+    // emitter names that slot after the ascend node - so it needs its own resolution.
+    def override preVisitExprAscend(expr : ExprAscend?) {
+        assume subT = expr.subexpr._type
+        if (subT.isHandle) {
+            let name = uid.get_ascend_new(expr)
+            if (initialize(name)) {
+                register_new_from_annotation(name, subT.annotation)
+            }
+        }
+    }
+
     def override preVisitExprNew(expr : ExprNew?) {
         if (expr.typeexpr.isHandle) {
             let name = uid.get_new(expr.typeexpr)

modules/dasLLVM/daslib/llvm_jit.das

@@ -3256,35 +3256,19 @@ class public LlvmJitVisitor : AstVisitor {
                 }
             } elif (expr.op == "%") {
                 if (opType.isSignedInteger || (opType.isVectorType && opType.vectorBaseType == Type.tInt)) {
-                    if (opType.baseType == Type.tInt) {
-                        setE(expr, LLVMBuildSRemInt32(g_builder, types, left, right, ""))
-                    } else {
-                        setE(expr, LLVMBuildSRem(g_builder, left, right, ""))
-                    }
+                    setE(expr, LLVMBuildSRem(g_builder, left, right, ""))
                 } elif (opType.isUnsignedInteger || (opType.isVectorType && opType.vectorBaseType == Type.tUInt)) {
-                    if (opType.baseType == Type.tUInt) {
-                        setE(expr, LLVMBuildURemUInt32(g_builder, types, left, right, ""))
-                    } else {
-                        setE(expr, LLVMBuildURem(g_builder, left, right, ""))
-                    }
+                    setE(expr, LLVMBuildURem(g_builder, left, right, ""))
                 } elif (opType.isFloatOrDouble || (opType.isVectorType && opType.vectorBaseType == Type.tFloat)) {
                     setE(expr, LLVMBuildFRem(g_builder, left, right, ""))
                 } else {
                     failed_E(expr, "unsupported % type {describe(opType)}")
                 }
             } elif (expr.op == "%=") {
                 if (opType.isSignedInteger || (opType.isVectorType && opType.vectorBaseType == Type.tInt)) {
-                    if (opType.baseType == Type.tInt) {
-                        setE(expr, LLVMBuildStoreAligned(g_builder, LLVMBuildSRemInt32(g_builder, types, r2v_left, right, ""), left, uint(expr.left._type.alignOf)))
-                    } else {
-                        setE(expr, LLVMBuildStoreAligned(g_builder, LLVMBuildSRem(g_builder, r2v_left, right, ""), left, uint(expr.left._type.alignOf)))
-                    }
+                    setE(expr, LLVMBuildStoreAligned(g_builder, LLVMBuildSRem(g_builder, r2v_left, right, ""), left, uint(expr.left._type.alignOf)))
                 } elif (opType.isUnsignedInteger || (opType.isVectorType && opType.vectorBaseType == Type.tUInt)) {
-                    if (opType.baseType == Type.tUInt) {
-                        setE(expr, LLVMBuildStoreAligned(g_builder, LLVMBuildURemUInt32(g_builder, types, r2v_left, right, ""), left, uint(expr.left._type.alignOf)))
-                    } else {
-                        setE(expr, LLVMBuildStoreAligned(g_builder, LLVMBuildURem(g_builder, r2v_left, right, ""), left, uint(expr.left._type.alignOf)))
-                    }
+                    setE(expr, LLVMBuildStoreAligned(g_builder, LLVMBuildURem(g_builder, r2v_left, right, ""), left, uint(expr.left._type.alignOf)))
                 } elif (opType.isFloatOrDouble || (opType.isVectorType && opType.vectorBaseType == Type.tFloat)) {
                     setE(expr, LLVMBuildStoreAligned(g_builder, LLVMBuildFRem(g_builder, r2v_left, right, ""), left, uint(expr.left._type.alignOf)))
                 } else {

modules/dasLLVM/daslib/llvm_jit_code.das

@@ -1,6 +1,7 @@
 options gen2
 options indenting = 4
 options strict_smart_pointers = false
+options no_global_variables = false
 
 module llvm_jit_code shared private
 

modules/dasLLVM/daslib/llvm_jit_intrin.das

@@ -7,6 +7,7 @@ options unsafe_table_lookup = false
 // intrinsic emitters carry per-function lowering contracts (gates, exactness preconditions,
 // saturation bounds) that don't compress to the 3-line cap — keep them at the emitters
 options _comment_hygiene = false
+options no_global_variables = false
 
 module llvm_jit_intrin shared private
 

modules/dasLLVM/daslib/llvm_jit_run.das

@@ -2,6 +2,7 @@ options gen2
 options indenting = 4
 options strict_smart_pointers = false
 options stack = 4_194_304
+options no_global_variables = false
 
 require llvm/daslib/llvm_boost
 require llvm/daslib/llvm_dll_utils
@@ -36,11 +37,11 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole
 // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das,
 // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump
 // makes every previously written DLL miss the cache on the next run and get GC'd.
-let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x59ul   // aarch64 hosts append +i8mm when cpu_supports says so (0x58: [hint(unsafe_division_check)] drops the sdiv/srem guards)
+let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5aul   // srem/urem for 32-bit %, Context grew the adBySid memo (0x59: aarch64 hosts append +i8mm when cpu_supports says so)
 
 // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources
 // (normalized to LF; file list in the test)
-let LLVM_JIT_EMITTER_HASH : uint64 = 0xfcc23a40234a227ul
+let LLVM_JIT_EMITTER_HASH : uint64 = 0x5a3d11a87421b38eul
 
 let JIT_FNV_PRIME : uint64 = 1099511628211ul
 
@@ -187,6 +188,8 @@ def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_
     }
     h = (h ^ uint64(CONTEXT_OFFSET_OF_GLOBALS)) * JIT_FNV_PRIME
     h = (h ^ uint64(CONTEXT_OFFSET_OF_SHARED)) * JIT_FNV_PRIME
+    h = (h ^ uint64(CONTEXT_OFFSET_OF_STOP_FLAGS)) * JIT_FNV_PRIME
+    h = (h ^ uint64(CONTEXT_OFFSET_OF_EVAL_TOP)) * JIT_FNV_PRIME
     // full host-CPU identity when the TargetMachine bakes it - the cpuid bits below cover only the matrix-tier features; a wider box's object is an illegal instruction on a narrower one
     if (use_host_cpu) {
         let host_cpu = LLVMGetHostCPUName()
@@ -687,17 +690,22 @@ def private run_split_codegen(prog : Program?; ctx : Context?; funcs : array<Fun
     return <- (objs <- objs, t_irgen = tIr, t_optemit = tOptEmit)
 }
 
+def private jit_selects(fun : FunctionPtr; rqj : bool; aot_host : bool; jit_all : bool; var ctx : Context?) : bool {
+    if (rqj) return true
+    if (!aot_host) return jit_all
+    return !is_aot_function(get_function_by_mangled_name_hash(hash(get_mangled_name(fun)), *ctx))
+}
+
 //! Run the full JIT backend for an already-compiled & simulated program: collect the functions to
 //! JIT (honoring [jit]/[no_jit] and jit_all), content-address the DLL, generate and optimize IR,
 //! and emit a .dll / .exe / .wasm per the program's jit policies. Returns true.
 [macro_function]
 def public run_jit(prog : Program?; var ctx : Context?) : bool {  // nolint:STYLE037,STYLE038 - the JIT backend driver: flat policy/flag resolution feeding one linear codegen pipeline; the phases share the module/target state and do not survive a split
-    // AOT-object host binding: compiled jit-aware + policies.aot, so linkCppAot binds each
-    // function from the statically-linked .o (SimNode_Jit) -- no in-memory codegen needed
-    // here. (Emit mode sets jit_emit_object, never aot, so it skips this.)
-    if (prog.policies.aot && !prog.policies.jit_emit_object) {
+    // the LLVM-AOT object host needs no codegen here; a host asking for the runtime JIT does
+    if (prog.policies.aot && !prog.policies.jit_emit_object && !prog.policies.jit_enabled) {
         return true
     }
+    let aot_host = prog.policies.aot && prog.policies.jit_enabled
     var funcs : array<FunctionPtr>
     var disabled : array<FunctionPtr>
     var visitorDisabled = 0
@@ -821,7 +829,7 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool {  // nolint:STYL
                 // In object mode emit the whole used, non-noAot set (all modules +
                 // per-program generic instantiations), mirroring C++ AOT whole-program
                 // coverage; noAot functions interpret as Program::linkCppAot skips them.
-                if (fun.flags.used && (jit_all_functions || rqj) && (!emit_aot_object || !fun.flags.noAot)) {
+                if (fun.flags.used && (!emit_aot_object || !fun.flags.noAot) && jit_selects(fun, rqj, aot_host, jit_all_functions, ctx)) {
                     if (!fun.moreFlags.requestNoJit) {
                         disableJitVisitor.disable = false
                         // if DisableJitVisitor ever grows a preVisitFunction check again, it must
@@ -965,6 +973,14 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool {  // nolint:STYL
                 aot_dtor = pair._1
             }
             generate_global_ctors_dtors(g_prim_t, emit_aot_object ? false : !(use_dll || gen_exe), aot_ctor, aot_dtor)
+            if (emit_aot_object) {
+                // before optimization: the folding this catches also deletes the evidence
+                let unresolved = collect_unresolved_address_globals(g_mod)
+                if (!(unresolved |> empty())) {
+                    reset_jit_globals_after_failure()
+                    panic("Internal jit error. AOT address globals left unresolved by the globinit ctor: {join(unresolved, ", ")}\n")
+                }
+            }
             if (g_jit_fast_math) {
                 apply_fast_math_to_module(g_mod)   // EXPERIMENT: stamp all FP ops fast (non-bit-exact ceiling)
             }
@@ -1051,9 +1067,8 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool {  // nolint:STYL
         jit_log_report(funcs, recompile_prog, opt_level, tier_known, jit_split, split_parts, log_jit_time, totalTime,
             t_hash, t_init, t_declare, t_probe, t_irgen, t_opt, t_emit, t_install, t_finalize)
     } else {
-        // a -jit run that jitted NOTHING is indistinguishable from jit working (just slower) -
-        // the whole-program interpret must be loud, not inferred from timings
-        to_log(LOG_ERROR, "LLVM JIT: 0 functions to jit ({length(disabled)} marked no_jit, {visitorDisabled} disabled by content) - the WHOLE program runs interpreted\n")
+        // a -jit run that jitted NOTHING looks like jit working, only slower, so say it
+        to_log(LOG_INFO, "LLVM JIT: 0 functions to jit ({length(disabled)} marked no_jit, {visitorDisabled} disabled by content) - the WHOLE program runs interpreted\n")
     }
     return true
 }

modules/dasLLVM/daslib/llvm_tune.das

@@ -1,6 +1,10 @@
 options gen2
 options indenting = 4
 options no_aot = true
+options no_global_variables = false
+// the sidecar reader hands back a table built from json values, which the aliasing pass sees as
+// reaching json's null sentinel - a host that compiles with no_aliasing must still load the tuner
+options no_aliasing = false
 
 module llvm_tune shared public
 

modules/dasLLVM/daslib/llvm_user_modules.das

@@ -1,6 +1,7 @@
 options gen2
 options indenting = 4
 options _dasllama_internal = true
+options no_global_variables = false
 
 module llvm_user_modules shared private