Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion modules/authentication/src/handlers/twoFa.ts
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,7 @@ export class TwoFa implements IAuthenticationStrategy {
}

private async enableAuthenticator2Fa(user: User): Promise<string> {
const secret = await node2fa.generateSecret({
const secret = node2fa.generateSecret({
//to do: add logic for app name insertion
name: 'Conduit',
// add another string when mail is not available
Expand Down
2 changes: 1 addition & 1 deletion modules/functions/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"build:bundle": "rimraf bundle && node ../../libraries/service-bundle/dist/cli.js generate-manifest && tsup && node ../../libraries/service-bundle/dist/cli.js copy-assets && node ../../libraries/service-bundle/dist/cli.js generate-lockfile",
"prepare": "npm run build",
"generateTypes": "sh build.sh",
"test": "tsc -p tsconfig.test.json && node --test dist-test/controllers/*.test.js",
"test": "tsc -p tsconfig.test.json && node --test dist-test/__tests__/*.test.js",
"build:docker": "docker build -t ghcr.io/conduitplatform/functions:latest -f ./Dockerfile ../../ && docker push ghcr.io/conduitplatform/functions:latest"
},
"directories": {
Expand Down
1 change: 0 additions & 1 deletion modules/functions/src/Functions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ export default class Functions extends ManagedModule<Config> {
if (moduleName !== 'router' || !serving || this.isRunning) return;
this.isRunning = true;
this.functionsController = new FunctionController(this.grpcServer, this.grpcSdk);
CronQueueController.getInstance(this.grpcSdk);
this.adminRouter = new AdminHandlers(
this.grpcServer,
this.grpcSdk,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
parseCronJobFunctionId,
planCronSync,
validateCronPattern,
} from './cron.utils.js';
} from '../controllers/cron.utils.js';

describe('cron.utils', () => {
describe('parseCronJobFunctionId', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk';
import { FunctionExecutions } from '../models/index.js';
import type { Functions } from '../models/index.js';
import { compileFunctionCode, executeBackgroundFunction } from './utils.js';
import { compileFunctionCode, executeBackgroundFunction } from '../controllers/utils.js';

const originalGetInstance = FunctionExecutions.getInstance;

Expand Down
123 changes: 57 additions & 66 deletions modules/functions/src/controllers/cronQueue.controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,40 @@ export class CronQueueController {
return this.cronQueue.getRepeatableJobs();
}

private async executeCronJob(job: Job<{ functionId: string }>): Promise<void> {
const func = await Functions.getInstance().findOne(
{ _id: job.data.functionId },
{ readPreference: 'primary' },
);
if (!func || func.functionType !== 'cron') {
return;
}
const cronPattern = getCronPatternFromInputs(func.inputs);
if (!cronPattern) {
ConduitGrpcSdk.Logger.warn(
`Cron function ${func.name} (${func._id}) has no pattern; skipping tick`,
);
return;
}
const compiled =
this.compiledFunctions.get(func._id) ?? compileFunctionCode(func.functionCode);
const scheduledAt = new Date().toISOString();
ConduitGrpcSdk.Logger.log(
`Cron tick for ${func.name} (${cronPattern}) at ${scheduledAt}`,
);
await executeBackgroundFunction(
func,
{
scheduledAt,
cronPattern,
trigger: 'cron',
},
compiled,
this.grpcSdk,
);
ConduitGrpcSdk.Logger.log(`Cron execution completed for ${func.name}`);
}

async ensureWorker(lockDuration: number): Promise<Worker | undefined> {
if (!this.shouldSchedule()) {
return this.cronWorker;
Expand All @@ -73,50 +107,14 @@ export class CronQueueController {
this.cronWorker = undefined;
this.workerLockDuration = 0;
}
this.cronWorker = new Worker(
CRON_QUEUE_NAME,
async (job: Job<{ functionId: string }>) => {
const func = await Functions.getInstance().findOne(
{ _id: job.data.functionId },
{ readPreference: 'primary' },
);
if (!func || func.functionType !== 'cron') {
return;
}
const cronPattern = getCronPatternFromInputs(func.inputs);
if (!cronPattern) {
ConduitGrpcSdk.Logger.warn(
`Cron function ${func.name} (${func._id}) has no pattern; skipping tick`,
);
return;
}
const compiled =
this.compiledFunctions.get(func._id) ?? compileFunctionCode(func.functionCode);
const scheduledAt = new Date().toISOString();
ConduitGrpcSdk.Logger.log(
`Cron tick for ${func.name} (${cronPattern}) at ${scheduledAt}`,
);
await executeBackgroundFunction(
func,
{
scheduledAt,
cronPattern,
trigger: 'cron',
},
compiled,
this.grpcSdk,
);
ConduitGrpcSdk.Logger.log(`Cron execution completed for ${func.name}`);
},
{
concurrency: 1,
lockDuration,
maxStalledCount: 0,
removeOnComplete: { age: 3600, count: 1000 },
removeOnFail: { age: 24 * 3600 },
connection: this.redisConnection,
},
);
this.cronWorker = new Worker(CRON_QUEUE_NAME, job => this.executeCronJob(job), {
concurrency: 1,
lockDuration,
maxStalledCount: 0,
removeOnComplete: { age: 3600, count: 1000 },
removeOnFail: { age: 24 * 3600 },
connection: this.redisConnection,
});
this.workerLockDuration = lockDuration;
this.setupWorkerEventHandlers(this.cronWorker);
if (!this.shouldSchedule()) {
Expand Down Expand Up @@ -220,25 +218,21 @@ export class CronQueueController {
}
}

private lockDurationFromTimeouts(timeouts: Array<number | undefined>): number {
let maxTimeout = DEFAULT_FUNCTION_TIMEOUT_MS;
for (const timeout of timeouts) {
maxTimeout = Math.max(maxTimeout, timeout ?? DEFAULT_FUNCTION_TIMEOUT_MS);
}
return maxTimeout + LOCK_BUFFER_MS;
}

private async lockDurationForCronFunctions(): Promise<number> {
type CronTimeout = Pick<Functions, 'timeout'>;
const cronDocs = (await Functions.getInstance().findMany(
{ functionType: 'cron' },
{ select: 'timeout', readPreference: 'primary' },
)) as CronTimeout[];
const maxTimeout = cronDocs.reduce(
(max, func) => Math.max(max, func.timeout ?? DEFAULT_FUNCTION_TIMEOUT_MS),
DEFAULT_FUNCTION_TIMEOUT_MS,
);
return maxTimeout + LOCK_BUFFER_MS;
}

private lockDurationFromFunctions(cronFunctions: Functions[]): number {
const maxTimeout = cronFunctions.reduce(
(max, func) => Math.max(max, func.timeout ?? DEFAULT_FUNCTION_TIMEOUT_MS),
DEFAULT_FUNCTION_TIMEOUT_MS,
);
return maxTimeout + LOCK_BUFFER_MS;
return this.lockDurationFromTimeouts(cronDocs.map(func => func.timeout));
}

private async reconcileCronJobs(): Promise<number> {
Expand Down Expand Up @@ -270,9 +264,6 @@ export class CronQueueController {
try {
if (item.existingKey) {
await this.cronQueue.removeRepeatableByKey(item.existingKey);
updated += 1;
} else {
registered += 1;
}
await this.cronQueue.add(
CRON_JOB_NAME,
Expand All @@ -284,23 +275,23 @@ export class CronQueueController {
removeOnFail: { age: 24 * 3600 },
},
);
if (item.existingKey) {
updated += 1;
} else {
registered += 1;
}
} catch (err) {
ConduitGrpcSdk.Logger.error(
`Failed to schedule cron job ${item.jobId}: ${(err as Error).message}`,
);
errors += 1;
if (item.existingKey) {
updated -= 1;
} else {
registered -= 1;
}
}
}

ConduitGrpcSdk.Logger.log(
`Cron sync complete: registered=${registered}, updated=${updated}, unchanged=${plan.unchangedJobIds.length}, removed=${removed}, skipped=${plan.skipped.length}, errors=${errors}`,
);
return this.lockDurationFromFunctions(cronFunctions);
return this.lockDurationFromTimeouts(cronFunctions.map(func => func.timeout));
}

private setupWorkerEventHandlers(worker: Worker): void {
Expand Down
127 changes: 59 additions & 68 deletions modules/functions/src/controllers/function.controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ import {
ConduitSocketOptions,
} from '@conduitplatform/grpc-sdk';
import {
ConfigController,
GrpcServer,
RequestHandlers,
RoutingManager,
SocketEventHandler,
} from '@conduitplatform/module-tools';
import { ConfigController } from '@conduitplatform/module-tools';

import { Functions } from '../models/index.js';
import { CronQueueController } from './cronQueue.controller.js';
Expand All @@ -33,8 +33,10 @@ type Middleware = {
handler: RequestHandlers;
};

type FunctionRoute = Route | Socket | Middleware;

export class FunctionController {
private functionRoutes: (Route | Socket | Middleware)[] = [];
private functionRoutes: FunctionRoute[] = [];
private readonly compiledCronFunctions = new Map<string, CompiledUserFunction>();

private _routingManager: RoutingManager;
Expand All @@ -54,77 +56,66 @@ export class FunctionController {
});
}

refreshRoutes() {
return Functions.getInstance()
.findMany({}, { readPreference: 'primary' })
.then(async r => {
if (!r || r.length == 0) {
ConduitGrpcSdk.Logger.log('No functions to register');
}
this.functionRoutes = [];
this.compiledCronFunctions.clear();
async refreshRoutes() {
try {
const functions = await Functions.getInstance().findMany(
{},
{ readPreference: 'primary' },
);
if (!functions || functions.length === 0) {
ConduitGrpcSdk.Logger.log('No functions to register');
}
this.functionRoutes = [];
this.compiledCronFunctions.clear();

for (const func of r) {
try {
if (func.functionType === 'cron') {
try {
this.compiledCronFunctions.set(func._id, tryPrepareCronFunction(func));
} catch (err) {
ConduitGrpcSdk.Logger.error(
`Failed to prepare cron function ${func.name} (${func._id})`,
);
ConduitGrpcSdk.Logger.error(err as Error);
}
continue;
}
const route = createFunctionRoute(func, this.grpcSdk);
if (route) {
this.functionRoutes.push(route as any);
}
} catch (err) {
ConduitGrpcSdk.Logger.error(
`Failed to process function ${func.name} (${func._id}); skipping`,
);
ConduitGrpcSdk.Logger.error(err as Error);
for (const func of functions) {
try {
if (func.functionType === 'cron') {
this.compiledCronFunctions.set(func._id, tryPrepareCronFunction(func));
continue;
}
}
this._routingManager.clear();
this.functionRoutes.forEach(route => {
if ((route as Socket).events) {
this._routingManager.socket(
(route as Socket).input,
(route as Socket).events,
);
} else if (!(route as Middleware).hasOwnProperty('returnType')) {
this._routingManager.middleware(
(route as Middleware).input,
(route as Middleware).handler,
);
} else {
this._routingManager.route(
(route as Route).input,
(route as Route).returnType,
(route as Route).handler,
);
}
});
await this._routingManager.registerRoutes();
ConduitGrpcSdk.Logger.log('Refreshed routes');

if (ConfigController.getInstance().config.active) {
const cronQueue = CronQueueController.getInstance(this.grpcSdk);
cronQueue.setCompiledFunctions(this.compiledCronFunctions);
if (ConfigController.getInstance().config.active) {
await cronQueue.syncCronJobs();
const route = createFunctionRoute(func, this.grpcSdk);
if (route) {
this.functionRoutes.push(route as FunctionRoute);
}
} catch (err) {
ConduitGrpcSdk.Logger.error(
`Failed to process function ${func.name} (${func._id}); skipping`,
);
ConduitGrpcSdk.Logger.error(err as Error);
}
}
this._routingManager.clear();
this.functionRoutes.forEach(route => {
if ((route as Socket).events) {
this._routingManager.socket((route as Socket).input, (route as Socket).events);
} else if (!(route as Middleware).hasOwnProperty('returnType')) {
this._routingManager.middleware(
(route as Middleware).input,
(route as Middleware).handler,
);
} else {
this._routingManager.route(
(route as Route).input,
(route as Route).returnType,
(route as Route).handler,
);
}
})
.catch((err: Error) => {
ConduitGrpcSdk.Logger.error(
'Something went wrong when loading functions to the router',
);
ConduitGrpcSdk.Logger.error(err);
});
await this._routingManager.registerRoutes();
ConduitGrpcSdk.Logger.log('Refreshed routes');

if (ConfigController.getInstance().config.active) {
const cronQueue = CronQueueController.getInstance(this.grpcSdk);
cronQueue.setCompiledFunctions(this.compiledCronFunctions);
await cronQueue.syncCronJobs();
}
} catch (err) {
ConduitGrpcSdk.Logger.error(
'Something went wrong when loading functions to the router',
);
ConduitGrpcSdk.Logger.error(err as Error);
}
}

refreshEndpoints(): void {
Expand Down
1 change: 0 additions & 1 deletion modules/functions/src/migrations/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
import { Functions } from '../models/index.js';
import { getCronPatternFromInputs } from '../controllers/cron.utils.js';

export async function runMigrations(_grpcSdk: ConduitGrpcSdk) {

Check notice on line 5 in modules/functions/src/migrations/index.ts

View check run for this annotation

codefactor.io / CodeFactor

modules/functions/src/migrations/index.ts#L5

'_grpcSdk' is defined but never used. (@typescript-eslint/no-unused-vars)
void _grpcSdk;
const cronFunctions = await Functions.getInstance().findMany({
functionType: 'cron',
});
Expand Down
9 changes: 8 additions & 1 deletion modules/functions/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -67,5 +67,12 @@
"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist", "bundle", "tsup.config.ts", "src/**/*.test.ts"]
"exclude": [
"node_modules",
"dist",
"bundle",
"tsup.config.ts",
"src/**/*.test.ts",
"src/__tests__"
]
}
Loading
Loading