Skip to content

Commit d501aea

Browse files
neSpeccneSpecc
committed
Merge branch 'master' into improve-billing-notifications
2 parents c8a5a99 + 4b11afa commit d501aea

6 files changed

Lines changed: 413 additions & 2 deletions

File tree

‎src/metrics/graphql.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import client from 'prom-client';
22
import { ApolloServerPlugin, GraphQLRequestContext, GraphQLRequestListener } from 'apollo-server-plugin-base';
33
import { GraphQLError } from 'graphql';
44
import HawkCatcher from '@hawk.so/nodejs';
5+
import { notifySlowOperation } from './slowOperationAlert';
6+
import { buildGraphqlRequestContext, formatGraphqlErrorsForAlert } from './graphqlRequestDetails';
57
/**
68
* GraphQL operation duration histogram
79
* Tracks GraphQL operation duration by operation name and type
@@ -93,6 +95,19 @@ export const graphqlMetricsPlugin: ApolloServerPlugin = {
9395
},
9496
});
9597

98+
notifySlowOperation(
99+
`Slow GraphQL operation: ${operationType} ${operationName}`,
100+
durationMs,
101+
{
102+
operationType,
103+
operationName,
104+
...buildGraphqlRequestContext(ctx),
105+
...(hasErrors && {
106+
errors: formatGraphqlErrorsForAlert(ctx.errors!),
107+
}),
108+
}
109+
);
110+
96111
// Track errors if any
97112
if (hasErrors) {
98113
ctx.errors!.forEach((error: GraphQLError) => {

‎src/metrics/graphqlRequestDetails.ts‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
import { GraphQLRequestContext } from 'apollo-server-plugin-base';
2+
import { GraphQLError } from 'graphql';
3+
import { ResolverContextBase } from '../types/graphql';
4+
import { truncateText } from './slowOperationAlert';
5+
6+
const MAX_ALERT_ERRORS = 10;
7+
const MAX_ALERT_ERRORS_LENGTH = 1200;
8+
9+
const SENSITIVE_VARIABLE_KEYS = new Set([
10+
'password',
11+
'token',
12+
'accesstoken',
13+
'refreshtoken',
14+
'secret',
15+
'authorization',
16+
]);
17+
18+
const HIGHLIGHTED_VARIABLE_KEYS = new Set([
19+
'projectid',
20+
'workspaceid',
21+
'eventid',
22+
'originaleventid',
23+
'release',
24+
'search',
25+
'assignee',
26+
'cursor',
27+
]);
28+
29+
/**
30+
* Redact sensitive GraphQL variables before sending alerts.
31+
*
32+
* @param value - variable value
33+
* @param key - variable key
34+
* @returns sanitized value
35+
*/
36+
function sanitizeVariableValue(value: unknown, key: string): unknown {
37+
if (SENSITIVE_VARIABLE_KEYS.has(key.toLowerCase())) {
38+
return '[redacted]';
39+
}
40+
41+
if (Array.isArray(value)) {
42+
return value.map((item, index) => sanitizeVariableValue(item, `${key}[${index}]`));
43+
}
44+
45+
if (value && typeof value === 'object') {
46+
// eslint-disable-next-line @typescript-eslint/no-use-before-define
47+
return sanitizeVariables(value as Record<string, unknown>);
48+
}
49+
50+
return value;
51+
}
52+
53+
/**
54+
* Redact sensitive GraphQL variables before sending alerts.
55+
*
56+
* @param variables - GraphQL request variables
57+
* @returns sanitized variables
58+
*/
59+
function sanitizeVariables(
60+
variables: Record<string, unknown> | null | undefined
61+
): Record<string, unknown> {
62+
/**
63+
* Null / non-object values are treated as empty — many clients send
64+
* `variables: null` for operations without variables, and arrays are not a
65+
* valid GraphQL variables map.
66+
*/
67+
if (variables == null || typeof variables !== 'object' || Array.isArray(variables)) {
68+
return {};
69+
}
70+
71+
return Object.fromEntries(
72+
Object.entries(variables).map(([key, value]) => [key, sanitizeVariableValue(value, key)])
73+
);
74+
}
75+
76+
/**
77+
* Extract useful identifiers from nested GraphQL variables.
78+
*
79+
* @param value - variable value
80+
* @param prefix - nested path prefix
81+
* @param result - accumulator for extracted ids
82+
* @returns extracted identifiers
83+
*/
84+
function collectHighlightedIds(
85+
value: unknown,
86+
prefix = '',
87+
result: Record<string, string | number | boolean> = {}
88+
): Record<string, string | number | boolean> {
89+
if (!value || typeof value !== 'object') {
90+
return result;
91+
}
92+
93+
for (const [key, nestedValue] of Object.entries(value as Record<string, unknown>)) {
94+
const path = prefix ? `${prefix}.${key}` : key;
95+
96+
if (
97+
HIGHLIGHTED_VARIABLE_KEYS.has(key.toLowerCase()) &&
98+
(typeof nestedValue === 'string' || typeof nestedValue === 'number' || typeof nestedValue === 'boolean')
99+
) {
100+
result[path] = nestedValue;
101+
continue;
102+
}
103+
104+
if (nestedValue && typeof nestedValue === 'object' && !Array.isArray(nestedValue)) {
105+
collectHighlightedIds(nestedValue, path, result);
106+
}
107+
}
108+
109+
return result;
110+
}
111+
112+
/**
113+
* Flatten GraphQL errors into a capped string for Hawk alert context.
114+
* sanitizeContext() only truncates top-level strings, not values nested in arrays.
115+
* Reserves space for an omitted-count suffix and truncateText()'s ellipsis.
116+
*
117+
* @param errors - GraphQL errors from the request
118+
* @returns flattened and truncated error messages
119+
*/
120+
export function formatGraphqlErrorsForAlert(errors: readonly GraphQLError[]): string {
121+
const messages = errors.slice(0, MAX_ALERT_ERRORS).map((error) => error.message);
122+
const omittedCount = errors.length - messages.length;
123+
const omittedSuffix = omittedCount > 0 ? `; …(+${omittedCount} more)` : '';
124+
const maxMessagesLength = Math.max(0, MAX_ALERT_ERRORS_LENGTH - omittedSuffix.length - 1);
125+
126+
return `${truncateText(messages.join('; '), maxMessagesLength)}${omittedSuffix}`;
127+
}
128+
129+
/**
130+
* Build request context for slow GraphQL operation alerts.
131+
*
132+
* @param ctx - GraphQL request context
133+
* @returns alert context
134+
*/
135+
export function buildGraphqlRequestContext(ctx: GraphQLRequestContext): Record<string, unknown> {
136+
const context = ctx.context as ResolverContextBase | undefined;
137+
const variables = sanitizeVariables(
138+
ctx.request.variables as Record<string, unknown> | null | undefined
139+
);
140+
const highlightedIds = collectHighlightedIds(variables);
141+
const alertContext: Record<string, unknown> = {};
142+
143+
if (context?.user?.id) {
144+
alertContext.userId = context.user.id;
145+
}
146+
147+
if (Object.keys(highlightedIds).length > 0) {
148+
alertContext.ids = highlightedIds;
149+
}
150+
151+
const variablesJson = JSON.stringify(variables);
152+
153+
if (variablesJson && variablesJson !== '{}') {
154+
alertContext.variables = truncateText(variablesJson, 1200);
155+
}
156+
157+
return alertContext;
158+
}

‎src/metrics/mongodb.ts‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import promClient from 'prom-client';
22
import { MongoClient, MongoClientOptions } from 'mongodb';
33
import { Effect, sgr } from '../utils/ansi';
44
import HawkCatcher from '@hawk.so/nodejs';
5+
import { notifySlowOperation, truncateText } from './slowOperationAlert';
56

67
/**
78
* MongoDB command duration histogram
@@ -156,6 +157,7 @@ function colorizeDuration(duration: number): string {
156157
*/
157158
interface StoredCommandInfo {
158159
formattedCommand: string;
160+
plainFormattedCommand: string;
159161
timestamp: number;
160162
}
161163

@@ -202,7 +204,8 @@ setInterval(cleanupStaleCommandInfo, COMMAND_INFO_TIMEOUT_MS);
202204
*/
203205
function storeCommandInfo(event: any): void {
204206
const collectionRaw = extractCollectionFromCommand(event.command, event.commandName);
205-
const collection = sgr(normalizeCollectionName(collectionRaw), Effect.ForegroundGreen);
207+
const collectionName = normalizeCollectionName(collectionRaw);
208+
const collection = sgr(collectionName, Effect.ForegroundGreen);
206209
const db = event.databaseName || 'unknown db';
207210
const commandName = sgr(event.commandName, Effect.ForegroundRed);
208211
const filter = event.command.filter;
@@ -212,11 +215,12 @@ function storeCommandInfo(event: any): void {
212215
const params = filter || update || pipeline;
213216
const paramsStr = formatParams(params);
214217
const projectionStr = projection ? ` projection: ${formatParams(projection)}` : '';
215-
218+
const plainFormattedCommand = `[${event.requestId}] ${db}.${collectionName}.${event.commandName}(${paramsStr})${projectionStr}`;
216219
const formattedCommand = `[${event.requestId}] ${db}.${collection}.${commandName}(${paramsStr})${projectionStr}`;
217220

218221
commandInfoMap.set(event.requestId, {
219222
formattedCommand,
223+
plainFormattedCommand,
220224
timestamp: Date.now(),
221225
});
222226
}
@@ -232,9 +236,24 @@ function logCommandSucceeded(event: any): void {
232236

233237
if (info) {
234238
console.log(`${info.formattedCommand} ✓ ${durationStr}`);
239+
notifySlowOperation(
240+
`Slow MongoDB command: ${event.commandName}`,
241+
event.duration,
242+
{
243+
requestId: event.requestId,
244+
command: truncateText(info.plainFormattedCommand),
245+
}
246+
);
235247
commandInfoMap.delete(event.requestId);
236248
} else {
237249
console.log(`[${event.requestId}] ${event.commandName} ✓ ${durationStr}`);
250+
notifySlowOperation(
251+
`Slow MongoDB command: ${event.commandName}`,
252+
event.duration,
253+
{
254+
requestId: event.requestId,
255+
}
256+
);
238257
}
239258
}
240259

@@ -250,9 +269,26 @@ function logCommandFailed(event: any): void {
250269

251270
if (info) {
252271
console.error(`${info.formattedCommand} ✗ ${errorMsg} ${durationStr}`);
272+
notifySlowOperation(
273+
`Slow MongoDB command: ${event.commandName}`,
274+
event.duration,
275+
{
276+
requestId: event.requestId,
277+
command: truncateText(info.plainFormattedCommand),
278+
error: truncateText(errorMsg, 500),
279+
}
280+
);
253281
commandInfoMap.delete(event.requestId);
254282
} else {
255283
console.error(`[${event.requestId}] ${event.commandName} ✗ ${errorMsg} ${durationStr}`);
284+
notifySlowOperation(
285+
`Slow MongoDB command: ${event.commandName}`,
286+
event.duration,
287+
{
288+
requestId: event.requestId,
289+
error: truncateText(errorMsg, 500),
290+
}
291+
);
256292
}
257293
}
258294

‎src/metrics/slowOperationAlert.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import HawkCatcher from '@hawk.so/nodejs';
2+
3+
export const SLOW_OPERATION_THRESHOLD_MS = 10000;
4+
const MAX_CONTEXT_STRING_LENGTH = 2500;
5+
6+
/**
7+
* Truncate text for slow operation context fields.
8+
*
9+
* @param value - text to truncate
10+
* @param maxLength - max allowed length
11+
* @returns truncated text
12+
*/
13+
function truncateText(value: string, maxLength = MAX_CONTEXT_STRING_LENGTH): string {
14+
if (value.length <= maxLength) {
15+
return value;
16+
}
17+
18+
return `${value.slice(0, maxLength)}…`;
19+
}
20+
21+
/**
22+
* Truncate long string values in alert context.
23+
*
24+
* @param context - alert context
25+
* @returns sanitized context
26+
*/
27+
function sanitizeContext(context: Record<string, unknown>): Record<string, unknown> {
28+
return Object.fromEntries(
29+
Object.entries(context).map(([key, value]) => {
30+
if (typeof value === 'string') {
31+
return [key, truncateText(value)];
32+
}
33+
34+
return [key, value];
35+
})
36+
);
37+
}
38+
39+
/**
40+
* Send slow operation alert to Hawk via HawkCatcher.
41+
*
42+
* @param message - short alert message
43+
* @param durationMs - operation duration in milliseconds
44+
* @param context - additional alert context
45+
*/
46+
export function notifySlowOperation(
47+
message: string,
48+
durationMs: number,
49+
context: Record<string, unknown> = {}
50+
): void {
51+
if (
52+
process.env.NODE_ENV === 'test' ||
53+
process.env.NODE_ENV === 'e2e' ||
54+
durationMs < SLOW_OPERATION_THRESHOLD_MS
55+
) {
56+
return;
57+
}
58+
59+
try {
60+
HawkCatcher.send(new Error(message), {
61+
durationMs,
62+
...sanitizeContext(context),
63+
});
64+
} catch (error) {
65+
console.log('Couldn\'t send slow operation alert to Hawk', error);
66+
}
67+
}
68+
69+
export { truncateText };

0 commit comments

Comments
 (0)