[Hubspot] - unit tests and caching for new Actions (#2393)

* adding 2 tests for custom events * adding more tests for customEvents * Add subscription metadata to execute bundle * starting to add caching * reverting cache stuff * reverting cache * dfsdf * saving progress * not test - caching for custom function * tested locally - ci failing * adding instrumentation * cache test added * fixing auth token * updating readme --------- Co-authored-by: Varadarajan V <[email protected]>
segmentio · Sep 24, 2024 · 3d28a5b · 3d28a5b
1 parent c538beb
commit 3d28a5b
Show file tree

Hide file tree

Showing 16 changed files with 1,240 additions and 368 deletions.
diff --git a/README.md b/README.md
@@ -534,6 +534,7 @@ The `perform` method accepts two arguments, (1) the request client instance (ext
 - `dataFeedCache` - DataFeedCache can only be used by internal Twilio/Segment employees. DataFeedCache cannot be used for Partner builds.
 - `transactionContext` - An object, containing transaction variables and a method to update transaction variables which are required for few segment developed actions. Transaction Context cannot be used for Partner builds.
 - `stateContext` - An object, containing context variables and a method to get and set context variables which are required for few segment developed actions. State Context cannot be used for Partner builds.
+- `subscriptionMetadata` - an object, containing variables which identify the instance of a Destination and Action as well as other metadata. Subscription Metadata cannot be used for Partner builds.
 
 A basic example:
 

diff --git a/packages/cli/src/lib/server.ts b/packages/cli/src/lib/server.ts
@@ -297,7 +297,8 @@ function setupRoutes(def: DestinationDefinition | null): void {
  audienceSettings: req.body.payload?.context?.personas?.audience_settings || {},
  mapping: mapping || req.body.payload || {},
  auth: req.body.auth || {},
- features: req.body.features || {}
+ features: req.body.features || {},
+ subscriptionMetadata: req.body.subscriptionMetadata || {}
  }
 
  if (Array.isArray(eventParams.data)) {

diff --git a/packages/core/src/create-test-integration.ts b/packages/core/src/create-test-integration.ts
@@ -1,7 +1,14 @@
 import { createTestEvent } from './create-test-event'
 import { StateContext, Destination, TransactionContext } from './destination-kit'
 import { mapValues } from './map-values'
-import type { DestinationDefinition, StatsContext, Logger, DataFeedCache, RequestFn } from './destination-kit'
+import type {
+ DestinationDefinition,
+ StatsContext,
+ Logger,
+ DataFeedCache,
+ RequestFn,
+ SubscriptionMetadata
+} from './destination-kit'
 import type { JSONObject } from './json-object'
 import type { SegmentEvent } from './segment-event'
 import { AuthTokens } from './destination-kit/parse-settings'
@@ -46,6 +53,7 @@ interface InputData<Settings> {
  dataFeedCache?: DataFeedCache
  transactionContext?: TransactionContext
  stateContext?: StateContext
+ subscriptionMetadata?: SubscriptionMetadata
 }
 
 class TestDestination<T, AudienceSettings = any> extends Destination<T, AudienceSettings> {
@@ -79,7 +87,8 @@ class TestDestination<T, AudienceSettings = any> extends Destination<T, Audience
  logger,
  dataFeedCache,
  transactionContext,
- stateContext
+ stateContext,
+ subscriptionMetadata
  }: InputData<T>
  ): Promise<Destination['responses']> {
  this.results = []
@@ -101,7 +110,8 @@ class TestDestination<T, AudienceSettings = any> extends Destination<T, Audience
  logger: logger ?? ({ info: noop, error: noop } as Logger),
  dataFeedCache: dataFeedCache ?? ({} as DataFeedCache),
  transactionContext: transactionContext ?? ({} as TransactionContext),
- stateContext: stateContext ?? ({} as StateContext)
+ stateContext: stateContext ?? ({} as StateContext),
+ subscriptionMetadata: subscriptionMetadata ?? ({} as SubscriptionMetadata)
  })
 
  const responses = this.responses
@@ -123,7 +133,8 @@ class TestDestination<T, AudienceSettings = any> extends Destination<T, Audience
  logger,
  dataFeedCache,
  transactionContext,
- stateContext
+ stateContext,
+ subscriptionMetadata
  }: Omit<InputData<T>, 'event'> & { events?: SegmentEvent[] }
  ): Promise<Destination['responses']> {
  this.results = []
@@ -149,7 +160,8 @@ class TestDestination<T, AudienceSettings = any> extends Destination<T, Audience
  logger: logger ?? ({} as Logger),
  dataFeedCache: dataFeedCache ?? ({} as DataFeedCache),
  transactionContext: transactionContext ?? ({} as TransactionContext),
- stateContext: stateContext ?? ({} as StateContext)
+ stateContext: stateContext ?? ({} as StateContext),
+ subscriptionMetadata: subscriptionMetadata ?? ({} as SubscriptionMetadata)
  })
 
  this.results = [

diff --git a/packages/core/src/destination-kit/action.ts b/packages/core/src/destination-kit/action.ts
@@ -25,7 +25,7 @@ import { validateSchema } from '../schema-validation'
 import { AuthTokens } from './parse-settings'
 import { ErrorCodes, IntegrationError, MultiStatusErrorReporter } from '../errors'
 import { removeEmptyValues } from '../remove-empty-values'
-import { Logger, StatsContext, TransactionContext, StateContext, DataFeedCache } from './index'
+import { Logger, StatsContext, TransactionContext, StateContext, DataFeedCache, SubscriptionMetadata } from './index'
 import { get } from '../get'
 
 type MaybePromise<T> = T | Promise<T>
@@ -213,6 +213,7 @@ interface ExecuteBundle<T = unknown, Data = unknown, AudienceSettings = any, Act
  dataFeedCache?: DataFeedCache | undefined
  transactionContext?: TransactionContext
  stateContext?: StateContext
+ subscriptionMetadata?: SubscriptionMetadata
 }
 
 type FillMultiStatusResponseInput = {
@@ -351,9 +352,9 @@ export class Action<Settings, Payload extends JSONLikeObject, AudienceSettings =
  audienceSettings: bundle.audienceSettings,
  hookOutputs,
  syncMode: isSyncMode(syncMode) ? syncMode : undefined,
- matchingKey: matchingKey ? String(matchingKey) : undefined
+ matchingKey: matchingKey ? String(matchingKey) : undefined,
+ subscriptionMetadata: bundle.subscriptionMetadata
  }
-
  // Construct the request client and perform the action
  const output = await this.performRequest(this.definition.perform, dataBundle)
  results.push({ data: output as JSONObject, output: 'Action Executed' })
@@ -451,6 +452,7 @@ export class Action<Settings, Payload extends JSONLikeObject, AudienceSettings =
  dataFeedCache: bundle.dataFeedCache,
  transactionContext: bundle.transactionContext,
  stateContext: bundle.stateContext,
+ subscriptionMetadata: bundle.subscriptionMetadata,
  hookOutputs,
  syncMode: isSyncMode(syncMode) ? syncMode : undefined,
  matchingKey: matchingKey ? String(matchingKey) : undefined

diff --git a/packages/core/src/destination-kit/index.ts b/packages/core/src/destination-kit/index.ts
@@ -194,6 +194,10 @@ export interface Subscription {
  partnerAction: string
  subscribe: string
  mapping?: JSONObject
+ ActionID?: string
+ ConfigID?: string
+ ID?: string
+ ProjectID?: string
 }
 
 export interface OAuth2ClientCredentials extends AuthTokens {
@@ -279,6 +283,13 @@ export type AuthenticationScheme<Settings = any> =
  | OAuth2Authentication<Settings>
  | OAuthManagedAuthentication<Settings>
 
+export type SubscriptionMetadata = {
+ actionConfigId?: string
+ destinationConfigId?: string
+ actionId?: string
+ sourceId?: string
+}
+
 interface EventInput<Settings> {
  readonly event: SegmentEvent
  readonly mapping: JSONObject
@@ -292,6 +303,7 @@ interface EventInput<Settings> {
  readonly dataFeedCache?: DataFeedCache
  readonly transactionContext?: TransactionContext
  readonly stateContext?: StateContext
+ readonly subscriptionMetadata?: SubscriptionMetadata
 }
 
 interface BatchEventInput<Settings> {
@@ -307,6 +319,7 @@ interface BatchEventInput<Settings> {
  readonly dataFeedCache?: DataFeedCache
  readonly transactionContext?: TransactionContext
  readonly stateContext?: StateContext
+ readonly subscriptionMetadata?: SubscriptionMetadata
 }
 
 export interface DecoratedResponse extends ModifiedResponse {
@@ -572,6 +585,7 @@ export class Destination<Settings = JSONObject, AudienceSettings = JSONObject> {
  {
  event,
  mapping,
+ subscriptionMetadata,
  settings,
  auth,
  features,
@@ -603,7 +617,8 @@ export class Destination<Settings = JSONObject, AudienceSettings = JSONObject> {
  logger,
  dataFeedCache,
  transactionContext,
- stateContext
+ stateContext,
+ subscriptionMetadata
  })
  }
 
@@ -612,6 +627,7 @@ export class Destination<Settings = JSONObject, AudienceSettings = JSONObject> {
  {
  events,
  mapping,
+ subscriptionMetadata,
  settings,
  auth,
  features,
@@ -644,7 +660,8 @@ export class Destination<Settings = JSONObject, AudienceSettings = JSONObject> {
  logger,
  dataFeedCache,
  transactionContext,
- stateContext
+ stateContext,
+ subscriptionMetadata
  })
  }
 
@@ -678,6 +695,12 @@ export class Destination<Settings = JSONObject, AudienceSettings = JSONObject> {
  const actionSlug = subscription.partnerAction
  const input = {
  mapping: subscription.mapping || {},
+ subscriptionMetadata: {
+ actionConfigId: subscription.ID,
+ destinationConfigId: subscription.ConfigID,
+ actionId: subscription.ActionID,
+ sourceId: subscription.ProjectID
+ } as SubscriptionMetadata,
  settings,
  auth,
  features: options?.features || {},

diff --git a/packages/core/src/destination-kit/types.ts b/packages/core/src/destination-kit/types.ts
@@ -1,4 +1,12 @@
-import { StateContext, Logger, StatsContext, TransactionContext, DataFeedCache, ActionHookType } from './index'
+import {
+ StateContext,
+ Logger,
+ StatsContext,
+ TransactionContext,
+ DataFeedCache,
+ ActionHookType,
+ SubscriptionMetadata
+} from './index'
 import type { RequestOptions } from '../request-client'
 import type { JSONLikeObject, JSONObject } from '../json-object'
 import { AuthTokens } from './parse-settings'
@@ -68,6 +76,7 @@ export interface ExecuteInput<
  readonly dataFeedCache?: DataFeedCache
  readonly transactionContext?: TransactionContext
  readonly stateContext?: StateContext
+ readonly subscriptionMetadata?: SubscriptionMetadata
 }
 
 export interface DynamicFieldResponse {

diff --git a/packages/destination-actions/package.json b/packages/destination-actions/package.json
@@ -56,7 +56,8 @@
  "kafkajs": "^2.2.4",
  "liquidjs": "^10.8.4",
  "lodash": "^4.17.21",
- "ssh2-sftp-client": "^9.1.0"
+ "ssh2-sftp-client": "^9.1.0",
+ "lru-cache": "10.4.3"
  },
  "jest": {
  "preset": "ts-jest",