[feature] Support logging for generic cloud events, UI for viewing an…

…d system config option. (#1133)

* support generic tracking loggin

* [feature] Log surfacing for Slack configuration

* slowly working the way through systemconfig

* slack config now mostly working, working on encryption now

* random headers turning up still ready to be encrypted but cannot be

* fixed formatting and now appears working

* problems with generics

* end testing worked!

* all testing passing on branch

* UI fixups

* various updates + slack button

* system config updates

* turn off the system config by default

* tests for webhooks and stuff all working

* auto connect for oauth token complete

* disable everything by default

---------

Co-authored-by: Irina Southwell <“[email protected]”>

adks/e2e-sdk/app/steps/cloud-events.ts

@@ -5,7 +5,7 @@ import {expect} from "chai";
 import {SdkWorld} from "../support/world";
 import {UpdateEnvironment} from "../apis/mr-service";
 import {sleep} from "../support/random";
-import {featurehubCloudEventBodyParser} from 'featurehub-cloud-event-tools';
+import {featurehubCloudEventBodyParser} from "featurehub-cloud-event-tools";
 
 When('I clear cloud events', function() {
  resetCloudEvents();
@@ -27,7 +27,7 @@ Then('I receive a cloud event of type {string}', async function (ceType: string)
 });
 
 Given(/^I update the environment for Slack$/, async function() {
- const world = this as SdkWorld;
+ const world= this as SdkWorld;
 
  if (world.environment === undefined) {
  const app = await world.applicationApi.getApplication(world.application.id, true);

adks/e2e-sdk/app/steps/setup.ts

@@ -46,11 +46,9 @@ Given(/^I create an application$/, async function () {
 Given(/^I update the environment for feature webhooks$/, async function() {
  const world = this as SdkWorld;
 
- if (world.environment === undefined) {
- const app = await world.applicationApi.getApplication(world.application.id, true);
- world.application = app.data;
- world.environment = app.data.environments[0];
- }
+ const app = await world.applicationApi.getApplication(world.application.id, true);
+ world.application = app.data;
+ world.environment = app.data.environments[0];
 
  // console.log("ensure previous environment has propagated");
  // await sleep(3);

adks/e2e-sdk/app/steps/system_config.ts

@@ -0,0 +1,37 @@
+import {Given} from "@cucumber/cucumber";
+import {getWebserverExternalAddress} from "../support/make_me_a_webserver";
+import {SdkWorld} from "../support/world";
+import {expect} from "chai";
+import {UpdatedSystemConfig, UpdatedSystemConfigs} from "../apis/mr-service";
+
+
+Given('I update the system config for Slack delivery', async function() {
+ const slackApiUrl = `${getWebserverExternalAddress()}/featurehub/slack`;
+
+ const fields = {
+ 'slack.enabled': true,
+ 'slack.bearerToken': '1234',
+ 'slack.defaultChannel': 'Cabcde',
+ 'slack.delivery.url': slackApiUrl,
+ };
+
+ const world = (this as SdkWorld);
+ const sData = await world.systemConfigApi.getSystemConfig(['slack.']);
+ expect(sData.status).to.eq(200);
+ const configs = sData.data;
+
+ const update = new UpdatedSystemConfigs({configs: []});
+ for(let field in fields) {
+ const existingVal = configs.configs.find(i => i.key == field);
+ // @ts-ignore
+ const v = fields[field];
+ const val = new UpdatedSystemConfig({
+ key: field,
+ version: existingVal?.version ?? -1,
+ value: v
+ });
+ update.configs.push(val);
+ }
+
+ await world.systemConfigApi.createOrUpdateSystemConfigs(update);
+});

adks/e2e-sdk/app/steps/webhook_steps.ts

@@ -9,6 +9,7 @@ import DataTable from '@cucumber/cucumber/lib/models/data_table';
 import {EnrichedFeatures} from '../apis/webhooks';
 import {logger} from '../support/logging';
 import {CloudEvent} from "cloudevents";
+import {featurehubCloudEventBodyParser} from "featurehub-cloud-event-tools";
 
 When('I wait for {int} seconds', async function (seconds: number) {
  await sleep(seconds * 1000);
@@ -27,34 +28,44 @@ Given(/^I test the webhook$/, async function () {
  }));
 });
 
+Given('I clear the cloud events', function () {
+ resetCloudEvents();
+});
+
 function ourEnrichedFeatures(world: SdkWorld): CloudEvent<EnrichedFeatures>[] {
  return cloudEvents.filter(ce => ce.type == 'enriched-feature-v1'
  && (ce.data as EnrichedFeatures)?.environment?.environment?.id === world.environment.id)
  .map(ce => ce as CloudEvent<EnrichedFeatures>);
 }
 
-Then(/^we receive a webhook with (.*) flag that is (locked|unlocked) and (off|on)$/, async function(flagName: string, lockedStatus: string, flag: string) {
+Then(/^we receive a webhook with (.*) flag that is (locked|unlocked) and (off|on) and version (.*)$/, async function(flagName: string,
+ lockedStatus: string, flag: string, version: string) {
  if (!process.env.EXTERNAL_NGROK && process.env.REMOTE_BACKEND) {
  return;
  }
 
  const world = this as SdkWorld;
 
+ console.log('looking for ', flagName, lockedStatus, flag, version);
  await waitForExpect(async () => {
  const enrichedData = ourEnrichedFeatures(world);
 
  expect(enrichedData.length, `filtered events for enriched and for our environment and found none ${cloudEvents}`).to.be.gt(0);
 
  const ourFeature = enrichedData.filter(ce => {
- const featureData = ce.data as EnrichedFeatures;
+ const featureData = featurehubCloudEventBodyParser(ce) as EnrichedFeatures;
+ console.log('feature data is ', featureData, featureData?.environment?.fv);
  const feature = featureData.environment?.fv?.find(fv => fv.feature.key === flagName);
- return (feature?.value?.locked === (lockedStatus === 'locked') &&
- feature?.value?.value === (flag === 'on') &&
- feature?.value.pId === world.person.id.id);
+ return (feature != null && feature.value != null && feature.value.locked === (lockedStatus === 'locked') &&
+ feature.value.value === (flag === 'on') &&
+ feature.value.version == parseInt(version) &&
+ feature.value.pId === world.person.id.id);
  });
 
  expect(ourFeature, `could not find feature ${flagName} in status ${lockedStatus} with value ${flag} and person ${world.person.id.id} in ${enrichedData}`)
  .to.not.be.undefined;
+ expect(ourFeature, `could not find feature ${flagName} in status ${lockedStatus} with value ${flag} and person ${world.person.id.id} in ${enrichedData}`)
+ .to.not.be.empty;
  }, 10000, 200);
 });
 

adks/e2e-sdk/app/support/hooks.ts

@@ -4,6 +4,7 @@ import { makeid } from './random';
 import { SdkWorld } from './world';
 import { discover } from './discovery';
 import {resetCloudEvents, startWebServer, terminateServer} from './make_me_a_webserver';
+import {logger} from "./logging";
 
 const superuserEmailAddress = '[email protected]';
 // const superuserEmailAddress = '[email protected]';
@@ -62,10 +63,14 @@ Before(function () {
 
 After(function () {
  const world = this as SdkWorld;
- if (world.edgeServer) {
- console.log('shutting down edge connection', world.edgeServer.getApiKeys(), world.edgeServer.url());
- world.edgeServer.close();
- console.log('edge connection closed');
+ try {
+ if (world.edgeServer) {
+ console.log('shutting down edge connection', world.edgeServer.getApiKeys(), world.edgeServer.url());
+ world.edgeServer.close();
+ console.log('edge connection closed');
+ }
+ } catch (e) {
+ logger.error('failed', e);
  }
 });
 
@@ -74,7 +79,11 @@ Before('@needs-webserver', async function() {
 });
 
 After('@needs-webserver', async function() {
- await terminateServer();
+ try {
+ await terminateServer();
+ } catch (e) {
+ logger.error('failed to terminate web server', e);
+ }
 });
 
 BeforeAll(async function() {

adks/e2e-sdk/app/support/make_me_a_webserver.ts

@@ -3,7 +3,8 @@ import * as restify from 'restify';
 import {networkInterfaces} from 'os';
 import {IncomingHttpHeaders} from 'http';
 import {logger} from './logging';
-import {CloudEvent, HTTP} from "cloudevents";
+import {CloudEvent, CloudEventV1, HTTP} from "cloudevents";
+import * as Zlib from 'zlib';
 
 const nets = networkInterfaces();
 const results: any = {};
@@ -31,12 +32,14 @@ if (networkName === undefined) {
  results[networkName] = ['localhost'];
 }
 
+const port = 3001;
+
 export function getWebserverExternalAddress(): string | undefined {
  if (process.env.EXTERNAL_NGROK) {
  return process.env.EXTERNAL_NGROK;
  }
 
- return networkName ? `http://${results[networkName][0]}:3000` : undefined;
+ return networkName ? `http://${results[networkName][0]}:${port}` : undefined;
 }
 
 let server: restify.Server;
@@ -46,9 +49,13 @@ export const cloudEvents: Array<CloudEvent<any>> = [];
 export function resetCloudEvents() {
  cloudEvents.length = 0;
  logger.info("------------\ncloud events reset\n--------")
+ console.log("------------\ncloud events reset\n--------")
 }
 
+
 function mergeCloudEvent<T>(body: T, headers: IncomingHttpHeaders) : CloudEvent<T>[] {
+ console.log('headers are ', headers);
+ console.log(`body is of type ${typeof body}`);
  const ce = HTTP.toEvent<T>({headers: headers, body: body});
 
  let events = Array.isArray(ce) ? (ce as CloudEvent<T>[]) : [ce as CloudEvent<T>];
@@ -64,7 +71,7 @@ function setupServer() {
  server.use(restify.plugins.acceptParser(server.acceptable));
  server.use(restify.plugins.queryParser());
  server.use (function(req, res, next) {
- logger.info(`received request on path ${req.path()}`);
+ logger.info(`received request on path ${req.path()} of content-type ${req.contentType()}`);
  if (req.contentType() === 'application/json') {
  var data='';
  req.setEncoding('utf8');
@@ -73,35 +80,57 @@ function setupServer() {
  });
 
  req.on('end', function() {
+ logger.debug(`'------------------------------\\nbody was ${data}\n---------------------------'`);
  req.body = JSON.parse(data);
- logger.info(`'------------------------------\\nbody was ${data}\n---------------------------'`);
  next();
  });
  } else if (req.contentType() === 'application/json+gzip') {
  let data = Buffer.from([]);
 
+ console.log(`data created is of type ${typeof data}`);
+
  req.on('data', function(chunk) {
  data = Buffer.concat([data, Buffer.from(chunk)]);
- data += chunk;
  });
 
  req.on('end', function() {
- req.body = data;
+ try {
+
+ // const inflated = Zlib.inflateSync(data).toString();
+ // logger.info(`'------------------------------\\nbody was ${inflated}\n---------------------------'`);
+ console.log(`data is of type ${typeof data}`);
+ req.body = data;
+ } catch (e) {
+ logger.error('failed to parse', e);
+ res.send(500, 'failed to parse');
+ }
  next();
  });
+ } else {
+ next();
  }
  });
 
  // server.use(restify.plugins.bodyParser());
 
  server.post('/featurehub/slack', function (req, res, next) {
- mergeCloudEvent(req.body, req.headers);
+ console.log('received');
+ try {
+ mergeCloudEvent(req.body, req.headers);
+ } catch (e) {
+ logger.error("failed", e);
+ }
+ console.log('responding ok');
  res.send(200, 'ok');
  return next();
  });
 
  server.post('/webhook', function (req, res, next) {
- mergeCloudEvent(req.body, req.headers);
+ try {
+ mergeCloudEvent(req.body, req.headers);
+ } catch (e) {
+ logger.error("failed", e);
+ }
 
  res.send( 200,'Ok');
  return next();
@@ -118,7 +147,8 @@ export function startWebServer(): Promise<void> {
  setupServer();
 
  try {
- server.listen(3000, function () {
+ server.listen(port, function () {
+ console.log(`${server.name} listening at ${server.url}`);
  logger.info(`${server.name} listening at ${server.url}`);
  resolve();
  });

adks/e2e-sdk/app/support/world.ts

@@ -9,6 +9,7 @@ import {
  Feature, FeatureGroup, FeatureGroupListGroup, FeatureGroupServiceApi, FeatureHistoryServiceApi,
  FeatureServiceApi,
  FeatureValue, Person, PersonServiceApi,
+ SystemConfigServiceApi,
  Portfolio,
  PortfolioServiceApi, ServiceAccount,
  ServiceAccountPermission,
@@ -55,6 +56,7 @@ export class SdkWorld extends World {
  public readonly featureValueApi: EnvironmentFeatureServiceApi;
  public readonly edgeApi: EdgeService;
  public readonly historyApi: FeatureHistoryServiceApi;
+ public readonly systemConfigApi: SystemConfigServiceApi;
 
  public readonly webhookApi: WebhookServiceApi;
  private _clientContext: ClientContext;
@@ -89,6 +91,7 @@ export class SdkWorld extends World {
  this.featureValueApi = new EnvironmentFeatureServiceApi(this.adminApiConfig);
  this.webhookApi = new WebhookServiceApi(this.adminApiConfig);
  this.historyApi = new FeatureHistoryServiceApi(this.adminApiConfig);
+ this.systemConfigApi = new SystemConfigServiceApi(this.adminApiConfig);
 
  const edgeConfig = new EdgeConfig({ basePath: this.featureUrl, axiosInstance: this.adminApiConfig.axiosInstance});
  this.edgeApi = new EdgeService(edgeConfig);

adks/e2e-sdk/features/slack.feature

@@ -6,6 +6,7 @@ Feature: System supports Slack
  Given I am logged in and have a person configured
  And I create a new portfolio
  And I create an application
+ And I update the system config for Slack delivery
  And I update the environment for Slack
  And I wait for 5 seconds
  And I clear cloud events

adks/e2e-sdk/features/webhooks.feature

@@ -9,15 +9,21 @@ Feature: Webhooks work as expected
  And I update the environment for feature webhooks
  And I clear cloud events
  And I wait for 5 seconds
+ # creating a feature flag creates an event
  When There is a feature flag with the key FEATURE_TITLE_TO_UPPERCASE
- And I wait for 5 seconds
+ Then we should have 1 messages in the list of webhooks
+ Then we receive a webhook with FEATURE_TITLE_TO_UPPERCASE flag that is locked and off and version 1
+ When I clear the cloud events
  And I test the webhook
- Then we receive a webhook with FEATURE_TITLE_TO_UPPERCASE flag that is locked and off
+ Then we should have 1 messages in the list of webhooks
+ Then we receive a webhook with FEATURE_TITLE_TO_UPPERCASE flag that is locked and off and version 1
+ When I clear the cloud events
  And I set the feature flag to unlocked and on
- Then we receive a webhook with FEATURE_TITLE_TO_UPPERCASE flag that is unlocked and on
+ Then we receive a webhook with FEATURE_TITLE_TO_UPPERCASE flag that is unlocked and on and version 2
+ And we should have 1 messages in the list of webhooks
  And I set the feature flag to unlocked and off
- Then we receive a webhook with FEATURE_TITLE_TO_UPPERCASE flag that is unlocked and off
- And we should have 4 messages in the list of webhooks
+ Then we receive a webhook with FEATURE_TITLE_TO_UPPERCASE flag that is unlocked and off and version 3
+ And we should have 1 messages in the list of webhooks
 
  @webhook2 @webhooks
  Scenario: I test a webhook that is triggered by a TestSDK changing a value

adks/e2e-sdk/package.json

@@ -5,6 +5,7 @@
  "scripts": {
  "clean": "rm -rf target/dist",
  "build": "npm run clean && node ./node_modules/typescript/bin/tsc",
+ "ws": "node_modules/.bin/ts-node app/support/run_webserver.ts",
  "debug-test": "PUBSUB_EMULATOR_HOST=localhost:8075 npm run clean && node --inspect node_modules/.bin/cucumber-js --require-module ts-node/register --require 'app/**/*.ts' ",
  "setup": "npm link featurehub-javascript-client-sdk featurehub-javascript-node-sdk",
  "test": "npm run clean && node node_modules/.bin/cucumber-js --require-module ts-node/register --require 'app/**/*.ts' ",

admin-frontend/app_mr_layer/pom.xml

@@ -67,6 +67,7 @@
  <file>${project.basedir}/../../backend/mr-api/feature-group.yaml</file>
  <file>${project.basedir}/../../backend/mr-api/feature-history.yaml</file>
  <file>${project.basedir}/../../backend/mr-api/application-strategies.yaml</file>
+ <file>${project.basedir}/../../backend/mr-api/system-api.yaml</file>
  </files>
  <finalYaml>${project.basedir}/final.yaml</finalYaml>
  </configuration>

admin-frontend/open_admin_app/lib/api/client_api.dart

@@ -66,6 +66,7 @@ class ManagementRepositoryClientBloc implements Bloc {
  late ApplicationServiceApi applicationServiceApi;
  late GroupServiceApi groupServiceApi;
  late WebhookServiceApi webhookServiceApi;
+ late TrackEventsServiceApi trackEventsServiceApi;
  static late FHRouter router;
 
  // this reflects actual requests to change the route driven externally, so a user clicks on
@@ -276,6 +277,7 @@ class ManagementRepositoryClientBloc implements Bloc {
  applicationServiceApi = ApplicationServiceApi(client);
  groupServiceApi = GroupServiceApi(client);
  webhookServiceApi = WebhookServiceApi(client);
+ trackEventsServiceApi = TrackEventsServiceApi(client);
  _errorSource.add(null);
  streamValley.apiClient = this;
 
@@ -377,7 +379,8 @@ class ManagementRepositoryClientBloc implements Bloc {
  if (routeChange) {
  routeSlot(RouteSlot.portfolio);
  }
- }).catchError((_) {
+ }).catchError((e) {
+ _log.fine("failed to request own details $e");
  setBearerToken(null);
  routeSlot(RouteSlot.login);
  });
@@ -401,6 +404,7 @@ class ManagementRepositoryClientBloc implements Bloc {
  }
 
  void routeSlot(RouteSlot slot) {
+ _log.finest("swapping to slot $slot");
  if (_siteInitialisedSource.value != RouteSlot.nowhere) {
  _siteInitialisedSource.add(slot);
  }