Nano version 2.4.2 (#1)

   * more modular code
   * better synchronization
   * consistent throughput calculation
   * iostat output display
   * structured output (detailed csv, aggregated csv), decoupling logging from results
   * more tests (cpu, random read from cache)
   * better documentation
   * object storage support
   * easier configuration
   * docker support
   * code quality improvements
   * persisting execution environment (parameters, kdb+ version, etc)

Author: FerencBodon-Kx

.gitignore

@@ -0,0 +1,3 @@
+results/**
+logs/**
+qpbuild/**

.gitlab-ci.yml

@@ -0,0 +1,46 @@
+default:
+  tags:
+    - kxi-gitlab-runner
+
+
+stages:
+  - build
+
+.docker-beforescript:
+  stage: build
+  image: "docker:20.10.23"
+  before_script:
+    - |
+      if [ -z "$DOCKER_AUTH_CONFIG" ]
+      then
+        echo "\$DOCKER_AUTH_CONFIG not set, unable to authenticate against container registries"
+        exit 1
+      fi
+
+    - mkdir -p $HOME/.docker && echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
+    - echo ${INT_NEXUS_PASS_RW} | docker login ${INSIGHTS_DEV_NEXUS_REGISTRY} -u ${INT_NEXUS_USER_RW} --password-stdin
+  script:
+    - wget https://${INT_NEXUS_USER_RO}:${INT_NEXUS_PASS_RO}@ext-nexus.kxi-dev.kx.com/repository/kx-insights-packages/kdbInsightsCore/0.41.1/kdb-insights-core.tar
+    - docker load < kdb-insights-core.tar
+    - wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    - chmod a+x /usr/local/bin/yq
+    - IMAGE_URL="$INSIGHTS_DEV_NEXUS_REGISTRY/benchmarking/nano:$(yq -r ".${TARGET}" version.yaml)"
+    - docker build -t "${IMAGE_URL}" .
+    - docker push "${IMAGE_URL}"
+    - docker rmi "${IMAGE_URL}"
+
+docker-buildpub-push:
+  variables:
+    TARGET: "pub"
+  extends:
+    - .docker-beforescript
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+
+docker-builddev-push:
+  variables:
+    TARGET: "dev"
+  extends:
+    - .docker-beforescript
+  rules:
+    - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'

Dockerfile

@@ -0,0 +1,30 @@
+FROM kdb-insights-core:0.41.1
+
+WORKDIR /opt/kx/app
+
+COPY src ./src
+COPY mthread.sh runSeveral.sh version.yaml ./
+COPY flush ./flush
+
+ENV QBIN=/opt/kx/kdb/l64/q
+
+ENV RESULTDIR=/appdir/results
+ENV LOGDIR=/appdir/logs
+ENV THREADNR=1
+ENV COMPRESS=""
+
+ENV MEMUSAGETYPE=ratio
+ENV MEMUSAGEVALUE=0.6
+ENV RANDOMREADFILESIZETYPE=ratio
+ENV RANDOMREADFILESIZEVALUE=3
+ENV DBSIZE=full
+
+ENV FLUSH=/opt/kx/app/flush/directmount.sh
+
+RUN echo "/data" > ./partitions
+
+RUN yum upgrade -y
+RUN yum install -y wget sysstat
+RUN wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_386 -O /usr/bin/yq && chmod +x /usr/bin/yq
+
+ENTRYPOINT [ "/bin/bash", "mthread.sh" ]

README.md

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# make sure you did:
+# docker login ${REGISTRY}
+# that probably needs an access key
+
+if [ $# -eq 0 ]; then
+  TARGET=dev
+else
+  TARGET=$1
+fi
+
+IMAGENAME=nano
+VERSION=$(yq ".${TARGET}" version.yaml)
+echo "Building verion ${VERSION}"
+
+docker build -t ${IMAGENAME}:${VERSION} .
+docker tag nano:${VERSION} nano:latest
+
+IMAGE_URL=${REGISTRY}/${PROJECTDIR}/${IMAGENAME}:${VERSION}
+
+docker tag nano:${VERSION} ${IMAGE_URL}
+docker push ${IMAGE_URL}
+
+echo "$0 done"

build/build.sh

@@ -0,0 +1,2 @@
+export REGISTRY="registry.gitlab.com"
+export PROJECTDIR="kxdev/benchmarking/nano"

build/gitlab

@@ -0,0 +1,2 @@
+export REGISTRY="ext-dev-registry.kxi-dev.kx.com"
+export PROJECTDIR="benchmarking"

build/nexus

@@ -0,0 +1,41 @@
+
+export COMPRESS=""
+
+# Number of threads each kdb+ process
+export THREADNR=1
+
+export FLUSH=./flush/directmount.sh
+
+##############
+# Settings for object storage
+# See this for more details: https://code.kx.com/insights/1.4/core/objstor/main.html
+export OBJSTORELOCTMPDIRBASE=/mnt/${USER}/OBJSTORETEST
+# Comment line below for testing without object storage cache
+export KX_OBJSTR_CACHE_PATH=/mnt/${USER}/OBJSTORECACHE
+
+export AWS_REGION=eu-west-1
+export GCLOUD_PROJECT_ID=myproject
+export AZURE_STORAGE_ACCOUNT=mystorageaccount
+export AZURE_STORAGE_SHARED_KEY=mysharedkey
+
+
+###############################################
+#### You probably dont need to change this
+
+export RESULTDIR="./results"
+export LOGDIR="./logs"
+
+# Memory used for the sequential read/write tests. This determines the size of the vector created in memory.
+# You can specify an absolute value in MB or the ratio to the physical memory size.
+export MEMUSAGETYPE=ratio    # Either ratio or abs
+export MEMUSAGEVALUE=0.6     # ratio to the physical memory
+
+# Disk space used for the mid append and randome read tests.
+export RANDOMREADFILESIZETYPE=ratio
+export RANDOMREADFILESIZEVALUE=2
+
+
+# Mainly for testing, you can generate small or tiny data
+export DBSIZE=full
+
+

config/env

@@ -0,0 +1,16 @@
+if ! command -v q &> /dev/null
+then
+  export QHOME=$HOME/q     # SET QHOME MANUALLY
+
+  if [ `uname -s` = "Darwin" ]; then
+    QSUBDIR=w64
+  else
+    QSUBDIR=l64
+  fi
+  export QBIN="$QHOME/$QSUBDIR/q"
+else
+  export QBIN=$(which q)
+fi
+
+echo "QBIN is set to $QBIN"
+