Skip to content

Commit

Permalink
Merge pull request #3949 from devgateway/develop
Browse files Browse the repository at this point in the history 
AMP-30352 3.5.5 release
  • Loading branch information
@jdeanquin-dg
jdeanquin-dg authored Oct 14, 2022
2 parents f3261b1 + 5de62ef commit fac79af
Show file tree
Hide file tree
Showing 226 changed files with 157,636 additions and 16,987 deletions.
8 changes: 0 additions & 8 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,7 @@
amp/TEMPLATE/ampTemplate/node_modules/amp-settings/dist/
amp/TEMPLATE/ampTemplate/node_modules/amp-settings/src/compiled-css/amp-settings.css
amp/TEMPLATE/ampTemplate/node_modules/amp-settings/src/compiled-js/amp-settings.js
amp/TEMPLATE/reamp/modules/gpi-reports/script.min.js
amp/TEMPLATE/reamp/modules/gpi-data/script.min.js
amp/TEMPLATE/reamp/modules/admin/resource-manager-admin/script.min.js
amp/TEMPLATE/reamp/modules/admin/data-freeze-manager/script.min.js
amp/TEMPLATE/reamp/modules/admin/dashboard/script.min.js
amp/TEMPLATE/reamp/modules/admin/currency/deflator/script.min.js
amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate/src/compiled-js/amp-boilerplate.js
amp/TEMPLATE/reamp/node/
amp/TEMPLATE/ampTemplate/node_modules_10/
amp/TEMPLATE/ampTemplate/node_modules/amp-filter/node/node_modules/
amp/TEMPLATE/ampTemplate/node_modules/amp-filter/node/
Expand All @@ -18,7 +11,6 @@ amp/TEMPLATE/ampTemplate/node_modules/amp-filter/dist/amp-filter.css
amp/TEMPLATE/ampTemplate/node_modules/amp-filter/src/compiled-css/amp-filter.css
amp/TEMPLATE/ampTemplate/node_modules/amp-filter/src/compiled-js/amp-filter.js
amp/TEMPLATE/ampTemplate/node_modules/amp-filter/dist/amp-filter.js
amp/TEMPLATE/reamp/modules/admin/performance-alert-manager/script.min.js
amp/TEMPLATE/ampTemplate/node_modules/amp-state/node/
amp/TEMPLATE/ampTemplate/node_modules/amp-translate/node/
amp/TEMPLATE/ampTemplate/node_modules/amp-translate/node_modules/
Expand Down
162 changes: 43 additions & 119 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ println "Pull request: ${pr}"
println "Tag: ${tag}"

def dbVersion
def pgVersion = 14
def country
def ampUrl
def dockerRepo = "registry.developmentgateway.org/"

def updateGitHubCommitStatus(context, message, state) {
repoUrl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim()
Expand All @@ -45,42 +47,11 @@ def updateGitHubCommitStatus(context, message, state) {
])
}

// Run checkstyle only for PR builds
stage('Checkstyle') {
when (branch == null) {
node {
try {
checkout scm

updateGitHubCommitStatus('jenkins/checkstyle', 'Checkstyle in progress', 'PENDING')

withEnv(["PATH+MAVEN=${tool 'M339'}/bin"]) {
sh "cd amp && mvn inccheckstyle:check -DbaseBranch=remotes/origin/${CHANGE_TARGET}"
}

updateGitHubCommitStatus('jenkins/checkstyle', 'Checkstyle success', 'SUCCESS')
} catch(e) {
updateGitHubCommitStatus('jenkins/checkstyle', 'Checkstyle found violations', 'ERROR')
}
}
}
}

def legacyMvnOptions = "-Djdbc.user=amp " +
"-Djdbc.password=amp122006 " +
"-Djdbc.db=amp " +
"-Djdbc.host=db " +
"-Djdbc.port=5432 " +
"-DdbName=postgresql " +
"-Djdbc.driverClassName=org.postgresql.Driver"

def launchedByUser = currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause').size() > 0
def codeVersion
def countries

// Run fail fast tests
stage('Quick Test') {
// Find list of countries which have database dumps compatible with ${codeVersion}
stage('Build') {

node {
checkout scm

Expand All @@ -89,108 +60,61 @@ stage('Quick Test') {
println "AMP Version: ${codeVersion}"

countries = sh(returnStdout: true,
script: "ssh sulfur.migrated.devgateway.org 'cd /opt/amp_dbs && amp-db ls ${codeVersion} | sort'")
script: "ssh boad.aws.devgateway.org 'cd /opt/amp_dbs && amp-db ls ${codeVersion} | sort'")
.trim()
if (countries == "") {
println "There are no database backups compatible with ${codeVersion}"
currentBuild.result = 'FAILURE'
}
}

// Allow user to specify country before tests are run
if (launchedByUser) {
timeout(15) {
milestone()
country = input(
message: "Proceed with test, build and deploy?",
parameters: [choice(choices: countries, name: 'country')])
milestone()
}
timeout(15) {
milestone()
country = input(
message: "Proceed with build and deploy?",
parameters: [choice(choices: countries, name: 'country')])
milestone()
}

node {
try {
checkout scm

updateGitHubCommitStatus('jenkins/failfasttests', 'Testing in progress', 'PENDING')

withEnv(["PATH+MAVEN=${tool 'M339'}/bin"]) {
def testStatus = sh returnStatus: true, script: "cd amp && mvn clean test -Dskip.npm -Dskip.gulp ${legacyMvnOptions}"

// Archive unit test report
junit 'amp/target/surefire-reports/TEST-*.xml'

if (testStatus != 0) {
error "Tests command returned an error code!"
}
}

updateGitHubCommitStatus('jenkins/failfasttests', 'Fail fast tests: success', 'SUCCESS')
} catch (e) {
updateGitHubCommitStatus('jenkins/failfasttests', 'Fail fast tests: error', 'ERROR')

throw e
}
}
}

stage('Build') {

if (country == null) {
timeout(15) {
milestone()
country = input(
message: "Proceed with build and deploy?",
parameters: [choice(choices: countries, name: 'country')])
milestone()
}
}

ampUrl = "http://amp-${country}-${tag}-tc9.ampsite.net/"
ampUrl = "http://amp-${country}-${tag}.stg.ampsite.net/"

node {
checkout scm

def image = "${dockerRepo}amp-webapp:${tag}"
def format = branch != null ? "%H" : "%P"
def hash = sh(returnStdout: true, script: "git log --pretty=${format} -n 1").trim()
sh(returnStatus: true, script: "docker pull phosphorus.migrated.devgateway.org:5000/amp-webapp:${tag} > /dev/null")
sh(returnStatus: true, script: "docker pull ${image} > /dev/null")
def imageIds = sh(returnStdout: true, script: "docker images -q -f \"label=git-hash=${hash}\"").trim()
sh(returnStatus: true, script: "docker rmi phosphorus.migrated.devgateway.org:5000/amp-webapp:${tag} > /dev/null")
sh(returnStatus: true, script: "docker rmi ${image} > /dev/null")

if (imageIds.equals("")) {
withEnv(["PATH+MAVEN=${tool 'M339'}/bin"]) {
try {
sh returnStatus: true, script: 'tar -xf ../amp-node-cache.tar'

// Build AMP
sh "cd amp && mvn -T 4 clean compile war:exploded ${legacyMvnOptions} -DskipTests -DbuildSource=${tag} -e"

// Build Docker images & push it
sh "docker build -q -t phosphorus.migrated.devgateway.org:5000/amp-webapp:${tag} --build-arg AMP_EXPLODED_WAR=target/amp --build-arg AMP_PULL_REQUEST='${pr}' --build-arg AMP_BRANCH='${branch}' --build-arg AMP_REGISTRY_PRIVATE_KEY='${registryKey}' --label git-hash='${hash}' amp"
sh "docker push phosphorus.migrated.devgateway.org:5000/amp-webapp:${tag} > /dev/null"
} finally {

// Cleanup after Docker & Maven
sh returnStatus: true, script: "docker rmi phosphorus.migrated.devgateway.org:5000/amp-webapp:${tag}"
sh returnStatus: true, script: "cd amp && mvn clean -Djdbc.db=dummy"
sh returnStatus: true, script: "tar -cf ../amp-node-cache.tar --remove-files" +
" amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate/node" +
" amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate/node_modules" +
" amp/TEMPLATE/ampTemplate/node_modules/gis-layers-manager/node" +
" amp/TEMPLATE/ampTemplate/node_modules/gis-layers-manager/node_modules" +
" amp/TEMPLATE/ampTemplate/node_modules/amp-settings/node" +
" amp/TEMPLATE/ampTemplate/node_modules/amp-settings/node_modules" +
" amp/TEMPLATE/ampTemplate/node_modules/amp-translate/node" +
" amp/TEMPLATE/ampTemplate/node_modules/amp-translate/node_modules" +
" amp/TEMPLATE/ampTemplate/node_modules/amp-state/node" +
" amp/TEMPLATE/ampTemplate/node_modules/amp-state/node_modules" +
" amp/TEMPLATE/ampTemplate/gisModule/dev/node" +
" amp/TEMPLATE/ampTemplate/gisModule/dev/node_modules" +
" amp/TEMPLATE/ampTemplate/dashboard/dev/node" +
" amp/TEMPLATE/ampTemplate/dashboard/dev/node_modules" +
" amp/TEMPLATE/reamp/node" +
" amp/TEMPLATE/reamp/node_modules"
try {
updateGitHubCommitStatus('jenkins/build', 'Build in progress', 'PENDING')

sshagent(credentials: ['GitHubDgReadOnlyKey']) {
withEnv(['DOCKER_BUILDKIT=1']) {
sh "docker build " +
"--progress=plain " +
"--ssh default " +
"-t ${image} " +
"--build-arg BUILD_SOURCE='${tag}' " +
"--build-arg AMP_PULL_REQUEST='${pr}' " +
"--build-arg AMP_BRANCH='${branch}' " +
"--build-arg AMP_REGISTRY_PRIVATE_KEY='${registryKey}' " +
"--label git-hash='${hash}' " +
"amp"
}
}
sh "docker push ${image} > /dev/null"

updateGitHubCommitStatus('jenkins/build', 'Built successfully', 'SUCCESS')
} catch (e) {
updateGitHubCommitStatus('jenkins/build', 'Build failed', 'ERROR')
throw e
} finally {
// Cleanup after Docker & Maven
sh returnStatus: true, script: "docker rmi ${image}"
}
}
}
Expand All @@ -203,10 +127,10 @@ stage('Deploy') {
node {
try {
// Find latest database version compatible with ${codeVersion}
dbVersion = sh(returnStdout: true, script: "ssh sulfur.migrated.devgateway.org 'cd /opt/amp_dbs && amp-db find ${codeVersion} ${country}'").trim()
dbVersion = sh(returnStdout: true, script: "ssh boad.aws.devgateway.org 'cd /opt/amp_dbs && amp-db find ${codeVersion} ${country}'").trim()

// Deploy AMP
sh "ssh sulfur.migrated.devgateway.org 'cd /opt/docker/amp && ./up.sh ${tag} ${country} ${dbVersion}'"
sh "ssh boad.aws.devgateway.org 'amp-up ${tag} ${country} ${dbVersion} ${pgVersion}'"

slackSend(channel: 'amp-ci', color: 'good', message: "Deploy AMP - Success\nDeployed ${changePretty} will be ready for testing at ${ampUrl} in about 3 minutes")

Expand All @@ -231,7 +155,7 @@ stage('Deploy again') {
}
node {
try {
sh "ssh sulfur.migrated.devgateway.org 'cd /opt/docker/amp && ./up.sh ${tag} ${country} ${dbVersion}'"
sh "ssh boad.aws.devgateway.org 'amp-up ${tag} ${country} ${dbVersion} ${pgVersion}'"

slackSend(channel: 'amp-ci', color: 'good', message: "Deploy AMP - Success\nDeployed ${changePretty} will be ready for testing at ${ampUrl} in about 3 minutes")

Expand Down
36 changes: 31 additions & 5 deletions amp/.dockerignore
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,31 @@
*
!META-INF
!Dockerfile
!docker
!target/amp*
.idea
*.iml
target
gen-src
META-INF
WEB-INF/classes
WEB-INF/lib
jackrabbit
lucene
api-docs
Dockerfile
TEMPLATE/ampTemplate/node_modules/amp-boilerplate/node
TEMPLATE/ampTemplate/node_modules/amp-boilerplate/node_modules
TEMPLATE/ampTemplate/node_modules/gis-layers-manager/node
TEMPLATE/ampTemplate/node_modules/gis-layers-manager/node_modules
TEMPLATE/ampTemplate/node_modules/amp-settings/node
TEMPLATE/ampTemplate/node_modules/amp-settings/node_modules
TEMPLATE/ampTemplate/node_modules/amp-translate/node
TEMPLATE/ampTemplate/node_modules/amp-translate/node_modules
TEMPLATE/ampTemplate/node_modules/amp-state/node
TEMPLATE/ampTemplate/node_modules/amp-state/node_modules
TEMPLATE/ampTemplate/node_modules/amp-filter/node
TEMPLATE/ampTemplate/node_modules/amp-filter/node_modules
TEMPLATE/ampTemplate/gisModule/dev/node
TEMPLATE/ampTemplate/gisModule/dev/node_modules
TEMPLATE/ampTemplate/dashboard/dev/node
TEMPLATE/ampTemplate/dashboard/dev/node_modules
TEMPLATE/reamp/node
TEMPLATE/reamp/node_modules
TEMPLATE/reampv2/node
TEMPLATE/reampv2/node_modules
67 changes: 56 additions & 11 deletions amp/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,65 @@
FROM tomcat:7.0-jdk8
FROM maven:3.8.4-jdk-8 as base
WORKDIR /tmp/amp
COPY . .

FROM base as compile
ARG BUILD_SOURCE
RUN mkdir -p -m 0600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
RUN --mount=type=ssh \
--mount=type=cache,target=/root/.m2 \
--mount=type=cache,target=/root/.npm \
mvn -B test war:exploded \
-DbuildSource=$BUILD_SOURCE \
-Djdbc.user=amp -Djdbc.password=amp122006 -Djdbc.db=amp -Djdbc.host=db \
-Djdbc.port=5432 -DdbName=postgresql -Djdbc.driverClassName=org.postgresql.Driver \
&& mv target/amp exploded \
&& rm -rf target \
TEMPLATE/ampTemplate/node_modules/amp-boilerplate/node \
TEMPLATE/ampTemplate/node_modules/amp-boilerplate/node_modules \
TEMPLATE/ampTemplate/node_modules/gis-layers-manager/node \
TEMPLATE/ampTemplate/node_modules/gis-layers-manager/node_modules \
TEMPLATE/ampTemplate/node_modules/amp-settings/node \
TEMPLATE/ampTemplate/node_modules/amp-settings/node_modules \
TEMPLATE/ampTemplate/node_modules/amp-translate/node \
TEMPLATE/ampTemplate/node_modules/amp-translate/node_modules \
TEMPLATE/ampTemplate/node_modules/amp-state/node \
TEMPLATE/ampTemplate/node_modules/amp-state/node_modules \
TEMPLATE/ampTemplate/node_modules/amp-filter/node \
TEMPLATE/ampTemplate/node_modules/amp-filter/node_modules \
TEMPLATE/ampTemplate/gisModule/dev/node \
TEMPLATE/ampTemplate/gisModule/dev/node_modules \
TEMPLATE/ampTemplate/dashboard/dev/node \
TEMPLATE/ampTemplate/dashboard/dev/node_modules \
TEMPLATE/reamp/node \
TEMPLATE/reamp/node_modules \
TEMPLATE/reampv2/node \
TEMPLATE/reampv2/node_modules

FROM tomcat:8.5.79-jdk8

RUN apt-get update \
&& apt-get install -y apt-transport-https ca-certificates curl gnupg --no-install-recommends \
&& curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
&& echo "deb https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
&& apt-get update \
&& apt-get install -y google-chrome-stable fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf --no-install-recommends \
&& apt-get purge --auto-remove -y curl gnupg \
&& rm -rf /var/lib/apt/lists/*

ENV CHROME_PATH /opt/google/chrome
ENV CHROME_NO_SANDBOX 1

COPY docker/wait-for-it.sh docker/setenv.sh bin/
COPY docker/server.xml docker/tomcat-users.xml conf/

ARG AMP_EXPLODED_WAR
ARG AMP_PULL_REQUEST
ARG AMP_BRANCH
ARG AMP_REGISTRY_PRIVATE_KEY

LABEL "pull-request"=$AMP_PULL_REQUEST
LABEL "branch"=$AMP_BRANCH

ADD docker/wait-for-it.sh bin/
ADD docker/setenv.sh bin/
ADD docker/server.xml conf/
ADD docker/tomcat-users.xml conf/

RUN rm -fr /usr/local/tomcat/webapps/ROOT && mkdir /usr/local/tomcat/webapps/ROOT
ADD $AMP_EXPLODED_WAR /usr/local/tomcat/webapps/ROOT
ADD META-INF/context.xml /usr/local/tomcat/webapps/ROOT/META-INF/

ENV AMP_REGISTRY_PRIVATE_KEY $AMP_REGISTRY_PRIVATE_KEY

RUN rm -fr /usr/local/tomcat/webapps/ROOT
COPY --from=compile /tmp/amp/exploded /usr/local/tomcat/webapps/ROOT/
6 changes: 2 additions & 4 deletions amp/TEMPLATE/ampTemplate/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,2 @@
# first, ignore everything in node modules, so we don't track build deps
node_modules/*
# then, unignore our own libraries that we want to have available on the `require` search path
!node_modules/amp-*
node_modules/*/node
node_modules/*/node_modules
Binary file added amp/TEMPLATE/ampTemplate/dashboard/dev/app/fonts/glyphicons-halflings-regular.woff2
View file
Binary file not shown.
Binary file added ...TEMPLATE/ampTemplate/dashboard/dev/app/img/jQueryUI/ui-icons_3383bb_256x240.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion amp/TEMPLATE/ampTemplate/dashboard/dev/app/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ <h2><small>Loading...</small></h2>
<div id="amp-footer">
</div>
<script src="/TEMPLATE/ampTemplate/commonJs/common.min.js"></script>
<script src="compiled-js/app.js"></script>
<script src="compiled-js/app.js"></script>
</body>
</html>
Loading

0 comments on commit fac79af

Please sign in to comment.