Skip to content

Commit

Permalink
Add Azure Pipelines CI/CD and use python -m pip instead of pip execut…
Browse files Browse the repository at this point in the history
…able

Add Azure Pipelines CI/CD.
Use python -m pip instead of pip2/3
  • Loading branch information
gusennan authored May 19, 2020
1 parent ead5ce0 commit 3595d64
Show file tree
Hide file tree
Showing 5 changed files with 178 additions and 64 deletions.
6 changes: 3 additions & 3 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,11 @@ The full text of the GPLv2 is included in the accompanying `LICENSE.txt`_
Installation
~~~~~~~~~~~~

on OSX:
on macOS:
-------

#. Install `Python`_, if not installed already.
* Both v2 and v3 are supported.
* While macOS comes with a python 2.7 interpreter, it's old and doesn't have the `pip` module OSPREY installation requires. If you're still using Python 2, only the latest maintenance release will work. In any case, we highly recommend installing Python3, either using homebrew or directly from the Python website.
* Install the `64-bit` verison, not the `32-bit` version.
#. Download the `newest Osprey Python release`_.
* Download either `osprey-osx-python2` or `osprey-osx-python3` to match your Python version.
Expand All @@ -83,7 +83,7 @@ on Windows:
Make sure you're running 64-bit Windows. Osprey is not supported on 32-bit Windows.

#. Install `Python`_.
* Both v2 and v3 are supported.
* Both v2 and v3 are supported. We highly recommend Python 3. Python 3 can be installed using the scoop package manager or directly from the Python website.
* Install the `x86-64` verison, not the `x86` version.
* During Python installation, enable the option to ``Add python.exe to Path``.
#. Download the `newest Osprey Python release`_.
Expand Down
181 changes: 146 additions & 35 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
@@ -1,38 +1,149 @@
# Starter pipeline
# Start with a minimal pipeline that you can customize to build and deploy your code.
# Add steps that build, run tests, deploy, and more:
# https://aka.ms/yaml

trigger:
- master

strategy:
matrix:
linux:
imageName: ubuntu-latest
openJdkUrl: https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb326336a2/7/GPL/
jdkName: openjdk-14.0.1_linux-x64_bin.tar.gz
javaHome: $(Build.BinariesDirectory)/java/jdk-14.0.1
gradleWrapper: gradlew
mac:
imageName: macos-latest
openJdkUrl: https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb326336a2/7/GPL/
jdkName: openjdk-14.0.1_osx-x64_bin.tar.gz
javaHome: $(Build.BinariesDirectory)/java/jdk-14.0.1.jdk/Contents/Home/
gradleWrapper: gradlew
windows:
imageName: windows-latest
openJdkUrl: https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb326336a2/7/GPL/
jdkName: openjdk-14.0.1_windows-x64_bin.zip
javaHome: $(Build.BinariesDirectory)/java/jdk-14.0.1
gradleWrapper: gradlew.bat

pool:
vmImage: $(imageName)

steps:
- script: python2 --version
displayName: 'Show python2 version'

- script: python3 --version
displayName: 'Show python3 version'
jobs:
- job: Assemble
timeoutInMinutes: 10
variables:
pythonDependencies: "sphinx==1.8 guzzle_sphinx_theme javalang"
strategy:
matrix:
Linux:
imageName: ubuntu-latest
openJdkUrl: https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb326336a2/7/GPL/
jdkName: openjdk-14.0.1_linux-x64_bin.tar.gz
javaHome: $(Build.BinariesDirectory)/java/jdk-14.0.1
gradleWrapper: gradlew
Mac:
imageName: macos-latest
openJdkUrl: https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb326336a2/7/GPL/
jdkName: openjdk-14.0.1_osx-x64_bin.tar.gz
javaHome: $(Build.BinariesDirectory)/java/jdk-14.0.1.jdk/Contents/Home/
gradleWrapper: gradlew
Windows:
imageName: windows-latest
openJdkUrl: https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb326336a2/7/GPL/
jdkName: openjdk-14.0.1_windows-x64_bin.zip
javaHome: $(Build.BinariesDirectory)/java/jdk-14.0.1
gradleWrapper: gradlew.bat

pool:
vmImage: $(imageName)

steps:
- task: UsePythonVersion@0
name: GetPython3
inputs:
versionSpec: '3.x'
addToPath: true
architecture: 'x64'

- task: UsePythonVersion@0
name: GetPython2
inputs:
versionSpec: '2.x'
addToPath: false
architecture: 'x64'

- task: Bash@3
displayName: Standardize Java and Python Paths
inputs:
targetType: 'inline'
script: |
function standardize_path {
echo `echo "$1" | tr '\\' '/'`
}
p2=`standardize_path '$(GetPython2.pythonLocation)/python'`
p3=`standardize_path '$(GetPython3.pythonLocation)/python'`
JAVA_HOME=`standardize_path '$(javaHome)'`
echo "Python2 -> ${p2}"
echo "Python3 -> ${p3})"
echo "JAVA_HOME -> ${JAVA_HOME}"
echo "##vso[task.setvariable variable=python2]${p2}"
echo "##vso[task.setvariable variable=python3]${p3}"
echo "##vso[task.setvariable variable=JAVA_HOME]${JAVA_HOME}"
- task: CmdLine@2
displayName: Install Python Dependencies
inputs:
script: |
echo "Installing Python 3 dependencies"
$(python3) -m pip install setuptools wheel
$(python3) -m pip install $(pythonDependencies)
echo "Installing Python 2 dependencies"
$(python2) -m pip install setuptools wheel
$(python2) -m pip install $(pythonDependencies)
# Download the JDK
- task: Bash@3
inputs:
targetType: 'inline'
script: 'curl --silent --remote-name $(openJdkUrl)$(jdkName)'
workingDirectory: '$(Build.BinariesDirectory)'
displayName: Download OpenJDK

- task: ExtractFiles@1
displayName: Extract OpenJDK
inputs:
archiveFilePatterns: |
$(Build.BinariesDirectory)/*.zip
$(Build.BinariesDirectory)/*.tar.gz
destinationFolder: $(Build.BinariesDirectory)/java
cleanDestinationFolder: true

- task: Bash@3
displayName: Assemble
inputs:
targetType: 'inline'
script: |
export JAVA_HOME=$(JAVA_HOME)
./$(gradleWrapper) -POSPREY_PYTHON3=$(python3) -POSPREY_PYTHON2=$(python2) -PAZURE_BUILD_ID=$(Build.BuildId) assemble
- task: CopyFiles@2
inputs:
SourceFolder: 'build/distributions'
Contents: |
*.zip
TargetFolder: '$(Build.ArtifactStagingDirectory)'
displayName: Copy distributions to staging dir

- task: PublishBuildArtifacts@1
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)'
ArtifactName: $(Agent.OS)
publishLocation: 'Container'
displayName: Make staging dir an artifact

- job: CreateRelease
pool:
vmImage: ubuntu-latest
dependsOn: Assemble
condition: ne(variables['Build.Reason'], 'PullRequest')
steps:
- bash: |
version=$(<resources/config/version)
echo "##vso[task.setvariable variable=version]${version}"
displayName: Set version variable from version file
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
downloadPath: '$(System.ArtifactsDirectory)'
displayName: Download assembled artifacts
- task: GitHubRelease@1
inputs:
gitHubConnection: 'github.com_gusennan'
repositoryName: '$(Build.Repository.Name)'
action: 'create'
target: '$(Build.SourceVersion)'
tagSource: 'userSpecifiedTag'
tag: '$(version).$(Build.BuildId)'
assets: '$(System.ArtifactsDirectory)/**/osprey-*python*.zip'
isDraft: true
changeLogCompareToRelease: 'lastFullRelease'
changeLogType: 'commitBased'
title: '$(version).$(Build.BuildId)'
displayName: Push draft release $(version).$(Build.BuildId)
51 changes: 27 additions & 24 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -209,11 +209,15 @@ val os = OperatingSystem.current()

fun isCommand(cmd: String) =
exec {
val output = ByteArrayOutputStream()
isIgnoreExitValue = true
standardOutput = output
errorOutput = output

when (os) {
OperatingSystem.MAC_OS,
OperatingSystem.LINUX -> commandLine("/bin/sh", "-c", "type $cmd > /dev/null")
OperatingSystem.WINDOWS -> commandLine("where", "/q", cmd)
OperatingSystem.LINUX -> commandLine("which", cmd)
OperatingSystem.WINDOWS -> commandLine("powershell", "get-command", cmd)
else -> throw Error("unrecognized operating system: $os")
}
}.exitValue == 0
Expand Down Expand Up @@ -242,17 +246,20 @@ class Python(val cmd: String) {
}

val pipCmd by lazy {
"pip$version".takeIf { isCommand(it) }
"-m pip"
}

override fun toString() = "Python $version"
}

// find out what pythons are available
val pythons by lazy {
val python3 = findProperty("OSPREY_PYTHON3")?.toString() ?: "python3"
val python2 = findProperty("OSPREY_PYTHON2")?.toString() ?: "python2"

listOf(
Python("python3"),
Python("python2"),
Python(python3),
Python(python2),
Python("python")
)
.filter { it.version != null }
Expand All @@ -275,8 +282,8 @@ val defaultPython by lazy {

println("""
| Pythons: ${pythons.map { it.cmd }}
| Python 2: ${if (python2 != null) "X" else ""}
| Python 3: ${if (python3 != null) "X" else ""}
| Python 2: ${if (python2 != null) "" else ""}
| Python 3: ${if (python3 != null) "" else ""}
| default Python: $defaultPython = ${defaultPython.cmd}
""".trimMargin())

Expand Down Expand Up @@ -404,18 +411,15 @@ tasks {

// read the version number
val versionFile = projectDir.resolve("build/resources/main/config/version").toFile()
var version = versionFile.readText()
var version = versionFile.readText().trim()

// append the travis build number if available
val travisBuildNumber = System.getenv("TRAVIS_BUILD_NUMBER")
if (travisBuildNumber != null) {
version += "-b$travisBuildNumber"

// otherwise, use a "-dev" build number
// append the CI build ID, if available
version += if (hasProperty("AZURE_BUILD_ID")) {
val versionId = property("AZURE_BUILD_ID")
".$versionId"
} else {
version += "-dev"
"-dev"
}

versionFile.writeText(version)
}
}
Expand Down Expand Up @@ -596,9 +600,9 @@ tasks {
writeScript(
pythonBuildDir, "install",
"""
|pip2 uninstall -y osprey JPype-py2
|pip2 install --user 'numpy>=1.6,<1.16'
|pip2 install --user osprey --no-index --find-link=wheelhouse --pre
|python -m pip uninstall -y osprey JPype-py2
|python -m pip install --user 'numpy>=1.6,<1.16'
|python -m pip install --user osprey --no-index --find-link=wheelhouse --pre
""".trimMargin()
)
}
Expand All @@ -611,9 +615,8 @@ tasks {
writeScript(
pythonBuildDir, "install",
"""
|pip3 uninstall -y osprey
|pip3 install --user 'numpy>=1.6,<1.16'
|pip3 install --user osprey --find-link=wheelhouse --pre
|python -m pip uninstall -y osprey
|python -m pip install --user osprey --find-link=wheelhouse --pre
""".trimMargin()
)
}
Expand All @@ -625,7 +628,7 @@ tasks {
doLast {
writeScript(
pythonBuildDir, "uninstall",
"pip2 uninstall -y osprey JPype-py2"
"python -m pip uninstall -y osprey JPype-py2"
)
}
}
Expand All @@ -636,7 +639,7 @@ tasks {
doLast {
writeScript(
pythonBuildDir, "uninstall",
"pip3 uninstall -y osprey"
"python -m pip uninstall -y osprey"
)
}
}
Expand Down
2 changes: 1 addition & 1 deletion examples/python.GMEC/findGMEC.LUTE.design.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

osprey.start()

# run this example AFTER you've trained a LUTE model using findGMEC.LUTE.design.py
# run this example AFTER you've trained a LUTE model using findGMEC.LUTE.train.py

# define the same conformation space as the training script
strand = osprey.Strand('1CC8.ss.pdb')
Expand Down
2 changes: 1 addition & 1 deletion resources/config/version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.2-beta1
3.2

0 comments on commit 3595d64

Please sign in to comment.