Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
mcovarr committed Feb 18, 2021
2 parents 5ca4099 + ade04f9 commit ae2618c
Show file tree
Hide file tree
Showing 30 changed files with 5,742 additions and 228 deletions.
282 changes: 282 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,282 @@
version: 2.1

orbs:
slack: circleci/[email protected]
build-tools: circleci/[email protected]

commands:
install_adoptopenjdk:
description: "Installing adoptopenjdk 8 and setting it as default Java"
steps:
- run: wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add -
- run: sudo add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/
- run: sudo apt update
- run: sudo apt install adoptopenjdk-8-hotspot
- run: sudo update-java-alternatives --set adoptopenjdk-8-hotspot-amd64

jobs:
test:
parameters:
machine-size:
type: string
default: large
branch-or-pr-build:
description: supported values are `branch` or `pr`
type: string
build-type:
type: string
build-mysql:
type: string
default: ""
build-mariadb:
type: string
default: ""
build-postgresql:
type: string
default: ""
machine:
image: ubuntu-2004:202010-01
resource_class: << parameters.machine-size >>
steps:
- when:
condition:
equal: [ pr, << parameters.branch-or-pr-build >> ]
steps:
- run:
name: Halt PR build if there is no PR associated with the commit
command: |
if [[ -z "${CI_PULL_REQUEST}" ]]; then
circleci-agent step halt
else
echo "Build associated with the following PR: '${CI_PULL_REQUEST}'"
fi
- install_adoptopenjdk
- checkout
- when:
condition:
equal: [ pr, << parameters.branch-or-pr-build >> ]
steps:
- build-tools/merge-with-parent
- restore_cache:
key: sbt-cache
- run:
command: src/ci/bin/test.sh
no_output_timeout: 1h
- run:
name: Do tricks to avoid unnecessary cache updates
command: |
find ~/.ivy2/cache -name "ivydata-*.properties" -print -delete
find ~/.sbt -name "*.lock" -print -delete
- save_cache:
key: sbt-cache
paths:
- "~/.ivy2/cache"
- "~/.sbt"
environment:
CIRCLE_COMMIT_RANGE: << pipeline.git.base_revision >>...<< pipeline.git.revision >>
BUILD_TYPE: << parameters.build-type >>
BUILD_MYSQL: << parameters.build-mysql >>
BUILD_MARIADB: << parameters.build-mariadb >>
BUILD_POSTGRESQL: << parameters.build-postgresql >>
BUILD_BRANCH_OR_PR: << parameters.branch-or-pr-build >>

workflows:
all-tests:
jobs:
- test:
name: testSbt-<< matrix.branch-or-pr-build >>
build-type: "sbt"
matrix:
parameters:
branch-or-pr-build: [branch, pr]
- test:
name: testSingleWorkflowRunner-<< matrix.branch-or-pr-build >>
build-type: "singleWorkflowRunner"
matrix:
parameters:
branch-or-pr-build: [branch, pr]
- test:
name: testDbms-<< matrix.branch-or-pr-build >>
build-type: "dbms"
matrix:
parameters:
branch-or-pr-build: [branch, pr]
- test:
name: testHoricromtalDeadlock-<< matrix.branch-or-pr-build >>
build-type: "horicromtalDeadlock"
matrix:
parameters:
branch-or-pr-build: [branch, pr]
- test:
name: testDockerScripts-<< matrix.branch-or-pr-build >>
build-type: "dockerScripts"
matrix:
parameters:
branch-or-pr-build: [branch, pr]
- test:
name: testReferenceDiskManifestBuilderApp-<< matrix.branch-or-pr-build >>
build-type: "referenceDiskManifestBuilderApp"
matrix:
parameters:
branch-or-pr-build: [branch, pr]
- test:
name: testCentaurAws-<< matrix.branch-or-pr-build >>
build-type: "centaurAws"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [branch, pr]
- test:
name: testCentaurBcs-<< matrix.branch-or-pr-build >>
build-type: "centaurBcs"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [branch, pr]
- test:
name: testCentaurDummy-<< matrix.branch-or-pr-build >>
build-type: "centaurDummy"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurEngineUpgradeLocal-<< matrix.branch-or-pr-build >>
build-type: "centaurEngineUpgradeLocal"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurEngineUpgradePapiV2alpha1-<< matrix.branch-or-pr-build >>
build-type: "centaurEngineUpgradePapiV2alpha1"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurHoricromtalPapiV2alpha1-<< matrix.branch-or-pr-build >>
build-type: "centaurHoricromtalPapiV2alpha1"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurHoricromtalPapiV2beta-MySQL-<< matrix.branch-or-pr-build >>
build-type: "centaurHoricromtalPapiV2beta"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurHoricromtalPapiV2beta-MariaDB-<< matrix.branch-or-pr-build >>
build-type: "centaurHoricromtalPapiV2beta"
build-mariadb: "10.3"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurHoricromtalEngineUpgradePapiV2alpha1-MySQL-<< matrix.branch-or-pr-build >>
build-type: "centaurHoricromtalEngineUpgradePapiV2alpha1"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurHoricromtalEngineUpgradePapiV2alpha1-MariaDB-<< matrix.branch-or-pr-build >>
build-type: "centaurHoricromtalEngineUpgradePapiV2alpha1"
build-mariadb: "10.3"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurPapiUpgradePapiV2alpha1-<< matrix.branch-or-pr-build >>
build-type: "centaurPapiUpgradePapiV2alpha1"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurPapiUpgradeNewWorkflowsPapiV2alpha1-<< matrix.branch-or-pr-build >>
build-type: "centaurPapiUpgradeNewWorkflowsPapiV2alpha1"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurLocal-MySQL-<< matrix.branch-or-pr-build >>
build-type: "centaurLocal"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurLocal-Postgresql-<< matrix.branch-or-pr-build >>
build-type: "centaurLocal"
build-postgresql: "11.3"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurPapiV2alpha1-<< matrix.branch-or-pr-build >>
build-type: "centaurPapiV2alpha1"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurPapiV2beta-<< matrix.branch-or-pr-build >>
build-type: "centaurPapiV2beta"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurSlurm-<< matrix.branch-or-pr-build >>
build-type: "centaurSlurm"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurTes-<< matrix.branch-or-pr-build >>
build-type: "centaurTes"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCentaurWdlUpgradeLocal-<< matrix.branch-or-pr-build >>
build-type: "centaurWdlUpgradeLocal"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testCheckPublish-<< matrix.branch-or-pr-build >>
build-type: "checkPublish"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testConformanceLocal-<< matrix.branch-or-pr-build >>
build-type: "conformanceLocal"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testConformancePapiV2beta-<< matrix.branch-or-pr-build >>
build-type: "conformancePapiV2beta"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
- test:
name: testConformanceTesk-<< matrix.branch-or-pr-build >>
build-type: "conformanceTesk"
build-mysql: "5.7"
matrix:
parameters:
branch-or-pr-build: [ branch, pr ]
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
# Cromwell Change Log


## 57 Release Notes

### Breaking configuration change to reference disk support on PAPI v2

Beginning with Cromwell 57, reference disk manifests are now specified completely within Cromwell configuration
rather than through a level of indirection to a manifest file stored in GCS. More details can be found
[here](https://cromwell.readthedocs.io/en/develop/backends/Google#reference-disk-support).

## 56 Release Notes

### Retry with More Memory as workflow option
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ task one {
Int rCopy = radius
}
runtime {
docker: "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:53a002b59dfcd43b4d15e97c1acbeae035ddd1b31a106659a312e9fe65f00afa"
}
}

Expand All @@ -25,7 +25,7 @@ task two{
Int rSquaredCopy = r2
}
runtime {
docker: "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:53a002b59dfcd43b4d15e97c1acbeae035ddd1b31a106659a312e9fe65f00afa"
}
}

Expand Down
2 changes: 1 addition & 1 deletion cromwell.example.backends/PAPIv2.conf
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ backend {
zones: ["us-central1-a", "us-central1-b"]
}

reference-disk-localization-manifest-files = ["gs://gcp-public-data--broad-references/refdisk_manifest.json"]
include "papi_v2_reference_image_manifest.conf"
}
}
}
Expand Down
42 changes: 20 additions & 22 deletions docs/backends/Google.md
Original file line number Diff line number Diff line change
Expand Up @@ -496,9 +496,11 @@ before eventually giving up and running the job. This behavior may be corrected
### Reference Disk Support

Cromwell 55 and later support mounting reference disks from prebuilt GCP disk images as an alternative to localizing large
input reference files on PAPI v2. Within the `config` stanza of a PAPI v2 backend the `reference-disk-localization-manifest-files`
key specifies an array of manifest JSONs in GCS:
input reference files on PAPI v2. Please note the configuration of reference disk manifests has changed starting with
Cromwell 57 and now uses the format documented below.

Within the `config` stanza of a PAPI v2 backend the `reference-disk-localization-manifests`
key specifies an array of reference disk manifests:

```hocon
backend {
Expand All @@ -509,32 +511,28 @@ backend {
actor-factory = "cromwell.backend.google.pipelines.v2beta.PipelinesApiLifecycleActorFactory"
config {
...
reference-disk-localization-manifest-files = ["gs://path/to/a/reference/disk/manifest.json"]
reference-disk-localization-manifests = [
{
"imageIdentifier" : "projects/broad-dsde-cromwell-dev/global/images/broad-references-disk-image",
"diskSizeGb" : 500,
"files" : [ {
"path" : "gcp-public-data--broad-references/Homo_sapiens_assembly19_1000genomes_decoy/Homo_sapiens_assembly19_1000genomes_decoy.fasta.nhr",
"crc32c" : 407769621
}, {
"path" : "gcp-public-data--broad-references/Homo_sapiens_assembly19_1000genomes_decoy/Homo_sapiens_assembly19_1000genomes_decoy.fasta.sa",
"crc32c" : 1902048083
},
...
},
...
]
...
}
}
}
}
```

Reference manifest JSONs have a format like:

```json
{
"imageIdentifier" : "projects/my_project/global/images/my-references-disk-image",
"diskSizeGb" : 30,
"files" : [ {
"path" : "my-references/enormous_reference.bam",
"crc32c" : 407769621
}, {
"path" : "my-references/enormous_reference.bam.bai",
"crc32c" : 1902048083
},
...
]
}
```

Reference disk usage is an opt-in feature, so workflow submissions must specify this workflow option:

```json
Expand All @@ -552,7 +550,7 @@ reference image without the leading `gs://`, Cromwell would
arrange for a reference disk based on this image to be mounted and for the call's input to refer to the
copy of the file on the reference disk, bypassing localization of the input.

The Cromwell git repository includes a Java-based tool to facilitate the creation of manifest files called
The Cromwell git repository includes a Java-based tool to facilitate the creation of manifests called
[CromwellRefdiskManifestCreatorApp](https://github.com/broadinstitute/cromwell/tree/develop/CromwellRefdiskManifestCreator).
Please see the help command of that tool for more details.

Expand Down
Loading

0 comments on commit ae2618c

Please sign in to comment.