Merge branch 'develop'

.travis.yml

@@ -6,5 +6,5 @@ jdk:
 before_script:
   - mysql -e "create database IF NOT EXISTS cromwell_test;"
 
-script: sbt clean coverage nodocker:test
+script: sbt -Dbackend.shared-filesystem.localization.0=copy clean coverage nodocker:test
 after_success: sbt coveralls

build.sbt

@@ -3,14 +3,19 @@ import sbtassembly.Plugin._
 import sbtrelease.ReleasePlugin._
 
 name := "cromwell"
-version := "0.9"
+
+version := "0.10"
+
 organization := "org.broadinstitute"
 
 scalaVersion := "2.11.7"
 
 val sprayV = "1.3.2"
+
 val DowngradedSprayV = "1.3.1"
+
 val akkaV = "2.3.12"
+
 val googleClientApiV = "1.20.0"
 
 libraryDependencies ++= Seq(
@@ -36,6 +41,7 @@ libraryDependencies ++= Seq(
   "com.google.api-client" % "google-api-client-jackson2" % googleClientApiV,
   "com.google.oauth-client" % "google-oauth-client" % googleClientApiV,
   "mysql" % "mysql-connector-java" % "5.1.35",
+  "org.scalaz" % "scalaz-core_2.11" % "7.1.3",
   //---------- Test libraries -------------------//
   "io.spray" %% "spray-testkit" % sprayV % Test,
   "org.scalatest" %% "scalatest" % "2.2.5" % Test,

src/main/java/cromwell/parser/BackendType.java

@@ -1,5 +1,5 @@
 package cromwell.parser;
 
 public enum BackendType {
-    LOCAL, JES;
+    LOCAL, JES, SGE;
 }

src/main/java/cromwell/parser/MemorySize.java

@@ -11,16 +11,16 @@ public enum MemorySize {
     GiB(KiB.multiplier * MiB.multiplier, "GiB"),
     TiB(KiB.multiplier * GiB.multiplier, "TiB"),
     KB(1 << 10, "KB"),
-    MB(KB.multiplier * KB.multiplier, "MB"),
-    GB(KB.multiplier * MB.multiplier, "GB"),
-    TB(KB.multiplier * GB.multiplier, "TB");
+    MB(KB.multiplier * KB.multiplier, "M", "MB"),
+    GB(KB.multiplier * MB.multiplier, "G", "GB"),
+    TB(KB.multiplier * GB.multiplier, "T", "TB");
 
     public final long multiplier;
-    public final String suffix;
+    public final String[] suffixes;
 
-    MemorySize(long multiplier, String suffix) {
+    MemorySize(long multiplier, String... suffixes) {
         this.multiplier = multiplier;
-        this.suffix = suffix;
+        this.suffixes = suffixes;
     }
 
     /** Convert from the units of this memory size to individual bytes. */

src/main/java/cromwell/parser/RuntimeKey.java

@@ -10,7 +10,7 @@ public enum RuntimeKey {
     CPU("cpu", JES),
     DEFAULT_DISKS("defaultDisks", JES),
     DEFAULT_ZONES("defaultZones", JES),
-    DOCKER("docker", new BackendType[]{JES}, LOCAL),
+    DOCKER("docker", new BackendType[]{JES}, LOCAL), // Alternate constructor due to both optional and mandatory backends
     FAIL_ON_STDERR("failOnStderr", LOCAL, JES),
     MEMORY("memory", JES),
     PREEMPTIBLE("preemptible", JES);

src/main/migrations/changelog.xml

@@ -13,4 +13,6 @@
     <include file="changesets/lengthen_wdl_value.xml" relativeToChangelogFile="true" />
     <include file="changesets/add_index_in_execution.xml" relativeToChangelogFile="true" />
     <include file="changesets/rename_iteration_to_index.xml" relativeToChangelogFile="true" />
+    <include file="changesets/sge.xml" relativeToChangelogFile="true" />
+    <include file="changesets/change_execution_unique_constraint.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>

src/main/migrations/changesets/change_execution_unique_constraint.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="tjeandet" id="change_execution_unique_constraint">
+        <comment>
+            Change unique constraint for Execution Table to include IDX column.  For MySQL this requires
+            first dropping the foreign key constraint, which we then restore after adding back the enhanced
+            unique constraint.  Both the unique and foreign key constraints are added back with shorter
+            Oracle-friendly (i.e. not more than 30 character) names.
+
+            Also rename the INDEX column to IDX because some databases will have a problem with that name
+        </comment>
+
+        <dropForeignKeyConstraint baseTableName="EXECUTION" constraintName="FK_EXECUTION_WORKFLOW_EXECUTION_ID" />
+
+        <dropUniqueConstraint
+                tableName="EXECUTION"
+                constraintName="UK_EX_WORKFLOW_EXECUTION_ID" />
+
+        <addUniqueConstraint
+                tableName="EXECUTION"
+                columnNames="WORKFLOW_EXECUTION_ID, CALL_FQN, INDEX"
+                constraintName="UK_WORKFLOW_CALL_INDEX"/>
+
+        <addForeignKeyConstraint constraintName="FK_EXECUTION_WORKFLOW"
+                                 baseTableName="EXECUTION" baseColumnNames="WORKFLOW_EXECUTION_ID"
+                                 referencedTableName="WORKFLOW_EXECUTION"
+                                 referencedColumnNames="WORKFLOW_EXECUTION_ID"/>
+
+        <renameColumn tableName="EXECUTION" oldColumnName="INDEX" newColumnName="IDX" columnDataType="INT"/>
+    </changeSet>
+
+</databaseChangeLog>

src/main/migrations/changesets/sge.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+    <changeSet author="scottfrazer" id="sge">
+
+        <createTable tableName="SGE_JOB">
+            <column autoIncrement="true" name="SGE_JOB_ID" type="INT">
+                <constraints primaryKey="true" primaryKeyName="PK_SGE_JOB"/>
+            </column>
+            <column name="EXECUTION_ID" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="SGE_JOB_NUMBER" type="INT">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+
+        <modifySql dbms="mysql">
+            <append value=" ENGINE=INNODB"/>
+        </modifySql>
+    </changeSet>
+
+    <changeSet author="scottfrazer" id="db_schema_constraints">
+        <addForeignKeyConstraint constraintName="FK_SGE_JOB_EXECUTION_ID"
+                                 baseTableName="SGE_JOB" baseColumnNames="EXECUTION_ID"
+                                 referencedTableName="EXECUTION" referencedColumnNames="EXECUTION_ID"/>
+    </changeSet>
+</databaseChangeLog>

src/main/resources/3step.wdl

@@ -8,17 +8,20 @@ task ps {
 }
 
 task cgrep {
+  String pattern
+  File in_file
   command {
-    grep '${pattern}' ${File in_file} | wc -l
+    grep '${pattern}' ${in_file} | wc -l
   }
   output {
     Int count = read_int(stdout())
   }
 }
 
 task wc {
+  File in_file
   command {
-    cat ${File in_file} | wc -l
+    cat ${in_file} | wc -l
   }
   output {
     Int count = read_int(stdout())

src/main/resources/application.conf

@@ -34,7 +34,8 @@ spray.can {
 
 
 backend {
-  backend = "local" // Either "JES" or "local", case insensitive
+  // Either "jes", "local", or "sge" (case insensitive)
+  backend = "local"
 
   // If backend is JES, must supply a 'jes' stanza:
   //jes {
@@ -43,6 +44,27 @@ backend {
   //  baseExecutionBucket = ""
   //  endpointUrl = ""
   //}
+
+  // For any backend that assumes a local filesystem (local, sge)
+  shared-filesystem {
+    // Root directory where Cromwell writes job results to.  This directory must be
+    // visible and writeable to the Cromwell process as well as the jobs that Cromwell
+    // launches
+    root: "cromwell-executions"
+
+    // Cromwell makes a link to your input files within <root>/<workflow UUID>/workflow-inputs
+    // The following are strategies used to make those links.  They are ordered.  If one fails
+    // The next one is tried:
+    //
+    // hard-link: attempt to create a hard-link to the file
+    // copy: copy the file
+    // soft-link: create a symbolic link to the file
+    //
+    // NOTE: soft-link will be skipped for Docker jobs
+    localization: [
+      "hard-link", "soft-link", "copy"
+    ]
+  }
 }
 
 // If authenticating with Google (e.g. if backend is 'JES') you must supply a top level stanza 'google':

src/main/resources/grammar.hgr

@@ -277,7 +277,7 @@ grammar {
       (-:unary) $e = :plus $e -> UnaryPlus(expression=$1)
       (-:unary) $e = :dash $e -> UnaryNegation(expression=$1)
       (*:left) $e = :identifier <=> :lparen list($e, :comma) :rparen -> FunctionCall(name=$0, params=$2)
-      (*:left) $e = :identifier <=> :lsquare $e :rsquare -> ArrayIndex(lhs=$0, rhs=$2)
+      (*:left) $e = :identifier <=> :lsquare $e :rsquare -> ArrayOrMapLookup(lhs=$0, rhs=$2)
       (*:left) $e = :identifier <=> :dot :identifier -> MemberAccess(lhs=$0, rhs=$2)
       # TODO: is there a better object literal syntax?
       (*:left) $e = :object :lbrace list($object_kv, :comma) :rbrace -> ObjectLiteral(map=$2)

src/main/resources/logback.xml

@@ -12,9 +12,11 @@
           <pattern>%date [%thread] %-5level %logger{35} - %msg%n</pattern>
         </encoder>
       </appender>
-      <root level="TRACE">
+      <root level="INFO">
         <appender-ref ref="SERVER_APPENDER" />
       </root>
+      <logger name="com.zaxxer.hikari" level="ERROR"/>
+      <logger name="HikariPool" level="ERROR"/>
     </then>
     <else>
       <appender name="CONSOLE_APPENDER" class="ch.qos.logback.core.ConsoleAppender">

src/main/scala/cromwell/Main.scala

@@ -97,7 +97,7 @@ object Main extends App {
 
       inputs foreach { case (k, v) => Log.info(s"input: $k => $v") }
       val singleWorkflowRunner = SingleWorkflowRunnerActor.props(wdlSource, wdlJson, inputs, workflowManagerSystem.workflowManagerActor)
-      workflowManagerSystem.actorSystem.actorOf(singleWorkflowRunner)
+      workflowManagerSystem.actorSystem.actorOf(singleWorkflowRunner, "SingleWorkflowRunnerActor")
       workflowManagerSystem.actorSystem.awaitTermination()
       // And now we just wait for the magic to happen
     } catch {
@@ -146,6 +146,13 @@ object Main extends App {
         |  via this sub-command and the 'validate' subcommand should
         |  be used for full validation
         |
+        |highlight <WDL file> <html|console>
+        |
+        |  Reformats and colorizes/tags a WDL file. The second
+        |  parameter is the output type.  "html" will output the WDL
+        |  file with <span> tags around elements.  "console" mode
+        |  will output colorized text to the terminal
+        |
         |server
         |
         |  Starts a web server on port 8000.  See the web server

src/main/scala/cromwell/binding/AstTools.scala

@@ -150,7 +150,12 @@ object AstTools {
     }
   }
 
-  /* All MemberAccess ASTs that are not contained in other MemberAccess ASTs */
+  /*
+    All MemberAccess ASTs that are not contained in other MemberAccess ASTs
+
+    The reason this returns a collection would be expressions such as "a.b.c + a.b.d", each one of those
+    would have its own MemberAccess - "a.b.c" and "a.b.d"
+  */
   def findTopLevelMemberAccesses(expr: AstNode): Iterable[Ast] = expr.findAstsWithTrail("MemberAccess").filterNot {
     case(k, v) => v.exists{case a:Ast => a.getName == "MemberAccess"}
   }.keys

src/main/scala/cromwell/binding/AttributeMap.scala

@@ -0,0 +1,27 @@
+package cromwell.binding
+
+import cromwell.parser.{BackendType, RuntimeKey}
+import AttributeMap._
+
+object AttributeMap {
+  implicit class EnhancedBackendType(val backendType: BackendType) extends AnyVal {
+    def supportedKeys: Set[RuntimeKey] = for {
+      key <- RuntimeKey.values().toSet
+      if key.supports(backendType)
+    } yield key
+  }
+}
+
+case class AttributeMap(attrs: Map[String, String]) {
+  def get(key: RuntimeKey): Option[String] = attrs.get(key.key)
+
+  def unsupportedKeys(backendType: BackendType): Seq[String] = {
+    val supportedKeys = backendType.supportedKeys map { _.key }
+    val unsupportedKeys = attrs.keySet -- supportedKeys
+
+    if (unsupportedKeys.isEmpty) Vector.empty
+    else Vector(s"Found unsupported keys for backend '$backendType': " + unsupportedKeys.toSeq.sorted.mkString(", "))
+  }
+}
+
+

src/main/scala/cromwell/binding/Call.scala

@@ -2,8 +2,7 @@ package cromwell.binding
 
 import cromwell.binding.AstTools.EnhancedAstNode
 import cromwell.parser.WdlParser.{Ast, SyntaxError, Terminal}
-
-import scala.util.{Success, Try}
+import scala.util.Try
 import scala.language.postfixOps
 
 object Call {
@@ -18,6 +17,7 @@ object Call {
     }
 
     val taskName = ast.getAttribute("task").sourceString
+
     val task = WdlNamespace.findTask(taskName, namespaces, tasks) getOrElse {
       throw new SyntaxError(wdlSyntaxErrorFormatter.callReferencesBadTaskName(ast, taskName))
     }
@@ -43,7 +43,9 @@ object Call {
       }
     }
 
-    new Call(alias, taskName, task, callInputSectionMappings, parent)
+    val prerequisiteCallNames = callInputSectionMappings.values flatMap { _.prerequisiteCallNames } toSet
+
+    new Call(alias, taskName, task, prerequisiteCallNames, callInputSectionMappings, parent)
   }
 
   private def processCallInput(ast: Ast,
@@ -71,58 +73,33 @@ object Call {
 case class Call(alias: Option[String],
                 taskFqn: FullyQualifiedName,
                 task: Task,
+                prerequisiteCallNames: Set[LocallyQualifiedName],
                 inputMappings: Map[String, WdlExpression],
                 parent: Option[Scope]) extends Scope {
   val name: String = alias getOrElse taskFqn
 
-  private def unsatisfiedTaskInputs: Seq[TaskInput] = task.inputs.filterNot {case i => inputMappings.contains(i.name)}
+  override lazy val prerequisiteScopes: Set[Scope] = {
+    val parent = this.parent.get // FIXME: In a world where Call knows it has a parent this wouldn't be icky
+    val parentPrereq = if (parent == this.rootScope) Nil else Set(parent)
+    prerequisiteCalls ++ parentPrereq
+  }
 
   /**
    * Returns a Seq[WorkflowInput] representing the inputs to the call that are
    * needed before its command can be constructed. This excludes inputs that
    * are satisfied via the 'input' section of the Call definition.
    */
-  def unsatisfiedInputs: Seq[WorkflowInput] = unsatisfiedTaskInputs.map {i =>
-    WorkflowInput(s"$fullyQualifiedName.${i.name}", i.wdlType, i.postfixQuantifier)
-  }
+  def unsatisfiedInputs: Seq[WorkflowInput] = for {
+    i <- task.inputs if !inputMappings.contains(i.name)
+  } yield WorkflowInput(s"$fullyQualifiedName.${i.name}", i.wdlType, i.postfixQuantifier)
 
   override def toString: String = s"[Call name=$name, task=$task]"
 
-  /**
-   * Find all calls upon which this call immediately depends, i.e. the result of this
-   * does not include transitive dependencies.  Currently this only works for member
-   * access expressions with a literal LHS, e.g.:
-   *
-   * {{{
-   *   call cgrep {
-   *     input: in_file=ps.procs
-   *   }
-   * }}}
-   *
-   * Here `ps` would be the prerequisite call for `cgrep`.
-   *
-   * Calls are de-duplicated into a returned `Set`, so if one call expresses dependencies on
-   * a prerequisite call multiple times (i.e. has multiple inputs depending on multiple outputs
-   * of a prerequisite call), that prerequisite call will appear only once in the output.
-   */
-  /*
-    TODO/FIXME: Not happy w/ having to include the namespace but since Calls are now built before there's a namespace the Call loses it's WdlNamespace field.
-   */
-  def prerequisiteCalls(namespace: NamespaceWithWorkflow): Iterable[Call] = {
-    for {
-      expr <- inputMappings.values
-      ast <- AstTools.findTopLevelMemberAccesses(expr.ast)
-      call <- namespace.getCallFromMemberAccessAst(ast) match {
-        case Success(c:Call) => Vector(c)
-        case _ => Vector()
-      }
-    } yield call
-  }
-
   /**
    * Instantiate the abstract command line corresponding to this call using the specified inputs.
    */
-  def instantiateCommandLine(inputs: CallInputs, functions: WdlFunctions): Try[String] = task.instantiateCommand(inputs, functions)
+  def instantiateCommandLine(inputs: CallInputs, functions: WdlFunctions): Try[String] =
+    task.instantiateCommand(inputs, functions)
 
   /**
    * Return the docker configuration value associated with this `Call`, if any.