It's a Gradle world

| Image by Nam Anh via Unsplash Copyright-free

Gradle is a build automation tool for multi-language software development. It controls the development process in the tasks of compilation and packaging to testing, deployment, and publishing (Wikipedia).

Gradle is a powerful tool. In comparison with Maven, it very often win all tests. In fact, Gradle gives us many more possibilities than Maven. What is also important is that in a similar situation, it simplifies the developer's work and reduces complexity.

In the IAM team, and based on my experience, we found one major flaw in this "ideal picture". The size of a build.gradle file.

Maven parent pom's

In the Maven world, introduced in every company I have worked for, developers introduced at least one parent pom that gives some common behaviour used or which should be used within each project for the company. For example, in such parent pom, we might find a dependency management section, the definition of maven repositories available in the company, or even prepared tasks reducing the complexity of the child pom files.

Gradle hell

In Gradle, we hit a stumbling block. Gradle gives the possibility to define 'multi-project builds', but this is not similar to that described in Maven above. Even so, we cannot just import some Gradle file with 'parent configuration' for any project. As a result, we always ended with one big build.gradle file - hard to read, hard to maintain, and hard to refactor. Moreover, I have often faced this situation, because of such cons, some sections in the file were repeated just because the developers did not find an existing one.

Hopeless situation?

No, no, no! We found a solution for this: scripts importing/applying! Gradle allows us to 'apply' everything that is defined in such a file from a separate Gradle script. Importantly, there is no plugin defined in such a script file, only the appropriate Gradle script.

Solution

Let's look closer to the average build.gradle file:‌

buildscript {
    repositories {
        maven { url 'https://packages.confluent.io/maven/' }
        maven { url 'https://jitpack.io' }
    }
}

plugins {
    id 'java'
    id "org.springframework.boot" version "2.3.3.RELEASE"
    id 'io.spring.dependency-management' version '1.0.10.RELEASE'
    id 'jacoco'
    id "org.sonarqube" version "2.7.1"
    id 'idea'
    id 'com.github.imflog.kafka-schema-registry-gradle-plugin' version '1.0.0'
}

sourceSets {
    contractTest {
        java.srcDir "$projectDir/tests/contract/java"
        resources.srcDir "$projectDir/tests/contract/resources"
        compileClasspath += main.output + test.output
        runtimeClasspath += main.output + test.output
    }
    serviceTest {
        java.srcDir "$projectDir/tests/service/java"
        resources.srcDir "$projectDir/tests/service/resources"
        compileClasspath += main.output + test.output
        runtimeClasspath += main.output + test.output
    }
    unitTest {
        java.srcDir "$projectDir/tests/unit/java"
        resources.srcDir "$projectDir/tests/unit/resources"
        compileClasspath += main.output + test.output
        runtimeClasspath += main.output + test.output
    }
}

task contractTest(type: Test) {
    description = "Consumer Driven Contract - PACT Based"
    group = "verification"

    systemProperty "pact.verifier.publishResults", "true"
    systemProperty "pactbroker.host", "pact-broker.wremitdev.com"
    systemProperty "pactbroker.port", "443"
    systemProperty "pactbroker.scheme", "https"

    testClassesDirs = sourceSets.contractTest.output.classesDirs
    classpath = sourceSets.contractTest.runtimeClasspath

    useJUnitPlatform {
    }
    testLogging {
        events "passed", "skipped", "failed"
    }
    jacoco {
        destinationFile = file("$buildDir/jacoco/contractTest.exec")
    }
}

task serviceTest(type: Test) {
    description = "Service Tests"
    group = "verification"

    testClassesDirs = sourceSets.serviceTest.output.classesDirs
    classpath = sourceSets.serviceTest.runtimeClasspath

    useJUnitPlatform {
    }
    testLogging {
        events "passed", "skipped", "failed"
    }
    jacoco {
        destinationFile = file("$buildDir/jacoco/serviceTest.exec")
    }
}

task unitTest(type: Test) {
    description = "Unit Tests"
    group = "verification"

    testClassesDirs = sourceSets.unitTest.output.classesDirs
    classpath = sourceSets.unitTest.runtimeClasspath

    useJUnitPlatform {
    }
    testLogging {
        events "passed", "skipped", "failed"
    }
    jacoco {
        destinationFile = file("$buildDir/jacoco/test.exec")
    }
}


group = 'com.worldremit'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = '11'

repositories {
    jcenter()
    maven { url 'https://repo.spring.io/milestone' }
    maven { url 'https://packages.confluent.io/maven/' }
    maven { url "https://nexus.shared.k8s.worldremit.com/repository/maven-releases/" }
}

ext {
    avroVersion = "1.10.0"
    confluentVersion = "5.5.1"
    springCloudVersion = "Hoxton.SR7"
    pactProviderVersion = "4.1.7"
    swaggerVersion = "3.0.0"
    manyTypesVersion = "0.0.3"
    logbackVersion = "6.4"
    jacksonVersion = "2.11.1"
    lombokVersion = "1.18.12"
    mapStructVersion = "1.3.1.Final"
    testApacheHttpClientVersion = "4.5.11"
    avroCommonFilesDirectory = "build/generated-avro-avsc/com/worldremit/avro/"
    avroIamCredentialFilesDirectory = "${avroCommonFilesDirectory}iam/credential/events/"
    avroIamAccountFilesDirectory = "${avroCommonFilesDirectory}iam/account/events/"
    avroGeneratedSourcesDir = file("build/generated-sources/avro/java")
}

compileJava.dependsOn(processResources)
compileJava.options.compilerArgs += [
        "-Amapstruct.defaultComponentModel=spring",
        "-Amapstruct.unmappedTargetPolicy=ERROR"
]

dependencyManagement {
    imports {
        mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
    }
}

test {
    useJUnitPlatform()

    testLogging {
        events "passed", "skipped", "failed"
    }
}

jacocoTestReport {
    executionData(fileTree(project.rootDir.absolutePath).include("**/build/jacoco/*.exec"))
    afterEvaluate {
        getClassDirectories().setFrom(classDirectories.files.collect {
            fileTree(dir: it)
        })
    }
    reports {
        xml.enabled true
        html.enabled true
    }
}
sonarqube {
    properties {
        property "sonar.coverage.jacoco.xmlReportPaths", "${projectDir}/build/reports/jacoco/test/jacocoTestReport.xml"
        property "sonar.exclusions", [
                '**/com/worldremit/usermanagement/dto/**',
                '**/com/worldremit/usermanagement/domain/**',
                '**/com/worldremit/usermanagement/configuration/**'
        ]
    }
}

dependencies {
    //AnnotationProcessors
    annotationProcessor "org.springframework.boot:spring-boot-configuration-processor"
    annotationProcessor "org.projectlombok:lombok:$lombokVersion"
    serviceTestAnnotationProcessor "org.projectlombok:lombok:$lombokVersion"
    annotationProcessor "org.mapstruct:mapstruct-processor:$mapStructVersion"

    //Web
    implementation 'org.springframework.boot:spring-boot-starter-actuator'
    implementation 'org.springframework.boot:spring-boot-starter-web'
    implementation 'org.springframework.boot:spring-boot-starter-validation'

    // Kafka cloud stream
    implementation "org.springframework.cloud:spring-cloud-stream-binder-kafka"

    // Avro
    implementation "org.apache.avro:avro:${avroVersion}"
    implementation "io.confluent:kafka-avro-serializer:${confluentVersion}"
    implementation "com.worldremit:many-types:${manyTypesVersion}"

    //Jpa
    implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
    runtimeOnly 'org.postgresql:postgresql'
    serviceTestRuntimeOnly "com.h2database:h2"

    //Lombok
    implementation "org.projectlombok:lombok:$lombokVersion"
    serviceTestImplementation "org.projectlombok:lombok:$lombokVersion"

    //MapStruct
    implementation "org.mapstruct:mapstruct:$mapStructVersion"

    //Liquibase
    implementation 'org.liquibase:liquibase-core'

    //Swagger
    implementation "io.springfox:springfox-boot-starter:$swaggerVersion"

    // Metrics
    implementation "io.micrometer:micrometer-core:latest.release"
    implementation "io.micrometer:micrometer-registry-prometheus:latest.release"

    //Logs
    implementation "net.logstash.logback:logstash-logback-encoder:$logbackVersion"
    implementation "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:$jacksonVersion"
    contractTestImplementation "com.fasterxml.jackson.module:jackson-module-scala_2.13:$jacksonVersion"

    //Tests
    testImplementation("org.springframework.boot:spring-boot-starter-test") {
        exclude group: "org.junit.vintage", module: "junit-vintage-engine"
    }
    testImplementation "org.junit.jupiter:junit-jupiter-api"
    testImplementation "org.junit.jupiter:junit-jupiter-params"
    testImplementation "org.junit.jupiter:junit-jupiter-engine"
    testImplementation 'org.springframework.kafka:spring-kafka-test'
    testImplementation 'com.worldremit:common-test-utils:0.0.0'
    contractTestImplementation "au.com.dius.pact.provider:junit5:$pactProviderVersion"
    contractTestImplementation "au.com.dius.pact.provider:junit5spring:$pactProviderVersion"

    testImplementation "org.apache.httpcomponents:httpclient:$testApacheHttpClientVersion"
}

task addSchemasToCheck {
    dependsOn generateSchema
    doLast {
        def commonNamespace = "com.worldremit.avro."
        def iamCredentialNamespace = "${commonNamespace}iam.credential.events."
        def iamAccountNamespace = "${commonNamespace}iam.credential.events."

        def verifyCompatibility = { avsc, avroDir, namespace ->
            def x = avsc.name.split("\\.")[0]
            def projectSchema = "${avroDir}${avsc.name}"
            def registrySchema = "${namespace}${x}"
            println "Adding compatibility check of file ${projectSchema} with registered schema ${registrySchema}"
            schemaRegistry {
                compatibility {
                    subject(registrySchema, projectSchema)
                }
            }
        }

        def addSchemaDirectory = { avroDir, namespace ->
            layout.files {
                file("${avroDir}").listFiles()
            }.filter { File f ->
                f.name.endsWith('.avsc')
            }.each { avsc ->
                verifyCompatibility(avsc, avroDir, namespace)
            }
        }

        addSchemaDirectory(avroIamCredentialFilesDirectory, iamCredentialNamespace)
        addSchemaDirectory(avroIamAccountFilesDirectory, iamAccountNamespace)
        addSchemaDirectory(avroCommonFilesDirectory, commonNamespace)
    }
}

testSchemasTask.dependsOn(addSchemasToCheck)

task("generateProtocol", type: com.commercehub.gradle.plugin.avro.GenerateAvroProtocolTask) {
    source file("src/main/avro")
    include("**/*.avdl")
    outputDir = file("${buildDir}/generated-avro-avpr")
}

task("generateSchema", type: com.commercehub.gradle.plugin.avro.GenerateAvroSchemaTask) {
    dependsOn generateProtocol
    source file("${buildDir}/generated-avro-avpr")
    include("**/*.avpr")
    outputDir = file("${buildDir}/generated-avro-avsc")
}

task("generateAvro", type: com.commercehub.gradle.plugin.avro.GenerateAvroJavaTask) {
    dependsOn generateSchema
    source file("${buildDir}/generated-avro-avsc")
    include("**/*.avsc")
    outputDir = file(avroGeneratedSourcesDir)
}

compileJava.source(generateAvro.outputs)
sourceSets.main.java.srcDirs += ext.avroGeneratedSourcesDir

We could find many sections that could be moved into separate, well-defined files, i.e. sections related to sonar, Aiven model generation, schema registry validation, definition of test tasks, and so on.

Let's do it.  Sample extracted files are shown below.

test-unit.gradle

sourceSets {
    unitTest {
        java.srcDir "$projectDir/tests/unit/java"
        resources.srcDir "$projectDir/tests/unit/resources"
        compileClasspath += main.output + test.output
        runtimeClasspath += main.output + test.output
    }
}

idea {
    module {
        testSourceDirs += project.sourceSets.unitTest.java.srcDirs
    }
}

configurations {
    unitTestImplementation.extendsFrom testImplementation
    unitTestRuntime.extendsFrom testRuntime
}

task unitTest(type: Test) {
    description = "Unit Tests"
    group = "verification"

    testClassesDirs = sourceSets.unitTest.output.classesDirs
    classpath = sourceSets.unitTest.runtimeClasspath

    useJUnitPlatform {
    }
    testLogging {
        events "passed", "skipped", "failed"
    }
    jacoco {
        destinationFile = file("$buildDir/jacoco/test.exec")
    }
}

test-service.gradle

sourceSets {
    serviceTest {
        java.srcDir "$projectDir/tests/service/java"
        resources.srcDir "$projectDir/tests/service/resources"
        compileClasspath += main.output + test.output
        runtimeClasspath += main.output + test.output
    }
}

idea {
    module {
        testSourceDirs += project.sourceSets.serviceTest.java.srcDirs
    }
}

configurations {
    serviceTestImplementation.extendsFrom testImplementation
    serviceTestRuntime.extendsFrom testRuntime
}

task serviceTest(type: Test) {
    description = "Service Tests"
    group = "verification"

    testClassesDirs = sourceSets.serviceTest.output.classesDirs
    classpath = sourceSets.serviceTest.runtimeClasspath

    useJUnitPlatform {
    }
    testLogging {
        events "passed", "skipped", "failed"
    }
    jacoco {
        destinationFile = file("$buildDir/jacoco/serviceTest.exec")
    }
}

generate-avro.gradle

buildscript {
    repositories {
        jcenter()
    }

    dependencies {
        classpath "com.commercehub.gradle.plugin:gradle-avro-plugin:0.21.0"
    }
}

ext {
    avroGeneratedSourcesDir = file("build/generated-sources/avro/java")
}

task("generateProtocol", type: com.commercehub.gradle.plugin.avro.GenerateAvroProtocolTask) {
    source file("src/main/avro")
    include("**/*.avdl")
    outputDir = file("${buildDir}/generated-avro-avpr")
}

task("generateSchema", type: com.commercehub.gradle.plugin.avro.GenerateAvroSchemaTask) {
    dependsOn generateProtocol
    source file("${buildDir}/generated-avro-avpr")
    include("**/*.avpr")
    outputDir = file("${buildDir}/generated-avro-avsc")
}

task("generateAvro", type: com.commercehub.gradle.plugin.avro.GenerateAvroJavaTask) {
    dependsOn generateSchema
    source file("${buildDir}/generated-avro-avsc")
    include("**/*.avsc")
    outputDir = file(avroGeneratedSourcesDir)
}

compileJava.source(generateAvro.outputs)
sourceSets.main.java.srcDirs += ext.avroGeneratedSourcesDir

Now that we have extracted the scripts, we can now simplify the original build.gradle file:

buildscript {
    repositories {
        maven { url 'https://packages.confluent.io/maven/' }
        maven { url 'https://jitpack.io' }
    }
}

plugins {
    id 'java'
    id "org.springframework.boot" version "2.3.3.RELEASE"
    id 'io.spring.dependency-management' version '1.0.10.RELEASE'
    id 'jacoco'
    id "org.sonarqube" version "2.7.1"
    id 'idea'
    id 'com.github.imflog.kafka-schema-registry-gradle-plugin' version '1.0.0'
}

apply from: 'test-unit.gradle'
apply from: 'test-service.gradle'
apply from: 'test-contract.gradle'
apply from: 'generate-avro.gradle'
apply from: 'validate-avro.gradle'

group = 'com.worldremit'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = '11'

repositories {
    jcenter()
    maven { url 'https://repo.spring.io/milestone' }
    maven { url 'https://packages.confluent.io/maven/' }
    maven { url "https://nexus.shared.k8s.worldremit.com/repository/maven-releases/" }
}

ext {
    avroVersion = "1.10.0"
    confluentVersion = "5.5.1"
    springCloudVersion = "Hoxton.SR7"
    pactProviderVersion = "4.1.7"
    swaggerVersion = "3.0.0"
    manyTypesVersion = "0.0.3"
    logbackVersion = "6.4"
    jacksonVersion = "2.11.1"
    lombokVersion = "1.18.12"
    mapStructVersion = "1.3.1.Final"
    testApacheHttpClientVersion = "4.5.11"
}

compileJava.dependsOn(processResources)
compileJava.options.compilerArgs += [
        "-Amapstruct.defaultComponentModel=spring",
        "-Amapstruct.unmappedTargetPolicy=ERROR"
]

dependencyManagement {
    imports {
        mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
    }
}

test {
    useJUnitPlatform()

    testLogging {
        events "passed", "skipped", "failed"
    }
}

jacocoTestReport {
    executionData(fileTree(project.rootDir.absolutePath).include("**/build/jacoco/*.exec"))
    afterEvaluate {
        getClassDirectories().setFrom(classDirectories.files.collect {
            fileTree(dir: it)
        })
    }
    reports {
        xml.enabled true
        html.enabled true
    }
}
sonarqube {
    properties {
        property "sonar.coverage.jacoco.xmlReportPaths", "${projectDir}/build/reports/jacoco/test/jacocoTestReport.xml"
        property "sonar.exclusions", [
                '**/com/worldremit/usermanagement/dto/**',
                '**/com/worldremit/usermanagement/domain/**',
                '**/com/worldremit/usermanagement/configuration/**'
        ]
    }
}

dependencies {
    //AnnotationProcessors
    annotationProcessor "org.springframework.boot:spring-boot-configuration-processor"
    annotationProcessor "org.projectlombok:lombok:$lombokVersion"
    serviceTestAnnotationProcessor "org.projectlombok:lombok:$lombokVersion"
    annotationProcessor "org.mapstruct:mapstruct-processor:$mapStructVersion"

    //Web
    implementation 'org.springframework.boot:spring-boot-starter-actuator'
    implementation 'org.springframework.boot:spring-boot-starter-web'
    implementation 'org.springframework.boot:spring-boot-starter-validation'

    // Kafka cloud stream
    implementation "org.springframework.cloud:spring-cloud-stream-binder-kafka"

    // Avro
    implementation "org.apache.avro:avro:${avroVersion}"
    implementation "io.confluent:kafka-avro-serializer:${confluentVersion}"
    implementation "com.worldremit:many-types:${manyTypesVersion}"

    //Jpa
    implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
    runtimeOnly 'org.postgresql:postgresql'
    serviceTestRuntimeOnly "com.h2database:h2"

    //Lombok
    implementation "org.projectlombok:lombok:$lombokVersion"
    serviceTestImplementation "org.projectlombok:lombok:$lombokVersion"

    //MapStruct
    implementation "org.mapstruct:mapstruct:$mapStructVersion"

    //Liquibase
    implementation 'org.liquibase:liquibase-core'

    //Swagger
    implementation "io.springfox:springfox-boot-starter:$swaggerVersion"

    // Metrics
    implementation "io.micrometer:micrometer-core:latest.release"
    implementation "io.micrometer:micrometer-registry-prometheus:latest.release"

    //Logs
    implementation "net.logstash.logback:logstash-logback-encoder:$logbackVersion"
    implementation "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:$jacksonVersion"
    contractTestImplementation "com.fasterxml.jackson.module:jackson-module-scala_2.13:$jacksonVersion"

    //Tests
    testImplementation("org.springframework.boot:spring-boot-starter-test") {
        exclude group: "org.junit.vintage", module: "junit-vintage-engine"
    }
    testImplementation "org.junit.jupiter:junit-jupiter-api"
    testImplementation "org.junit.jupiter:junit-jupiter-params"
    testImplementation "org.junit.jupiter:junit-jupiter-engine"
    testImplementation 'org.springframework.kafka:spring-kafka-test'
    testImplementation 'com.worldremit:common-test-utils:0.0.0'
    contractTestImplementation "au.com.dius.pact.provider:junit5:$pactProviderVersion"
    contractTestImplementation "au.com.dius.pact.provider:junit5spring:$pactProviderVersion"

    testImplementation "org.apache.httpcomponents:httpclient:$testApacheHttpClientVersion"
}

Please pay particular attention to the 'apply from' section, where 'magic' appears. Looks better? Of course, it does! :) Now the functionality is well-encapsulated, well-named and easier to understand.  

The future

The possibilities of this mechanism are more large-scale. We can imagine creating central repositories with well-defined scripts ready to apply, just like any other plugin in the Gradle world.

In the next step, we can also provide a script/plugin that will verify the correctness of used dependencies or plugins (Gradle is just a groovy script, we can do anything ;))

Finally, I encourage you and your team to refactor your build script. This will be the first step to simplifying your work with Gradle and reaping benefits of almost 'Maven parent poms'.