WorldRemit Technology Blog
| Image by Nam Anh via Unsplash Copyright-free
Gradle is a build automation tool for multi-language software development. It controls the development process in the tasks of compilation and packaging to testing, deployment, and publishing (Wikipedia).
Gradle is a powerful tool. In comparison with Maven, it very often win all tests. In fact, Gradle gives us many more possibilities than Maven. What is also important is that in a similar situation, it simplifies the developer's work and reduces complexity.
In the IAM team, and based on my experience, we found one major flaw in this "ideal picture". The size of a build.gradle file.
Maven parent pom's
In the Maven world, introduced in every company I have worked for, developers introduced at least one parent pom that gives some common behaviour used or which should be used within each project for the company. For example, in such parent pom, we might find a dependency management section, the definition of maven repositories available in the company, or even prepared tasks reducing the complexity of the child pom files.
Gradle hell
In Gradle, we hit a stumbling block. Gradle gives the possibility to define 'multi-project builds', but this is not similar to that described in Maven above. Even so, we cannot just import some Gradle file with 'parent configuration' for any project. As a result, we always ended with one big build.gradle file - hard to read, hard to maintain, and hard to refactor. Moreover, I have often faced this situation, because of such cons, some sections in the file were repeated just because the developers did not find an existing one.
Hopeless situation?
No, no, no! We found a solution for this: scripts importing/applying! Gradle allows us to 'apply' everything that is defined in such a file from a separate Gradle script. Importantly, there is no plugin defined in such a script file, only the appropriate Gradle script.
Solution
Let's look closer to the average build.gradle file:
buildscript {
repositories {
maven { url 'https://packages.confluent.io/maven/' }
maven { url 'https://jitpack.io' }
}
}
plugins {
id 'java'
id "org.springframework.boot" version "2.3.3.RELEASE"
id 'io.spring.dependency-management' version '1.0.10.RELEASE'
id 'jacoco'
id "org.sonarqube" version "2.7.1"
id 'idea'
id 'com.github.imflog.kafka-schema-registry-gradle-plugin' version '1.0.0'
}
sourceSets {
contractTest {
java.srcDir "$projectDir/tests/contract/java"
resources.srcDir "$projectDir/tests/contract/resources"
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
}
serviceTest {
java.srcDir "$projectDir/tests/service/java"
resources.srcDir "$projectDir/tests/service/resources"
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
}
unitTest {
java.srcDir "$projectDir/tests/unit/java"
resources.srcDir "$projectDir/tests/unit/resources"
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
}
}
task contractTest(type: Test) {
description = "Consumer Driven Contract - PACT Based"
group = "verification"
systemProperty "pact.verifier.publishResults", "true"
systemProperty "pactbroker.host", "pact-broker.wremitdev.com"
systemProperty "pactbroker.port", "443"
systemProperty "pactbroker.scheme", "https"
testClassesDirs = sourceSets.contractTest.output.classesDirs
classpath = sourceSets.contractTest.runtimeClasspath
useJUnitPlatform {
}
testLogging {
events "passed", "skipped", "failed"
}
jacoco {
destinationFile = file("$buildDir/jacoco/contractTest.exec")
}
}
task serviceTest(type: Test) {
description = "Service Tests"
group = "verification"
testClassesDirs = sourceSets.serviceTest.output.classesDirs
classpath = sourceSets.serviceTest.runtimeClasspath
useJUnitPlatform {
}
testLogging {
events "passed", "skipped", "failed"
}
jacoco {
destinationFile = file("$buildDir/jacoco/serviceTest.exec")
}
}
task unitTest(type: Test) {
description = "Unit Tests"
group = "verification"
testClassesDirs = sourceSets.unitTest.output.classesDirs
classpath = sourceSets.unitTest.runtimeClasspath
useJUnitPlatform {
}
testLogging {
events "passed", "skipped", "failed"
}
jacoco {
destinationFile = file("$buildDir/jacoco/test.exec")
}
}
group = 'com.worldremit'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = '11'
repositories {
jcenter()
maven { url 'https://repo.spring.io/milestone' }
maven { url 'https://packages.confluent.io/maven/' }
maven { url "https://nexus.shared.k8s.worldremit.com/repository/maven-releases/" }
}
ext {
avroVersion = "1.10.0"
confluentVersion = "5.5.1"
springCloudVersion = "Hoxton.SR7"
pactProviderVersion = "4.1.7"
swaggerVersion = "3.0.0"
manyTypesVersion = "0.0.3"
logbackVersion = "6.4"
jacksonVersion = "2.11.1"
lombokVersion = "1.18.12"
mapStructVersion = "1.3.1.Final"
testApacheHttpClientVersion = "4.5.11"
avroCommonFilesDirectory = "build/generated-avro-avsc/com/worldremit/avro/"
avroIamCredentialFilesDirectory = "${avroCommonFilesDirectory}iam/credential/events/"
avroIamAccountFilesDirectory = "${avroCommonFilesDirectory}iam/account/events/"
avroGeneratedSourcesDir = file("build/generated-sources/avro/java")
}
compileJava.dependsOn(processResources)
compileJava.options.compilerArgs += [
"-Amapstruct.defaultComponentModel=spring",
"-Amapstruct.unmappedTargetPolicy=ERROR"
]
dependencyManagement {
imports {
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
}
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}
jacocoTestReport {
executionData(fileTree(project.rootDir.absolutePath).include("**/build/jacoco/*.exec"))
afterEvaluate {
getClassDirectories().setFrom(classDirectories.files.collect {
fileTree(dir: it)
})
}
reports {
xml.enabled true
html.enabled true
}
}
sonarqube {
properties {
property "sonar.coverage.jacoco.xmlReportPaths", "${projectDir}/build/reports/jacoco/test/jacocoTestReport.xml"
property "sonar.exclusions", [
'**/com/worldremit/usermanagement/dto/**',
'**/com/worldremit/usermanagement/domain/**',
'**/com/worldremit/usermanagement/configuration/**'
]
}
}
dependencies {
//AnnotationProcessors
annotationProcessor "org.springframework.boot:spring-boot-configuration-processor"
annotationProcessor "org.projectlombok:lombok:$lombokVersion"
serviceTestAnnotationProcessor "org.projectlombok:lombok:$lombokVersion"
annotationProcessor "org.mapstruct:mapstruct-processor:$mapStructVersion"
//Web
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.springframework.boot:spring-boot-starter-validation'
// Kafka cloud stream
implementation "org.springframework.cloud:spring-cloud-stream-binder-kafka"
// Avro
implementation "org.apache.avro:avro:${avroVersion}"
implementation "io.confluent:kafka-avro-serializer:${confluentVersion}"
implementation "com.worldremit:many-types:${manyTypesVersion}"
//Jpa
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
runtimeOnly 'org.postgresql:postgresql'
serviceTestRuntimeOnly "com.h2database:h2"
//Lombok
implementation "org.projectlombok:lombok:$lombokVersion"
serviceTestImplementation "org.projectlombok:lombok:$lombokVersion"
//MapStruct
implementation "org.mapstruct:mapstruct:$mapStructVersion"
//Liquibase
implementation 'org.liquibase:liquibase-core'
//Swagger
implementation "io.springfox:springfox-boot-starter:$swaggerVersion"
// Metrics
implementation "io.micrometer:micrometer-core:latest.release"
implementation "io.micrometer:micrometer-registry-prometheus:latest.release"
//Logs
implementation "net.logstash.logback:logstash-logback-encoder:$logbackVersion"
implementation "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:$jacksonVersion"
contractTestImplementation "com.fasterxml.jackson.module:jackson-module-scala_2.13:$jacksonVersion"
//Tests
testImplementation("org.springframework.boot:spring-boot-starter-test") {
exclude group: "org.junit.vintage", module: "junit-vintage-engine"
}
testImplementation "org.junit.jupiter:junit-jupiter-api"
testImplementation "org.junit.jupiter:junit-jupiter-params"
testImplementation "org.junit.jupiter:junit-jupiter-engine"
testImplementation 'org.springframework.kafka:spring-kafka-test'
testImplementation 'com.worldremit:common-test-utils:0.0.0'
contractTestImplementation "au.com.dius.pact.provider:junit5:$pactProviderVersion"
contractTestImplementation "au.com.dius.pact.provider:junit5spring:$pactProviderVersion"
testImplementation "org.apache.httpcomponents:httpclient:$testApacheHttpClientVersion"
}
task addSchemasToCheck {
dependsOn generateSchema
doLast {
def commonNamespace = "com.worldremit.avro."
def iamCredentialNamespace = "${commonNamespace}iam.credential.events."
def iamAccountNamespace = "${commonNamespace}iam.credential.events."
def verifyCompatibility = { avsc, avroDir, namespace ->
def x = avsc.name.split("\\.")[0]
def projectSchema = "${avroDir}${avsc.name}"
def registrySchema = "${namespace}${x}"
println "Adding compatibility check of file ${projectSchema} with registered schema ${registrySchema}"
schemaRegistry {
compatibility {
subject(registrySchema, projectSchema)
}
}
}
def addSchemaDirectory = { avroDir, namespace ->
layout.files {
file("${avroDir}").listFiles()
}.filter { File f ->
f.name.endsWith('.avsc')
}.each { avsc ->
verifyCompatibility(avsc, avroDir, namespace)
}
}
addSchemaDirectory(avroIamCredentialFilesDirectory, iamCredentialNamespace)
addSchemaDirectory(avroIamAccountFilesDirectory, iamAccountNamespace)
addSchemaDirectory(avroCommonFilesDirectory, commonNamespace)
}
}
testSchemasTask.dependsOn(addSchemasToCheck)
task("generateProtocol", type: com.commercehub.gradle.plugin.avro.GenerateAvroProtocolTask) {
source file("src/main/avro")
include("**/*.avdl")
outputDir = file("${buildDir}/generated-avro-avpr")
}
task("generateSchema", type: com.commercehub.gradle.plugin.avro.GenerateAvroSchemaTask) {
dependsOn generateProtocol
source file("${buildDir}/generated-avro-avpr")
include("**/*.avpr")
outputDir = file("${buildDir}/generated-avro-avsc")
}
task("generateAvro", type: com.commercehub.gradle.plugin.avro.GenerateAvroJavaTask) {
dependsOn generateSchema
source file("${buildDir}/generated-avro-avsc")
include("**/*.avsc")
outputDir = file(avroGeneratedSourcesDir)
}
compileJava.source(generateAvro.outputs)
sourceSets.main.java.srcDirs += ext.avroGeneratedSourcesDir
We could find many sections that could be moved into separate, well-defined files, i.e. sections related to sonar, Aiven model generation, schema registry validation, definition of test tasks, and so on.
Let's do it. Sample extracted files are shown below.
test-unit.gradle
sourceSets {
unitTest {
java.srcDir "$projectDir/tests/unit/java"
resources.srcDir "$projectDir/tests/unit/resources"
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
}
}
idea {
module {
testSourceDirs += project.sourceSets.unitTest.java.srcDirs
}
}
configurations {
unitTestImplementation.extendsFrom testImplementation
unitTestRuntime.extendsFrom testRuntime
}
task unitTest(type: Test) {
description = "Unit Tests"
group = "verification"
testClassesDirs = sourceSets.unitTest.output.classesDirs
classpath = sourceSets.unitTest.runtimeClasspath
useJUnitPlatform {
}
testLogging {
events "passed", "skipped", "failed"
}
jacoco {
destinationFile = file("$buildDir/jacoco/test.exec")
}
}
test-service.gradle
sourceSets {
serviceTest {
java.srcDir "$projectDir/tests/service/java"
resources.srcDir "$projectDir/tests/service/resources"
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
}
}
idea {
module {
testSourceDirs += project.sourceSets.serviceTest.java.srcDirs
}
}
configurations {
serviceTestImplementation.extendsFrom testImplementation
serviceTestRuntime.extendsFrom testRuntime
}
task serviceTest(type: Test) {
description = "Service Tests"
group = "verification"
testClassesDirs = sourceSets.serviceTest.output.classesDirs
classpath = sourceSets.serviceTest.runtimeClasspath
useJUnitPlatform {
}
testLogging {
events "passed", "skipped", "failed"
}
jacoco {
destinationFile = file("$buildDir/jacoco/serviceTest.exec")
}
}
generate-avro.gradle
buildscript {
repositories {
jcenter()
}
dependencies {
classpath "com.commercehub.gradle.plugin:gradle-avro-plugin:0.21.0"
}
}
ext {
avroGeneratedSourcesDir = file("build/generated-sources/avro/java")
}
task("generateProtocol", type: com.commercehub.gradle.plugin.avro.GenerateAvroProtocolTask) {
source file("src/main/avro")
include("**/*.avdl")
outputDir = file("${buildDir}/generated-avro-avpr")
}
task("generateSchema", type: com.commercehub.gradle.plugin.avro.GenerateAvroSchemaTask) {
dependsOn generateProtocol
source file("${buildDir}/generated-avro-avpr")
include("**/*.avpr")
outputDir = file("${buildDir}/generated-avro-avsc")
}
task("generateAvro", type: com.commercehub.gradle.plugin.avro.GenerateAvroJavaTask) {
dependsOn generateSchema
source file("${buildDir}/generated-avro-avsc")
include("**/*.avsc")
outputDir = file(avroGeneratedSourcesDir)
}
compileJava.source(generateAvro.outputs)
sourceSets.main.java.srcDirs += ext.avroGeneratedSourcesDir
Now that we have extracted the scripts, we can now simplify the original build.gradle file:
buildscript {
repositories {
maven { url 'https://packages.confluent.io/maven/' }
maven { url 'https://jitpack.io' }
}
}
plugins {
id 'java'
id "org.springframework.boot" version "2.3.3.RELEASE"
id 'io.spring.dependency-management' version '1.0.10.RELEASE'
id 'jacoco'
id "org.sonarqube" version "2.7.1"
id 'idea'
id 'com.github.imflog.kafka-schema-registry-gradle-plugin' version '1.0.0'
}
apply from: 'test-unit.gradle'
apply from: 'test-service.gradle'
apply from: 'test-contract.gradle'
apply from: 'generate-avro.gradle'
apply from: 'validate-avro.gradle'
group = 'com.worldremit'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = '11'
repositories {
jcenter()
maven { url 'https://repo.spring.io/milestone' }
maven { url 'https://packages.confluent.io/maven/' }
maven { url "https://nexus.shared.k8s.worldremit.com/repository/maven-releases/" }
}
ext {
avroVersion = "1.10.0"
confluentVersion = "5.5.1"
springCloudVersion = "Hoxton.SR7"
pactProviderVersion = "4.1.7"
swaggerVersion = "3.0.0"
manyTypesVersion = "0.0.3"
logbackVersion = "6.4"
jacksonVersion = "2.11.1"
lombokVersion = "1.18.12"
mapStructVersion = "1.3.1.Final"
testApacheHttpClientVersion = "4.5.11"
}
compileJava.dependsOn(processResources)
compileJava.options.compilerArgs += [
"-Amapstruct.defaultComponentModel=spring",
"-Amapstruct.unmappedTargetPolicy=ERROR"
]
dependencyManagement {
imports {
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
}
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}
jacocoTestReport {
executionData(fileTree(project.rootDir.absolutePath).include("**/build/jacoco/*.exec"))
afterEvaluate {
getClassDirectories().setFrom(classDirectories.files.collect {
fileTree(dir: it)
})
}
reports {
xml.enabled true
html.enabled true
}
}
sonarqube {
properties {
property "sonar.coverage.jacoco.xmlReportPaths", "${projectDir}/build/reports/jacoco/test/jacocoTestReport.xml"
property "sonar.exclusions", [
'**/com/worldremit/usermanagement/dto/**',
'**/com/worldremit/usermanagement/domain/**',
'**/com/worldremit/usermanagement/configuration/**'
]
}
}
dependencies {
//AnnotationProcessors
annotationProcessor "org.springframework.boot:spring-boot-configuration-processor"
annotationProcessor "org.projectlombok:lombok:$lombokVersion"
serviceTestAnnotationProcessor "org.projectlombok:lombok:$lombokVersion"
annotationProcessor "org.mapstruct:mapstruct-processor:$mapStructVersion"
//Web
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.springframework.boot:spring-boot-starter-validation'
// Kafka cloud stream
implementation "org.springframework.cloud:spring-cloud-stream-binder-kafka"
// Avro
implementation "org.apache.avro:avro:${avroVersion}"
implementation "io.confluent:kafka-avro-serializer:${confluentVersion}"
implementation "com.worldremit:many-types:${manyTypesVersion}"
//Jpa
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
runtimeOnly 'org.postgresql:postgresql'
serviceTestRuntimeOnly "com.h2database:h2"
//Lombok
implementation "org.projectlombok:lombok:$lombokVersion"
serviceTestImplementation "org.projectlombok:lombok:$lombokVersion"
//MapStruct
implementation "org.mapstruct:mapstruct:$mapStructVersion"
//Liquibase
implementation 'org.liquibase:liquibase-core'
//Swagger
implementation "io.springfox:springfox-boot-starter:$swaggerVersion"
// Metrics
implementation "io.micrometer:micrometer-core:latest.release"
implementation "io.micrometer:micrometer-registry-prometheus:latest.release"
//Logs
implementation "net.logstash.logback:logstash-logback-encoder:$logbackVersion"
implementation "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:$jacksonVersion"
contractTestImplementation "com.fasterxml.jackson.module:jackson-module-scala_2.13:$jacksonVersion"
//Tests
testImplementation("org.springframework.boot:spring-boot-starter-test") {
exclude group: "org.junit.vintage", module: "junit-vintage-engine"
}
testImplementation "org.junit.jupiter:junit-jupiter-api"
testImplementation "org.junit.jupiter:junit-jupiter-params"
testImplementation "org.junit.jupiter:junit-jupiter-engine"
testImplementation 'org.springframework.kafka:spring-kafka-test'
testImplementation 'com.worldremit:common-test-utils:0.0.0'
contractTestImplementation "au.com.dius.pact.provider:junit5:$pactProviderVersion"
contractTestImplementation "au.com.dius.pact.provider:junit5spring:$pactProviderVersion"
testImplementation "org.apache.httpcomponents:httpclient:$testApacheHttpClientVersion"
}
Please pay particular attention to the 'apply from' section, where 'magic' appears. Looks better? Of course, it does! :) Now the functionality is well-encapsulated, well-named and easier to understand.
The future
The possibilities of this mechanism are more large-scale. We can imagine creating central repositories with well-defined scripts ready to apply, just like any other plugin in the Gradle world.
In the next step, we can also provide a script/plugin that will verify the correctness of used dependencies or plugins (Gradle is just a groovy script, we can do anything ;))
Finally, I encourage you and your team to refactor your build script. This will be the first step to simplifying your work with Gradle and reaping benefits of almost 'Maven parent poms'.
){kind=link}
