Getting logs from a docker container inside jenkins

Question

I am trying to get docker to setup a complete test environment that I can run integration tests in, but so far it is not really working and I am even having issues with getting logs back from it.

So I want to run it as a pipeline and I want to use jenkinsfile. This is what I got so far:

pipeline {
    agent any
    stages {
        stage('build war') {
            agent {
                docker { 
                    image 'gradle:latest'
                    reuseNode true 
                }
            }
            steps {
                sh 'gradle war -b oven/build.gradle'
            }
        }
        stage('test') {
            steps {
                script {
                    docker.image('mysql:latest').withRun('-e "MYSQL_ROOT_PASSWORD=password" -e "MYSQL_USER=root" -e "MYSQL_DATABASE=highlygroceries"') { c -> 
                        docker.image('munhunger/highly-oven').withRun('-e "test=test"') { h -> 
                            docker.image('mysql:latest').inside("--link ${c.id}:db") {
                                sh 'while ! mysqladmin ping -hdb --silent; do sleep 1; done'
                            }
                            docker.image('munhunger/highly-oven').inside("--link ${c.id}:db -e 'DB_URL=db:3306' -e 'DB_PASS=password' -e 'DB_USER=root'") {
                                sh 'sleep 5'
                            }
                            docker.image('gradle:latest').inside("--link ${h.id}:backend -e 'OVEN_URL=http://backend:8080'") {
                                sh 'gradle test -b oven/build.gradle'
                            }
                            sh "docker logs ${h.id}"
                        }
                    }
                }
            }
        }
        stage('build dockerimage') {
            steps {
                script {
                    dir('oven') {
                        def image = docker.build("munhunger/highly-oven")

                        docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
                            image.push("${env.BUILD_NUMBER}")
                            image.push("latest")
                        }
                    }
                }
            }
        }
    }
}

But there seems to be a problem with the connection between my backend and the database...

All I get from the build logs is this:

se.munhunger.oven.rest.UserTest > System is up and running, Creating a user, it returns 204 upon creation FAILED
    java.lang.AssertionError at UserTest.java:38

which points to:

                Assert.assertEquals("non 204 from backend", 204,
                                    client.target(baseURL + "/api/user")
                                        .request()
                                        .header("email", "[email protected]")
                                        .post(Entity.json(null))
                                        .getStatus());

I believe that the connection between the tester and the backend is working because the following test succeeds

            Assert.assertEquals(200,
                                client.target(baseURL + "/swagger")
                                    .request()
                                    .get()
                                    .getStatus());

Which leads me to I guess the main question of how do I get the logs from my backend docker image? I feel like without it, it is pretty much impossible to debug what is going wrong

Edit I've gotten some logs out, but at the wrong time. If I change to the following:

                docker.image('mysql:latest').withRun('-e "MYSQL_ROOT_PASSWORD=password" -e "MYSQL_USER=root" -e "MYSQL_DATABASE=highlygroceries"') { c -> 
                    docker.image('munhunger/highly-oven').withRun('-e "test=test"') { h -> 
                        docker.image('mysql:latest').inside("--link ${c.id}:db") {
                            sh 'while ! mysqladmin ping -hdb --silent; do sleep 1; done'
                        }
                        docker.image('munhunger/highly-oven').inside("--link ${c.id}:db -e 'DB_URL=db:3306' -e 'DB_PASS=password' -e 'DB_USER=root'") {
                            sh 'sleep 5'
                        }
                        sh "docker logs ${h.id}"
                        docker.image('gradle:latest').inside("--link ${h.id}:backend -e 'OVEN_URL=http://backend:8080'") {
                            sh 'gradle test -b oven/build.gradle'
                        }
                        sh "docker logs ${h.id}"
                    }
                }

I get all the logs from startup. but it is not printing out the logs from after the test failure

Solution 1

It helps when you are able to run the integration tests without using your Jenkins file, so instead of using these nested docker.image statements, you should use docker-compose.

I to integration testing like this:

stage('Run integration tests') {
  steps {
    script {
      try {
        timeout(30) {
          // Tear up integration test environment
          sh "docker-compose up -d"
          // Wait until it is ready
          waitUntil {
            "healthy" == sh(returnStdout: true,
              script: "docker inspect CONTAINER_NAME --format=\"{{ .State.Health.Status }}\"").trim()
          }

          docker.image('IMAGENAME').inside('--network projectname_default') {
            sh "gradle integrationTest"
          }
        }
      } finally {
        try {
          step([$class: 'JUnitResultArchiver', testResults: '**/build/integrationTest-results/TEST-*.xml'])
        } catch (Exception e) {
          // Ignore exception when there are no test results
        }
        sh "docker-compose logs >integration-test.log"
        sh "docker-compose down --rmi local --volumes --remove-orphans"
        archive 'integration-test.log'
      }
    }
  }
}

As you can see, I attach the Gradle testing container to the network that is set up by docker-compose. This allows you re-using your compose not only for testing.

Now you have to make sure that all containers that you use log to stdout. At the end, you get all your logs in archive integration-test.log. Of course, you can also extend this to get a separate log file for each container.

Solution 2

Turns out you can straight up catch that shit:

pipeline {
    agent any
    stages {
        stage('build war') {
            agent {
                docker { 
                    image 'gradle:latest'
                    reuseNode true 
                }
            }
            steps {
                sh 'gradle war -b oven/build.gradle'
            }
        }
        stage('test') {
            steps {
                script {
                    docker.image('mysql:latest').withRun('-e "MYSQL_ROOT_PASSWORD=password" -e "MYSQL_USER=root" -e "MYSQL_DATABASE=highlygroceries"') { c -> 
                        docker.image('munhunger/highly-oven').withRun('-e "test=test"') { h -> 
                            docker.image('mysql:latest').inside("--link ${c.id}:db") {
                                sh 'while ! mysqladmin ping -hdb --silent; do sleep 1; done'
                            }
                            docker.image('munhunger/highly-oven').inside("--link ${c.id}:db -e 'DB_URL=db:3306' -e 'DB_PASS=password' -e 'DB_USER=root'") {
                                sh 'sleep 5'
                            }
                            try {
                                docker.image('gradle:latest').inside("--link ${h.id}:backend -e 'OVEN_URL=http://backend:8080'") {
                                        sh 'gradle test -b oven/build.gradle'

                                }
                            }
                            catch (exc) {
                                sh "docker logs ${h.id}"
                                throw exc
                            }
                        }
                    }
                }
            }
        }
        stage('build dockerimage') {
            steps {
                script {
                    dir('oven') {
                        def image = docker.build("munhunger/highly-oven")

                        docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
                            image.push("${env.BUILD_NUMBER}")
                            image.push("latest")
                        }
                    }
                }
            }
        }
    }
}