From a6aa295399145ba32576fc46faf3e36eef540d4c Mon Sep 17 00:00:00 2001 From: Jim Schubert <james.schubert@gmail.com> Date: Sat, 4 May 2019 16:49:41 -0400 Subject: [PATCH 1/7] [travis] Avoiding build timeouts Travis CI will timeout if there is no activity printed to stdout or stderr for 10 minutes. Running our mvn commands with --quiet option may hit this limit for any number of reasons (more code, network conditions). To avoid this, we remove --quiet and adjust some other Maven configuration options. One, specifically, is the cached .m2/repository directory. We bind this to a non-user home location due to the -u binding found in run-in-docker.sh. We then provide a settings.xml and CLI option to tell maven where to look for our cached repository directory. This should allow both users and CI to run the script in a consistent way, but most importantly should allow Travis to continue to cache artifacts across builds. --- .travis.yml | 5 +++-- CI/run-in-docker-settings.xml | 7 +++++++ run-in-docker.sh | 7 ++++++- 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 CI/run-in-docker-settings.xml diff --git a/.travis.yml b/.travis.yml index 4ea6e747895..d5fa3631549 100644 --- a/.travis.yml +++ b/.travis.yml @@ -121,8 +121,9 @@ script: # fail if generators contain tab '\t' - /bin/bash ./bin/utils/detect_tab_in_java_class.sh # run integration tests defined in maven pom.xml - - ./run-in-docker.sh mvn --quiet --batch-mode clean install - - mvn --quiet --batch-mode verify -Psamples + # Travis will timeout after 10 minutes of no stdout/stderr activity…avoid using --quiet options for mvn commands below. + - ./run-in-docker.sh mvn --batch-mode --fail-at-end --settings=/gen/CI/run-in-docker-settings.xml --threads=2.0C clean install + - mvn --batch-mode verify -Psamples after_success: # push to maven repo - if [ $SONATYPE_USERNAME ] && [ -z $TRAVIS_TAG ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then diff --git a/CI/run-in-docker-settings.xml b/CI/run-in-docker-settings.xml new file mode 100644 index 00000000000..97370a231af --- /dev/null +++ b/CI/run-in-docker-settings.xml @@ -0,0 +1,7 @@ +<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 + https://maven.apache.org/xsd/settings-1.0.0.xsd"> + <!-- This directory matches what is defined in run-in-docker.sh --> + <localRepository>/var/maven/.m2/repository</localRepository> +</settings> \ No newline at end of file diff --git a/run-in-docker.sh b/run-in-docker.sh index 80a0f2b8866..1866f9fa98d 100755 --- a/run-in-docker.sh +++ b/run-in-docker.sh @@ -7,12 +7,17 @@ maven_cache_repo="${HOME}/.m2/repository" mkdir -p "${maven_cache_repo}" +# !! The -u option below needs to be defined so we don't write to a user's bound ~/.m2/repository as root. +# !! but using this also means we either need to setup a user with the same id, or we execute without a username and home directory. +# !! This means we can't bind the .m2 directory to any user's directory (like /root/.m2). +# !! We _must_ define $MAVEN_CONFIG explicitly as a location that is not /root/.m2; the user executing this may not have access to the container's user's directory. docker run --rm -it \ -w /gen \ -e GEN_DIR=/gen \ -e MAVEN_CONFIG=/var/maven/.m2 \ + -e MAVEN_OPTS="-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=/var/maven/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" \ -u "$(id -u):$(id -g)" \ -v "${PWD}:/gen" \ - -v "$HOME/.m2":/root/.m2 \ + -v "${maven_cache_repo}:/var/maven/.m2/repository" \ --entrypoint /gen/docker-entrypoint.sh \ maven:3-jdk-8 "$@" -- GitLab From 0da32c505286ecbc9ee4f584049ed203eebbc363 Mon Sep 17 00:00:00 2001 From: Jim Schubert <james.schubert@gmail.com> Date: Sun, 5 May 2019 08:52:22 -0400 Subject: [PATCH 2/7] Run mvn in Travis directly, rather than in Docker --- .travis.yml | 12 +++++++++--- run-in-docker.sh | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d5fa3631549..0461454f8c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,10 @@ language: java jdk: - openjdk8 +# See https://docs.travis-ci.com/user/languages/java/#caching +before_cache: + - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock + - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ cache: directories: - $HOME/.m2 @@ -121,9 +125,8 @@ script: # fail if generators contain tab '\t' - /bin/bash ./bin/utils/detect_tab_in_java_class.sh # run integration tests defined in maven pom.xml - # Travis will timeout after 10 minutes of no stdout/stderr activity…avoid using --quiet options for mvn commands below. - - ./run-in-docker.sh mvn --batch-mode --fail-at-end --settings=/gen/CI/run-in-docker-settings.xml --threads=2.0C clean install - - mvn --batch-mode verify -Psamples + - mvn ${MAVEN_CLI_OPTS} clean install + - mvn ${MAVEN_CLI_OPTS} verify -Psamples after_success: # push to maven repo - if [ $SONATYPE_USERNAME ] && [ -z $TRAVIS_TAG ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then @@ -163,3 +166,6 @@ after_success: env: - DOCKER_GENERATOR_IMAGE_NAME=openapitools/openapi-generator-online DOCKER_CODEGEN_CLI_IMAGE_NAME=openapitools/openapi-generator-cli NODE_ENV=test CC=gcc-5 CXX=g++-5 + - MAVEN_OPTS="-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" + # Travis will timeout after 10 minutes of no stdout/stderr activity…avoid using --quiet options for mvn cli options below. + - MAVEN_CLI_OPTS="--batch-mode --errors --fail-at-end --show-version --threads=2.0C" diff --git a/run-in-docker.sh b/run-in-docker.sh index 1866f9fa98d..c59a0fda72f 100755 --- a/run-in-docker.sh +++ b/run-in-docker.sh @@ -18,6 +18,7 @@ docker run --rm -it \ -e MAVEN_OPTS="-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=/var/maven/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" \ -u "$(id -u):$(id -g)" \ -v "${PWD}:/gen" \ + -v "${PWD}/CI/run-in-docker-settings.xml:/var/maven/.m2/settings.xml" \ -v "${maven_cache_repo}:/var/maven/.m2/repository" \ --entrypoint /gen/docker-entrypoint.sh \ maven:3-jdk-8 "$@" -- GitLab From 35044e4c4e7a5776522f8edac405818f3de8d09c Mon Sep 17 00:00:00 2001 From: Jim Schubert <james.schubert@gmail.com> Date: Sun, 5 May 2019 10:27:07 -0400 Subject: [PATCH 3/7] Limit mvn verify to only quiet/batch-mode --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0461454f8c1..ed999c76dc6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,7 +126,7 @@ script: - /bin/bash ./bin/utils/detect_tab_in_java_class.sh # run integration tests defined in maven pom.xml - mvn ${MAVEN_CLI_OPTS} clean install - - mvn ${MAVEN_CLI_OPTS} verify -Psamples + - mvn --quiet --batch-mode verify -Psamples after_success: # push to maven repo - if [ $SONATYPE_USERNAME ] && [ -z $TRAVIS_TAG ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then -- GitLab From 3b6ccee062825a29a96e1ffc24812d6bed94aa63 Mon Sep 17 00:00:00 2001 From: Jim Schubert <james.schubert@gmail.com> Date: Sun, 5 May 2019 10:28:52 -0400 Subject: [PATCH 4/7] Inline the MAVEN_CLI_OPTS variable --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed999c76dc6..8daf1d73822 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,7 +125,8 @@ script: # fail if generators contain tab '\t' - /bin/bash ./bin/utils/detect_tab_in_java_class.sh # run integration tests defined in maven pom.xml - - mvn ${MAVEN_CLI_OPTS} clean install + # Travis will timeout after 10 minutes of no stdout/stderr activity…avoid using --quiet options for mvn cli options below. + - mvn --batch-mode --errors --fail-at-end --show-version --threads=2.0C clean install - mvn --quiet --batch-mode verify -Psamples after_success: # push to maven repo @@ -167,5 +168,3 @@ after_success: env: - DOCKER_GENERATOR_IMAGE_NAME=openapitools/openapi-generator-online DOCKER_CODEGEN_CLI_IMAGE_NAME=openapitools/openapi-generator-cli NODE_ENV=test CC=gcc-5 CXX=g++-5 - MAVEN_OPTS="-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" - # Travis will timeout after 10 minutes of no stdout/stderr activity…avoid using --quiet options for mvn cli options below. - - MAVEN_CLI_OPTS="--batch-mode --errors --fail-at-end --show-version --threads=2.0C" -- GitLab From 2d1ef07dbd58b01e5acd03de24ffaf5a1f653839 Mon Sep 17 00:00:00 2001 From: Jim Schubert <james.schubert@gmail.com> Date: Sun, 5 May 2019 11:52:37 -0400 Subject: [PATCH 5/7] Apply 20 min travis_wait to mvn verify -Psamples --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8daf1d73822..622ca24a6c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -127,7 +127,7 @@ script: # run integration tests defined in maven pom.xml # Travis will timeout after 10 minutes of no stdout/stderr activity…avoid using --quiet options for mvn cli options below. - mvn --batch-mode --errors --fail-at-end --show-version --threads=2.0C clean install - - mvn --quiet --batch-mode verify -Psamples + - travis_wait 20 mvn --quiet --batch-mode verify -Psamples after_success: # push to maven repo - if [ $SONATYPE_USERNAME ] && [ -z $TRAVIS_TAG ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then -- GitLab From 76d540d33aac21adaec2890f59599e3346b67837 Mon Sep 17 00:00:00 2001 From: Jim Schubert <james.schubert@gmail.com> Date: Sun, 5 May 2019 12:55:54 -0400 Subject: [PATCH 6/7] Remove multi-threading (org.fortasoft:gradle-maven-plugin is not thread-safe), and use travis_wait 20 on mvn clean install --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 622ca24a6c9..7e3efc0adad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,9 +125,10 @@ script: # fail if generators contain tab '\t' - /bin/bash ./bin/utils/detect_tab_in_java_class.sh # run integration tests defined in maven pom.xml - # Travis will timeout after 10 minutes of no stdout/stderr activity…avoid using --quiet options for mvn cli options below. - - mvn --batch-mode --errors --fail-at-end --show-version --threads=2.0C clean install - - travis_wait 20 mvn --quiet --batch-mode verify -Psamples + # Travis will timeout after 10 minutes of no stdout/stderr activity, which is problematic with mvn --quiet. + # Also, travis_wait can't bind a wait to child processes (mvn exec plugin and others), so we wrap in a bash shell command. + - travis_wait 20 bash -c 'mvn --quiet --batch-mode --show-version clean install' + - travis_wait 20 bash -c 'mvn --quiet --batch-mode --show-version verify -Psamples' after_success: # push to maven repo - if [ $SONATYPE_USERNAME ] && [ -z $TRAVIS_TAG ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then -- GitLab From 8ec7b6ad35bc2393c7874a696555e4641bb7f66b Mon Sep 17 00:00:00 2001 From: Jim Schubert <james.schubert@gmail.com> Date: Sun, 5 May 2019 14:01:23 -0400 Subject: [PATCH 7/7] Use mvn commands from current master (PR is now only run-in-docker.sh fixes) --- .travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e3efc0adad..32cc18325a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,10 +125,9 @@ script: # fail if generators contain tab '\t' - /bin/bash ./bin/utils/detect_tab_in_java_class.sh # run integration tests defined in maven pom.xml - # Travis will timeout after 10 minutes of no stdout/stderr activity, which is problematic with mvn --quiet. - # Also, travis_wait can't bind a wait to child processes (mvn exec plugin and others), so we wrap in a bash shell command. - - travis_wait 20 bash -c 'mvn --quiet --batch-mode --show-version clean install' - - travis_wait 20 bash -c 'mvn --quiet --batch-mode --show-version verify -Psamples' + # WARN: Travis will timeout after 10 minutes of no stdout/stderr activity, which is problematic with mvn --quiet. + - mvn --quiet --batch-mode --show-version clean install + - mvn --quiet --batch-mode --show-version verify -Psamples after_success: # push to maven repo - if [ $SONATYPE_USERNAME ] && [ -z $TRAVIS_TAG ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then @@ -168,4 +167,4 @@ after_success: env: - DOCKER_GENERATOR_IMAGE_NAME=openapitools/openapi-generator-online DOCKER_CODEGEN_CLI_IMAGE_NAME=openapitools/openapi-generator-cli NODE_ENV=test CC=gcc-5 CXX=g++-5 - - MAVEN_OPTS="-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" + -- GitLab