Fix Kokoro release jobs (#4225)

* Fix Kokoro release jobs. Needs loopback pinentry-mode

* Skip install on deploy

* skip stagingProfileId

* Capture built artifacts

* Enable promote/drop jobs via environment variable

* Fix directory path and use -B for drop/release

* promote/drop need the settings file

* Fix path to settings file

* Update RELEASING documentation to use the Kokoro jobs

Author: chingor13

.kokoro/release/drop.sh

@@ -15,11 +15,20 @@
 
 set -eo pipefail
 
+# STAGING_REPOSITORY_ID must be set
+if [ -z "${STAGING_REPOSITORY_ID}" ]; then
+  echo "Missing STAGING_REPOSITORY_ID environment variable"
+  exit 1
+fi
+
 source $(dirname "$0")/common.sh
-MAVEN_SETTINGS_FILE=$(realpath $(dirname "$0")/../)/settings.xml
-pushd $(dirname "$0")/../
+MAVEN_SETTINGS_FILE=$(realpath $(dirname "$0")/../../)/settings.xml
+pushd $(dirname "$0")/../../
 
 setup_environment_secrets
 create_settings_xml_file "settings.xml"
 
-mvn nexus-staging:drop -P release
+mvn nexus-staging:drop -B \
+  -P release \
+  --settings ${MAVEN_SETTINGS_FILE} \
+  -DstagingRepositoryId=${STAGING_REPOSITORY_ID}

.kokoro/release/promote.sh

@@ -15,11 +15,20 @@
 
 set -eo pipefail
 
+# STAGING_REPOSITORY_ID must be set
+if [ -z "${STAGING_REPOSITORY_ID}" ]; then
+  echo "Missing STAGING_REPOSITORY_ID environment variable"
+  exit 1
+fi
+
 source $(dirname "$0")/common.sh
-MAVEN_SETTINGS_FILE=$(realpath $(dirname "$0")/../)/settings.xml
-pushd $(dirname "$0")/../
+MAVEN_SETTINGS_FILE=$(realpath $(dirname "$0")/../../)/settings.xml
+pushd $(dirname "$0")/../../
 
 setup_environment_secrets
 create_settings_xml_file "settings.xml"
 
-mvn nexus-staging:release -P release
+mvn nexus-staging:release -B \
+  -P release \
+  --settings ${MAVEN_SETTINGS_FILE} \
+  -DstagingRepositoryId=${STAGING_REPOSITORY_ID}

.kokoro/release/stage.cfg

@@ -6,7 +6,11 @@
  # Need to save the properties file
 action {
   define_artifacts {
-    regex: "github/google-cloud-java/target/nexus-staging/staging/*.properties"
-    strip_prefix: "github/google-cloud-java"
+    regex: "**/target/nexus-staging/staging/*.properties"
+    regex: "**/target/*.jar"
+    regex: "**/target/*.jar.asc"
+    regex: "**/target/*.pom"
+    regex: "**/target/*.pom.asc"
+    fail_if_no_artifacts: true
   }
 }

.kokoro/release/stage.sh

@@ -22,10 +22,9 @@ pushd $(dirname "$0")/../../
 setup_environment_secrets
 create_settings_xml_file "settings.xml"
 
-mvn clean install deploy \
+mvn clean deploy -B \
   -DskipTests=true \
   --settings ${MAVEN_SETTINGS_FILE} \
-  -DstagingProfileId=3187e4f20d328b \
   -Dgpg.executable=gpg \
   -Dgpg.passphrase=${GPG_PASSPHRASE} \
   -Dgpg.homedir=${GPG_HOMEDIR} \

RELEASING.md

@@ -1,20 +1,97 @@
-One-time setup
-==============
+## Setup
+
+### Install releasetool
+
+See [releasetool](https://github.com/googleapis/releasetool) for installation instructions. You will
+need python 3.6+ to run this tool.
+
+## Prepare a release
+
+1. Make sure the team agrees that it is time to release.
+
+2. Look over all of the commits since the last release and make sure there are no breaking changes
+   on the public surface. If there are any breaking changes, create and merge a new PR to revert the
+   surface back.
+
+  **Note:** this should just be a scan of the public surface that would appear in Java doc.
+  Implementation changes, README changes, and snippet changes can all be skipped for this check.
+
+3. Verify that all unit and integration tests for the last commit have passed.
+
+4. Run `releasetool start`. Select "minor" or "patch" for the release type. This will bump the
+   artifact versions, ask you to edit release notes, and create the release pull request.
+
+  **Note:** be sure to make these notes nice as they will be used for the release notes as well.
+
+## Pushing a release to Maven using Kokoro
+
+To manually publish the artifacts to Maven (rather than using Kokoro), you will need to follow the
+"Additional setup for manual publishing" steps below and follow the instructions in the
+"Manually publishing" section.
+
+1. Trigger the `google-cloud-java/release/stage` Kokoro job and wait for it to complete. This will
+   stage the built artifacts and prepare them for publishing.
+
+2. Look through the logs for the `google-cloud-java/release/stage` and find the staging repository
+   ids used. They will look like `comgoogleapi-XYZ` and `comgooglecloud-XYZ`.
+
+3. Promote or drop the staged repository.
+
+  a. To publish the staged repository, trigger the `google-cloud-java/release/promote` Kokoro job for
+     each staging repository. To specify the staging repository, add an environment variable
+     configuration with `STAGING_REPOSITORY_ID=<staging repository id>` from the UI. **Note: thi 
+     will need to be run for each staging repository. It may take a few hours for the released
+     versions to be available for users to download.**
+
+  b. To drop (abort) the staged repository, trigger the `google-cloud-java/release/drop` Kokoro job
+     with the same staging repository id configuration as if you were publishing.
+
+## Publish Javadoc
+
+1. Run `git clean -x -f -d` to put the repo in a clean state.
+
+2. Locally build the repo by running `mvn install -DskipTests`.
+
+3. Run `python utilities/stage_sites.py`. This script checks out `gh-pages` branch of the
+   repository, builds the documentation site and javadocs, copies them to the branch and commits it.
+   This script does not push the docs and it must be done manually on the later step. The script
+   assumes that there is no directory called `tmp_gh-pages` in the repository root. If it is
+   present, remove it before running the script.
+
+4. Run `cd tmp_gh-pages && git push && cd ..`.
+
+5. (Optional) Run `rm -rf tmp_gh-pages` to remove the generated docs directory from your local
+   machine.
+
+## Tag the release
+
+1. Run `releasetool tag` to publish a release on Github. It will list the last few merged PRs.
+   Select the newly merged release PR. Releasetool will create the GitHub release with notes
+   extracted from the pull request and tag the new release.
+
+## Prepare the next snapshot version
+
+1. Run `releasetool start` to bump the next snapshot version. Select "snapshot" when prompted for
+   the release type. This will bump the artifact versions and create a pull request.
+
+2. Review and submit the PR.
+
+## Additional setup for manual publishing
+
+### Set up Sonatype Account
 
-Set up Sonatype Account
------------------------
 * Sign up for a Sonatype JIRA account [here](https://issues.sonatype.org)
 * Click *Sign Up* in the login box, follow instructions
 
-Get access to repository
-------------------------
+### Get access to repository
+
 * Go to [community support](https://issues.sonatype.org/browse/OSSRH)
 * Ask for publish rights by creating an issue similar to [this one](https://issues.sonatype.org/browse/OSSRH-32032)
   * You must be logged in to create a new issue
   * Use the *Create* button at the top tab
 
-Set up PGP keys
----------------
+### Set up PGP keys
+
 * Install GNU Privacy Guard (GPG)
   * GPG is installed by default on Ubuntu systems
   * For other systems, see [GnuPG download page](https://www.gnupg.org/download/)
@@ -30,8 +107,8 @@ Set up PGP keys
 
 * Upload your public key to a public server: ```gpg --send-keys --keyserver hkp://pgp.mit.edu ABCDEFGH```
 
-Create a Maven settings file
-----------------------------
+### Create a Maven settings file
+
 * Create a file at ```$HOME/.m2/settings.xml``` with your passphrase and your sonatype username and password
 ```
 <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
@@ -72,42 +149,13 @@ Create a Maven settings file
 
 Make sure you are using Maven version 3.3 or higher to support the Nexus plugin required to stage a release.
 
-Install releasetool
--------------------
-See [releasetool](https://github.com/googleapis/releasetool) for installation instructions. You will
-need python 3.6+ to run this tool.
-
-To push a release version
-=========================
+### Manually publishing (from a local machine)
 
-1. Make sure the team agrees that it is time to release. 
+1. Run `git clean -x -f -d` to put the repo in a clean state.
 
-2. Look over all of the commits since the last release and make sure there are no breaking changes
-   on the public surface. If there are any breaking changes, create and merge a new PR to revert the
-   surface back.
-
-  **Note:** this should just be a scan of the public surface that would appear in Java doc.
-  Implementation changes, README changes, and snippet changes can all be skipped for this check.
+2. Locally build the repo by running `mvn install -DskipTests`.
 
-3. Verify that all unit and integration tests for the last commit have passed.
-
-4. Run `releasetool start`. Select "minor" or "patch" for the release type. This will bump the
-   artifact versions, ask you to edit release notes, and create the release pull request.
-
-  **Note:** be sure to make these notes nice as they will be used for the release notes as well.
-
-5. Run `git clean -x -f -d` to put the repo in a clean state.
-
-6. Locally build the repo by running `mvn install -DskipTests`.
-
-7. Run `python utilities/stage_sites.py`. This script checks out `gh-pages` branch of the
-   repository, builds the documentation site and javadocs, copies them to the branch and commits it.
-   This script does not push the docs and it must be done manually on the later step. The script
-   assumes that there is no directory called `tmp_gh-pages` in the repository root. If it is
-   present, remove it before running the script.
-
-8. Check that you are not trying to release a SNAPSHOT build (the artifacts versions do not have
-   "-SNAPSHOT" suffix) and then run `mvn clean deploy -DskipTests=true --settings ~/.m2/settings.xml -P release`
+3.  and then run `mvn clean deploy -DskipTests=true --settings ~/.m2/settings.xml -P release`
    command. It will build and deploy artifacts to the staging repository.
 
   **Note:** you may need to specify the stagingProfileId with `-DstagingProfileId=3187e4f20d328b`
@@ -116,20 +164,11 @@ To push a release version
 9. Run `mvn nexus-staging:release` to release the artifacts. If you wish to abort the release, run
    `mvn nexus-staging:drop`.
 
-10. Run `cd tmp_gh-pages && git push && cd ..` to push the previously generated docs (step 7).
-
-11. Run `rm -rf tmp_gh-pages` to remove the generated docs directory from your local machine.
-
-12. Run `releasetool tag` to publish a release on Github. It will list the last few merged PRs.
-    Select the newly merged release PR. Releasetool will create the GitHub release with notes
-    extracted from the pull request and tag the new release.
-
-13. Run `releasetool start` to bump the next snapshot version. Select "snapshot" when prompted for
-    the release type. This will bump the artifact versions and create a pull request.
-
-14. Review and submit the PR.
+## Improvements
 
-Improvements
-============
+* Automatic tagging is not currently implemented, though it was discussed in #119. If the version 
+  updates continue to be manual, a one-line git tag command can be added to `after_success.sh` to 
+  correctly tag releases.  However, automatically creating useful annotations for this tag will be 
+  difficult.  Also, if the release process becomes fully automated, tagging becomes a harder 
+  problem, as mentioned in that issue.
 
-Automatic tagging is not currently implemented, though it was discussed in [#119](https://github.com/googleapis/google-cloud-java/pull/119).  If the version updates continue to be manual, a one-line git tag command can be added to `after_success.sh` to correctly tag releases.  However, automatically creating useful annotations for this tag will be difficult.  Also, if the release process becomes fully automated, tagging becomes a harder problem, as mentioned in that issue.

google-api-grpc/pom.xml

@@ -960,6 +960,12 @@
                 <goals>
                   <goal>sign</goal>
                 </goals>
+                <configuration>
+                  <gpgArguments>
+                    <arg>--pinentry-mode</arg>
+                    <arg>loopback</arg>
+                  </gpgArguments>
+                </configuration>
               </execution>
             </executions>
           </plugin>

google-cloud-bom/pom.xml

@@ -1034,6 +1034,12 @@
                 <goals>
                   <goal>sign</goal>
                 </goals>
+                <configuration>
+                  <gpgArguments>
+                    <arg>--pinentry-mode</arg>
+                    <arg>loopback</arg>
+                  </gpgArguments>
+                </configuration>
               </execution>
             </executions>
           </plugin>

google-cloud-clients/pom.xml

@@ -942,6 +942,12 @@
                 <goals>
                   <goal>sign</goal>
                 </goals>
+                <configuration>
+                  <gpgArguments>
+                    <arg>--pinentry-mode</arg>
+                    <arg>loopback</arg>
+                  </gpgArguments>
+                </configuration>
               </execution>
             </executions>
           </plugin>

google-cloud-testing/google-cloud-bigtable-emulator/pom.xml

@@ -225,6 +225,12 @@
                 <goals>
                   <goal>sign</goal>
                 </goals>
+                <configuration>
+                  <gpgArguments>
+                    <arg>--pinentry-mode</arg>
+                    <arg>loopback</arg>
+                  </gpgArguments>
+                </configuration>
               </execution>
             </executions>
           </plugin>