Run the integration suite in CI again

Makefile

@@ -19,6 +19,14 @@ LDFLAGS := -X main.Version="$(VERSION)" -X main.BuildTime="$(BUILD_TIME)" -X mai
 BINARY_NAME := safe
 GO_FILES := $(shell find . -name '*.go' -type f -not -path "./vendor/*")
 
+# Integration suite. TEST_PATH is the suite script, SAFE_PATH the binary it
+# drives, and VAULT_VERSIONS the space-separated Vault versions to run
+# against. CI overrides all three; the defaults run the in-tree suite
+# against a freshly built binary over the suite's own default versions.
+TEST_PATH ?= ci/scripts/tests
+SAFE_PATH ?= ./$(BINARY_NAME)
+VAULT_VERSIONS ?=
+
 ##@ General
 
 .PHONY: help
@@ -77,6 +85,21 @@ test: ## Run all tests with race detector
 	@go test -race -v $(shell go list ./... | grep -v vendor)
 	@echo "$(GREEN)✓ Tests complete$(RESET)"
 
+.PHONY: test-integration
+test-integration: ## Run the end-to-end suite against real Vault servers
+	@echo "$(GREEN)Running integration suite...$(RESET)"
+	@test -x "$(TEST_PATH)" || { echo "$(YELLOW)No suite at $(TEST_PATH)$(RESET)"; exit 2; }
+# Rebuild when driving the in-tree binary so the suite never runs against a
+# stale build. An overridden SAFE_PATH is taken as-is: CI supplies the
+# release artifact it means to test.
+ifeq ($(SAFE_PATH),./$(BINARY_NAME))
+	@$(MAKE) --no-print-directory build
+else
+	@test -x "$(SAFE_PATH)" || { echo "$(YELLOW)No safe binary at $(SAFE_PATH)$(RESET)"; exit 2; }
+endif
+	@$(TEST_PATH) $(SAFE_PATH) $(VAULT_VERSIONS)
+	@echo "$(GREEN)✓ Integration suite complete$(RESET)"
+
 .PHONY: test-short
 test-short: ## Run tests in short mode (no race detector)
 	@echo "$(GREEN)Running short tests...$(RESET)"

ci/scripts/test

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Gate for the pull-request job. No Vault server is available here, so this
+# runs the hermetic checks only: fmt, vet, staticcheck, and the unit tests.
+# The end-to-end suite runs in the test-vault-* jobs, which have a server.
+
+set -eu
+
+export GOPATH="${PWD}/gopath"
+export PATH="${PATH}:${GOPATH}/bin"
+
+test -n "${MODULE:-}" || {
+  echo >&2 "MODULE must be set to the Go module path.  Did you misconfigure Concourse?"
+  exit 2
+}
+
+cd "${GOPATH}/src/${MODULE}"
+
+go version
+echo
+
+make check

ci/scripts/test-release-against-vault

@@ -21,7 +21,7 @@ bail() {
 	exit 2
 }
 test -n "${PROJECT:-}"        || bail "PROJECT must be set to the name of this package."
-test -n "${VAULT_VERSION:-}"  || bail "VAULT_VERSION must be set to the version of HashiCorp Vault to test against."
+test -n "${VAULT_VERSIONS:-}" || bail "VAULT_VERSIONS must be set to the version(s) of HashiCorp Vault to test against."
 
 test -f "${VERSION_FROM}"     || bail "Version file (${VERSION_FROM}) not found."
 VERSION=$(cat "${VERSION_FROM}")
@@ -33,10 +33,10 @@ ARCH=$(uname -m | sed -e 's/^x86_/amd/')
 [[ -e "$BUILD_ROOT/$PROJECT-$VERSION-$OS-$ARCH" ]] || \
   bail "Cannot find safe executable for v$VERSION on $OS/$ARCH"
 
-header "Testing $PROJECT v$VERSION ($OS/$ARCH) against Vault $VAULT_VERSION"
+header "Testing $PROJECT v$VERSION ($OS/$ARCH) against Vault $VAULT_VERSIONS"
 
 cd "$REPO_ROOT"
-make test \
+make test-integration \
   TEST_PATH="../$CI_ROOT/ci/scripts/tests" \
   SAFE_PATH="../$BUILD_ROOT/$PROJECT-$VERSION-$OS-$ARCH" \
   VAULT_VERSIONS="$VAULT_VERSIONS"