CI: Use precompiled Director release

ci/docker/openstack-cpi-release/Dockerfile

@@ -1,56 +1,29 @@
-FROM ubuntu:jammy
-
-ENV bosh_cli_version 7.7.2
-ENV DEBIAN_FRONTEND noninteractive
-
-RUN apt-get -y update && apt-get install -y locales && locale-gen en_US.UTF-8
-RUN update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
-
-RUN apt-get install -y \
-    sudo \
-    apt-utils \
-    curl wget tar make uuid-runtime \
-    sqlite3 libsqlite3-dev \
-    mysql-client libmysqlclient-dev \
-    postgresql postgresql-client libpq-dev \
-    build-essential checkinstall \
-    libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
-    whois \
-    netcat \
-    python3 \
-    python3-pip \
-    libyaml-dev jq && \
-    apt-get clean
-
-RUN pip3 install pytz && pip3 install pyrsistent python-openstackclient && \
-    pip3 install awscli
-
-# install newest git CLI
-RUN apt-get install software-properties-common -y; \
-    add-apt-repository ppa:git-core/ppa -y; \
-    apt-get update; \
-    apt-get install -y git git-lfs
-
-# ruby-install
-RUN mkdir /tmp/ruby-install && \
-    cd /tmp/ruby-install && \
-    curl https://codeload.github.com/postmodern/ruby-install/tar.gz/v0.8.3 | tar -xz && \
-    cd /tmp/ruby-install/ruby-install-0.8.3 && \
-    make install && \
-    rm -rf /tmp/ruby-install
-
-#Ruby
-RUN ruby-install --system ruby 3.1.0
-
-#BOSH CLI
-RUN curl -sL https://github.com/cloudfoundry/bosh-cli/releases/download/v${bosh_cli_version}/bosh-cli-${bosh_cli_version}-linux-amd64 \
+FROM ubuntu
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+ENV BOSH_CLI_VERSION=7.10.10
+
+RUN apt-get -y update \
+    && apt-get install -y --no-install-recommends \
+        locales \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+
+# ruby needed for BATS (bosh-acceptance-tests is an RSpec suite)
+RUN apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    git \
+    git-lfs \
+    jq \
+    openssh-client \
+    ruby \
+    ruby-bundler \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN curl -fsSL "https://github.com/cloudfoundry/bosh-cli/releases/download/v${BOSH_CLI_VERSION}/bosh-cli-${BOSH_CLI_VERSION}-linux-amd64" \
     > /usr/bin/bosh \
-    && chmod +x /usr/bin/bosh && \
-    cp /usr/bin/bosh /usr/local/bin/bosh-go
-
-#GitHub CLI
-RUN cd /tmp && \
-    curl -L https://github.com/github/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz | tar -xz && \
-    cp hub-linux-amd64-2.14.2/bin/hub /usr/local/bin && \
-    rm -rf /tmp/hub-linux-amd64-2.14.2 && \
-    rm -f /tmp/hub-linux-amd64-2.14.2.tgz
+    && chmod +x /usr/bin/bosh \
+    && cp /usr/bin/bosh /usr/local/bin/bosh-go

ci/ops_files/deployment-configuration.yml

@@ -1,30 +1,15 @@
 ---
-- type: replace
-  path: /releases/name=bosh
-  value:
-    name: bosh
-    url: file://bosh-release.tgz
-
 - type: replace
   path: /releases/name=bosh-openstack-cpi
   value:
     name: bosh-openstack-cpi
     url: file://bosh-openstack-cpi.tgz
 
 - type: replace
-  path: /resource_pools/name=vms
+  path: /resource_pools/name=vms/cloud_properties?
   value:
-    name: vms
-    network: default
-    stemcell:
-      url: file://stemcell.tgz
-    cloud_properties:
-      instance_type: ((openstack_flavor))
-      availability_zone: ((availability_zone))
-    env:
-      bosh:
-        mbus:
-          cert: ((mbus_bootstrap_ssl))
+    instance_type: ((openstack_flavor))
+    availability_zone: ((availability_zone))
 
 - type: replace
   path: /instance_groups/name=bosh/properties/hm/resurrector_enabled

ci/ops_files/use-golang-cpi.yml

@@ -0,0 +1,18 @@
+---
+# Deploy the director with the Go CPI job instead of the Ruby one (bosh-deployment/openstack/cpi.yml
+# wires openstack_cpi).
+- type: replace
+  path: /instance_groups/name=bosh/jobs/name=openstack_cpi
+  value:
+    name: openstack_cpi_golang
+    release: bosh-openstack-cpi
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/director/cpi_job
+  value: openstack_cpi_golang
+
+- type: replace
+  path: /cloud_provider/template
+  value:
+    name: openstack_cpi_golang
+    release: bosh-openstack-cpi

ci/tasks/deploy-manual-networking.sh

@@ -16,7 +16,6 @@ source bosh-openstack-cpi-release/ci/tasks/utils.sh
 : "${openstack_api_key:?}"
 : "${openstack_domain:?}"
 : "${DEBUG_BATS:?}"
-: "${distro:?}"
 openstack_ca_file_path="${openstack_ca_file_path:-}"
 optional_value availability_zone
 
@@ -56,13 +55,6 @@ pushd bosh-openstack-cpi-release
 
 popd
 cp ./bosh-openstack-cpi-dev.tgz "${deployment_dir}/${cpi_release_name}.tgz"
-cp ./stemcell-director/*.tgz "${deployment_dir}/stemcell.tgz"
-prepare_bosh_release "${distro}"
-
-echo "Calculating MD5 of original stemcell:"
-md5sum stemcell-director/*.tgz
-echo "Calculating MD5 of copied stemcell:"
-md5sum "${deployment_dir}/stemcell.tgz"
 
 cd "${deployment_dir}"
 
@@ -77,9 +69,9 @@ bosh-go int ../bosh-deployment/bosh.yml \
     -o ../bosh-deployment/openstack/cpi.yml \
     "${maybe_use_custom_ca_ops_file[@]}" \
     -o ../bosh-deployment/external-ip-not-recommended.yml \
-    -o ../bosh-deployment/misc/source-releases/bosh.yml \
     -o ../bosh-deployment/jumpbox-user.yml \
     -o ../bosh-openstack-cpi-release/ci/ops_files/deployment-configuration.yml \
+    -o ../bosh-openstack-cpi-release/ci/ops_files/use-golang-cpi.yml \
     -o ../bosh-openstack-cpi-release/ci/ops_files/custom-manual-networking.yml \
     -o ../bosh-openstack-cpi-release/ci/ops_files/timeouts.yml \
     -o ../bosh-openstack-cpi-release/ci/ops_files/remove-registry.yml \

ci/tasks/deploy-manual-networking.yml

@@ -6,8 +6,6 @@ image_resource:
     repository: boshcpi/openstack-cpi-release
 inputs:
   - name: bosh-openstack-cpi-release
-  - name: bosh-release
-  - name: stemcell-director
   - name: terraform-cpi
   - name: bosh-deployment
 outputs:
@@ -25,6 +23,5 @@ params:
   openstack_api_key:                   ""
   openstack_domain:                    ""
   DEBUG_BATS:                          ""
-  distro:                              ""
   openstack_file_path:                 ""
   availability_zone:                   replace-me

ci/tasks/run-manual-networking-bats.sh

@@ -1,12 +1,16 @@
 #!/usr/bin/env bash
 
-set -e -x
+# Network and credential variables are exported at runtime by the utils.sh
+# helpers (export_terraform_variable/optional_value), which shellcheck can't follow.
+# shellcheck disable=SC1091,SC2154
+
+set -eo pipefail
 
 source bosh-openstack-cpi-release/ci/tasks/utils.sh
 
-: ${stemcell_name:?}
-: ${openstack_flavor_with_ephemeral_disk:?}
-: ${openstack_flavor_with_no_ephemeral_disk:?}
+: "${stemcell_name:?}"
+: "${openstack_flavor_with_ephemeral_disk:?}"
+: "${openstack_flavor_with_no_ephemeral_disk:?}"
 
 optional_value availability_zone
 optional_value bats_rspec_tags
@@ -32,25 +36,27 @@ export_terraform_variable terraform-cpi/metadata "security_group"
 working_dir=$PWD
 # checked by BATs environment helper (bosh-acceptance-tests.git/lib/bat/env.rb)
 export BAT_STEMCELL="${working_dir}/stemcell/stemcell.tgz"
-export BAT_DIRECTOR=${director_public_ip}
+export BAT_DIRECTOR="${director_public_ip}"
 export BAT_INFRASTRUCTURE='openstack'
 export BAT_BOSH_CLI='bosh-go'
 
 export BOSH_ENVIRONMENT="${director_public_ip}"
 export BOSH_CLIENT="admin"
-export BOSH_CLIENT_SECRET="$( creds_path /admin_password )"
-export BOSH_CA_CERT="$( creds_path /director_ssl/ca )"
+BOSH_CLIENT_SECRET="$(creds_path /admin_password)"
+export BOSH_CLIENT_SECRET
+BOSH_CA_CERT="$(creds_path /director_ssl/ca)"
+export BOSH_CA_CERT
 
 ssh_private_key=$( creds_path /jumpbox_ssh/private_key | sed 's/$/\\n/' | tr -d '\n' )
 private_key_path=$(mktemp)
-echo -e "${ssh_private_key}" > ${private_key_path}
+echo -e "${ssh_private_key}" > "${private_key_path}"
 export BOSH_ALL_PROXY="ssh+socks5://jumpbox@${director_public_ip}:22?private-key=${private_key_path}"
 
 echo "using bosh CLI version..."
 bosh-go --version
 
 export BAT_DEPLOYMENT_SPEC="${working_dir}/bats-config.yml"
-cat > $BAT_DEPLOYMENT_SPEC <<EOF
+cat > "${BAT_DEPLOYMENT_SPEC}" <<EOF
 ---
 cpi: openstack
 properties:
@@ -95,4 +101,6 @@ EOF
 
 cd bats
 bundle install -j4
-bundle exec rspec --tag ~raw_ephemeral_storage --tag ~multiple_manual_networks ${bats_rspec_tags} spec
+# bats_rspec_tags may hold multiple space-separated flags; split into an array.
+read -ra rspec_tags <<< "${bats_rspec_tags:-}"
+bundle exec rspec --tag ~raw_ephemeral_storage --tag ~multiple_manual_networks "${rspec_tags[@]}" spec

ci/tasks/utils.sh

@@ -45,41 +45,6 @@ init_openstack_cli_env(){
 
 }
 
-prepare_bosh_release() {
-    local distribution=${1}
-    local bosh_release_version=${2}
-    local stemcell_version=${3}
-
-    use_compiled_release=true
-
-    local s3_path_to_bosh_release
-    s3_path_to_bosh_release=$(find_bosh_compiled_release "${distribution}" "${bosh_release_version}" "${stemcell_version}")
-
-    if [ -n "${s3_path_to_bosh_release}" ]; then
-        echo "Using compiled BOSH release: s3://bosh-compiled-release-tarballs/$s3_path_to_bosh_release"
-        # shellcheck disable=SC2154
-        aws --no-sign-request s3 cp "s3://bosh-compiled-release-tarballs/${s3_path_to_bosh_release}" "${deployment_dir}/bosh-release.tgz"
-    else
-        use_compiled_release=false
-    fi
-
-    if [ "${use_compiled_release}" = "false" ]; then
-       echo "Using BOSH release from sources"
-       # shellcheck disable=SC2154
-       cp ./bosh-release/*.tgz "${deployment_dir}/bosh-release.tgz"
-    fi
-}
-
-find_bosh_compiled_release(){
-    local distribution=$1
-    local bosh_release_version=${2:-$(cat ./bosh-release/version)}
-    local stemcell_version=${3:-$(cat ./stemcell-director/version)}
-
-    local s3_path_to_bosh_release
-    s3_path_to_bosh_release=$(aws --no-sign-request s3 ls s3://bosh-compiled-release-tarballs | grep -oE "[^ ](\w|-)*$bosh_release_version.+$distribution.+$stemcell_version.*\.tgz" | sort -r | head -1)
-    echo "${s3_path_to_bosh_release}"
-}
-
 export_terraform_variable() {
     local metadata=$1
     local variable_name=$2