Merge branch '61-container-purge-tmp-images-in-ci' into 'master'

Resolve "(container) purge tmp images in CI"

Closes #61

See merge request lydra/gitlab-ci-templates!48

.gitlab-ci.yml

@@ -76,6 +76,12 @@ container:delivery:my-image:
   variables:
     IMAGE_NAME: "my-image"
 
+container:security:trivy-fail:
+  extends:  container:security
+  variables:
+    IMAGE_NAME: "cached-image"
+  allow_failure: true
+
 container:security:
   allow_failure: true
 

doc/fr/livraison.md

@@ -17,12 +17,12 @@ Dans notre exemple le résultat de la livraison est une image de conteneur car c
 ### Branche de fonctionnalité
 
 A chaque `git push` sur GitLab dans une branche de fonctionnalité le pipeline génère une image temporaire dans le registre de conteneur du projet aillant comme nom :
-`mon_projet/branche-<nom_branche>:tmp-branch-<nom_de_branche>-<hash_court>`
+`mon_projet/tmp:tmp-branch-<nom_de_branche>-<hash_court>`
 
 ### Fusion dans la branche principale
 
 A chaque _merge_ suite à une _Merge Request_ dans GitLab dans la branche principale _master_ ou _main_ (à partir de [GitLab 14](https://docs.gitlab.com/ee/user/project/repository/branches/default.html)) le pipeline génère une image temporaire dans le registre du projet aillant comme nom :
-`mon_projet:tmp-branch-<nom_de_branche>-<hash_court>`
+`mon_projet/tmp:tmp-branch-<nom_de_branche>-<hash_court>`
 
 ### Création d’un tag
 

gci-templates/job/container/build.yml

@@ -3,7 +3,7 @@
 #   - https://cloud.google.com/build/docs/kaniko-cache
 #
 # This job use kaniko with cache
-# The image is taged with tmp-latest-${CI_COMMIT_SHA}
+# The image is taged with tmp-<branch_name>-${CI_COMMIT_SHA}
 #
 # Activate the container registery cleanup policy if you are on GitLab
 # doc : https://docs.gitlab.com/ee/user/packages/container_registry/#delete-images-by-using-a-cleanup-policy

templates/job/container/build.yml

@@ -3,7 +3,10 @@
 #   - https://cloud.google.com/build/docs/kaniko-cache
 #
 # This job use kaniko with cache
-# The image is taged with tmp-latest-${CI_COMMIT_SHA}
+# This Job create a container's temporary images
+# This image can be scanned by Trivy in the job container:security
+#
+# The image is taged with tmp-<branch_name>-${CI_COMMIT_SHA}
 #
 # Activate the container registery cleanup policy if you are on GitLab
 # doc : https://docs.gitlab.com/ee/user/packages/container_registry/#delete-images-by-using-a-cleanup-policy
@@ -15,3 +18,69 @@
 container:build:
   extends:  .container:build
   stage:    package
+
+# Purge of tempory images
+# docs : 
+# - https://docs.gitlab.com/ee/user/packages/container_registry/#delete-images
+# - https://docs.gitlab.com/ee/user/packages/container_registry/#delete-images-using-gitlab-cicd
+# - https://github.com/genuinetools/reg
+#
+# This job prune the container's tmp images
+# - merge trigger : delete all images of the merged branch
+
+
+container:purge:
+  extends:  .base_tpl
+  image:    buildpack-deps:buster-curl
+  stage:    clean
+  rules:
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  variables:
+    REG_SHA256: ade837fc5224acd8c34732bf54a94f579b47851cc6a7fd5899a98386b782e228
+    REG_VERSION: 0.16.1
+    REG_OPTS: "--insecure --auth-url ${CI_REGISTRY} -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD}"
+  before_script:
+    - curl --fail --show-error --location "https://github.com/genuinetools/reg/releases/download/v$REG_VERSION/reg-linux-amd64" --output /usr/local/bin/reg
+    - echo "$REG_SHA256  /usr/local/bin/reg" | sha256sum -c -
+    - chmod a+x /usr/local/bin/reg
+    - cd ${working_directory}
+  script: |
+    clean_images=false
+
+    if [[ -n "${CI_COMMIT_TAG}" ]]; then
+      echo "It's a tag 🔖"
+      SOURCE_BRANCH=${CI_DEFAULT_BRANCH}
+      clean_images=true
+    elif (echo ${CI_COMMIT_TITLE} |  grep "Merge branch '.*' into 'master'"); then
+      echo "It's a merge 🔀"
+      SOURCE_BRANCH=$(echo $CI_COMMIT_TITLE |  cut -d "'" -f 2)
+      clean_images=true
+    fi
+
+    if [[ "${clean_images}" = true ]]; then
+      echo "Source branch to purge : ${SOURCE_BRANCH}"
+
+      # Namespace
+      container_namespace="${CI_PROJECT_PATH}/${tpl_tmp_image_namespaces}"
+
+      # Image Name
+      if [[ -n "${IMAGE_NAME}" ]]; then
+        container_image_name="/${IMAGE_NAME}"
+      else
+        container_image_name=""
+      fi
+      full_image_name=${container_namespace}${container_image_name}
+      container_base_tag="tmp-branch-${SOURCE_BRANCH//\//_}"
+
+      # Clean Repository
+      echo "🧹 Clean tmp images"
+      echo "  🏭 registry : ${CI_REGISTRY}"
+      echo "  📦     name : ${full_image_name}"
+      for tag in $(reg tags ${REG_OPTS} ${full_image_name} | grep ${container_base_tag}); do
+        echo "      🏷️ tags : ${tag}"
+        reg rm ${REG_OPTS} ${full_image_name}:${tag}
+      done
+    else
+      echo "Nothing to purge in the GitLab container registry !"
+    fi

templates/job/container/security.yml

@@ -43,8 +43,7 @@ container:security:
     # update vulnerabilities db
     - time trivy --download-db-only --no-progress
     - |- # Namespace
-        TMP_IMAGE_BASENAME="branch-${CI_COMMIT_REF_NAME//\//_}"
-        container_namespace="${CI_PROJECT_PATH}/${TMP_IMAGE_BASENAME}"
+        container_namespace="${CI_PROJECT_PATH}/${tpl_tmp_image_namespaces}"
     - |- # Image name
         if [[ -n "${IMAGE_NAME}" ]]; then
           container_image_name="/${IMAGE_NAME}"
@@ -52,7 +51,7 @@ container:security:
           container_image_name=""      
         fi
     - |- # Tags
-        container_tag="tmp-${TMP_IMAGE_BASENAME}-$(echo ${CI_COMMIT_SHA} | cut -c1-8)"
+        container_tag="tmp-branch-${CI_COMMIT_REF_NAME//\//_}-$(echo ${CI_COMMIT_SHA} | cut -c1-8)"
     - |- # Full image name
         full_image_name=${container_namespace}${container_image_name}
         echo "⚙️ Test image: "

templates/stages.yml

@@ -6,10 +6,12 @@ stages:
   - test
   - delivery
   - deploy
+  - clean
 
 variables:
   working_directory: "${CI_PROJECT_DIR}"
   tpl_directory: "gci-templates"
+  tpl_tmp_image_namespaces: "tmp"
   MAVEN_OPTS: ""
   MAVEN_CLI_OPTS: ""
   MAVEN_SETTINGS: ""
@@ -17,6 +19,9 @@ variables:
   JAVA_VERSION: "11"
   JDK_TYPE: "openjdk"
   CONTAINER_CACHE_TTL: 6h
+  CI_DEBUG_TRACE: "false"
+  # true for activate CI debug tracing
+
 
 
 # /!\ removed soon /!\
@@ -126,7 +131,7 @@ variables:
     CONTAINER_CACHE_NAME: ${CI_REGISTRY_IMAGE}/cache
   script:
     - mkdir -p /kaniko/.docker
-    - |- # Test DELIVERY_REGISTRY_* vers
+    - |- # Test vars DELIVERY_REGISTRY_*
         delivery_cr=0
         if [[ "${CI_JOB_STAGE}" != "delivery" ]]; then
           DELIVERY_REGISTRY=""
@@ -151,11 +156,10 @@ variables:
         KANIKOCFG="{ ${KANIKOCFG} }"
         echo "${KANIKOCFG}" > /kaniko/.docker/config.json
     - |- # Namespace
-        TMP_IMAGE_BASENAME="branch-${CI_COMMIT_REF_NAME//\//_}"
         if [[ "${CI_JOB_STAGE}" == "delivery" ]]; then
           container_namespace="${CI_PROJECT_PATH}"
         else
-          container_namespace="${CI_PROJECT_PATH}/${TMP_IMAGE_BASENAME}"
+          container_namespace="${CI_PROJECT_PATH}/${tpl_tmp_image_namespaces}"
         fi
     - |- # Name
         if [[ -n "${IMAGE_NAME}" ]]; then
@@ -164,7 +168,7 @@ variables:
           container_image_name=""      
         fi
     - |- # Tags
-        container_default_tag="tmp-${TMP_IMAGE_BASENAME}-$(echo ${CI_COMMIT_SHA} | cut -c1-8)"
+        container_default_tag="tmp-branch-${CI_COMMIT_REF_NAME//\//_}-$(echo ${CI_COMMIT_SHA} | cut -c1-8)"
         if [[ "${CI_JOB_STAGE}" != "delivery" ]]; then
           container_tags="${container_default_tag}"
         else