Pour tout problème contactez-nous par mail : support@froggit.fr | La FAQ :grey_question: | Rejoignez-nous sur le Chat :speech_balloon:

Skip to content
Snippets Groups Projects
Unverified Commit 90ece354 authored by Idriss's avatar Idriss
Browse files

Handle multi-bucket destinations

parent d284fa75
No related branches found
No related tags found
No related merge requests found
......@@ -14,7 +14,7 @@ mirror:
deliver:
stage: deliver
script:
- setsid ./ci/docker-deliver.sh "bucket-backup" "1.0"
- setsid ./ci/docker-deliver.sh "bucket-backup" "2.0"
only:
refs:
- /^(main.*)$/
......
......@@ -29,6 +29,11 @@ You'll find a tutorial on how to use it with ansible on the [comwork cloud docum
* `DATE_FORMAT` (optional): backup date format (folder name). Default `+%F` which corresponds to `YYYY-MM-JJ`)
* `MAX_RETENTION` (optional): number of days to keep backup. Default: `5` days.s
Not: you can upload to multiple backup using an increment suffix (the suffix must be consecutives and start from `1`):
* `BUCKET_ENDPOINT_1`, `BUCKET_ACCESS_KEY_1`, `BUCKET_SECRET_KEY_1`, `BUCKET_NAME_1`
* `BUCKET_ENDPOINT_2`, `BUCKET_ACCESS_KEY_2`, `BUCKET_SECRET_KEY_2`, `BUCKET_NAME_2`
* etc
## Scaleway endpoints
With scaleway you either configure this container like this:
......
......@@ -8,12 +8,23 @@ DATE_FORMAT="{{ backup_date_format }}"
EXECUTION_DATE="$(date "$DATE_FORMAT")"
# Bucket file sync variables
{% if bucket_names is defined and bucket_names %}
{% set count = [] %}
{% for bn in bucket_names %}
{% set __ = count.append(1) %}
BUCKET_ENDPOINT_{{ count|length }}="{{ bucket_endpoint }}"
BUCKET_ACCESS_KEY_{{ count|length }}="{{ bucket_access_key }}"
BUCKET_SECRET_KEY_{{ count|length }}="{{ bucket_secret_key }}"
BUCKET_NAME_{{ count|length }}="{{ bn }}"
{% endfor %}
{% else %}
BUCKET_ENDPOINT="{{ bucket_endpoint }}"
BUCKET_ACCESS_KEY="{{ bucket_access_key }}"
BUCKET_SECRET_KEY="{{ bucket_secret_key }}"
{% if bucket_name is defined and bucket_name %}
BUCKET_NAME="{{ bucket_name }}"
{% endif %}
{% endif %}
{% if backup_cmd is defined and backup_cmd %}
{{ backup_cmd }}
......
......@@ -2,43 +2,83 @@
MC_BIN="/usr/bin/mc"
"${MC_BIN}" config host add cmw "${BUCKET_ENDPOINT}" "${BUCKET_ACCESS_KEY}" "${BUCKET_SECRET_KEY}"
# Dates to keep
[[ ! $DATE_FORMAT ]] && DATE_FORMAT="+%F"
TODAY="$(date "$DATE_FORMAT")"
FILE_EXTENSION="${FILE_NAME##*.}"
dates_to_keep_in_grep="${TODAY}"
DATES_TO_KEEP="${TODAY}"
for i in $(seq 1 "$MAX_RETENTION"); do
dates_to_keep_in_grep="$dates_to_keep_in_grep\|$(date --date="$i day ago" "$DATE_FORMAT")"
DATES_TO_KEEP="$DATES_TO_KEEP\|$(date --date="$i day ago" "$DATE_FORMAT")"
done
bucket_subpath=""
[[ $BUCKET_NAME ]] && bucket_subpath="${BUCKET_NAME}/"
bucket_backup() {
endpoint="${1}"
access_key="${2}"
secret_key="${3}"
dest="${4}"
bucket_name="${5}"
echo "Copying backup file. ${BACKUP_LOCATION} -> cmw/${bucket_subpath}${BACKUP_FOLDER}/${TODAY}.${FILE_EXTENSION}"
"${MC_BIN}" cp "${BACKUP_LOCATION}" "cmw/${bucket_subpath}${BACKUP_FOLDER}/${TODAY}.${FILE_EXTENSION}"
"${MC_BIN}" config host add "${dest}" "${endpoint}" "${access_key}" "${secret_key}"
echo "Deleting old backup data"
echo "Deleting data older than $(date --date="${MAX_RETENTION} days ago" "${DATE_FORMAT}")"
# If you use a scaleway endpoint, there's some specificities to handle
if [[ $BUCKET_ENDPOINT =~ https://.+s3.fr-par.scw.cloud ]]; then
BUCKET_NAME="$(echo $BUCKET_ENDPOINT|sed "s/https:\/\/\(.*\)\.s3\.fr\-par\.scw\.cloud/\1/g")"
BUCKET_ENDPOINT="https://s3.fr-par.scw.cloud"
bucket_subpath=""
[[ $BUCKET_NAME ]] && bucket_subpath="${BUCKET_NAME}/"
fi
"${MC_BIN}" config host add rcmw "${BUCKET_ENDPOINT}" "${BUCKET_ACCESS_KEY}" "${BUCKET_SECRET_KEY}"
echo "${MC_BIN}" ls -r "rcmw/${bucket_subpath}${BACKUP_FOLDER}/"
"${MC_BIN}" ls -r "rcmw/${bucket_subpath}${BACKUP_FOLDER}/" |
awk '{print $6}' |
grep -v -w "$dates_to_keep_in_grep" |
while read backup_file; do
echo "Removing rcmw/${bucket_subpath}${BACKUP_FOLDER}/$backup_file"
"${MC_BIN}" rm "rcmw/${bucket_subpath}${BACKUP_FOLDER}/$backup_file"
done
[[ $bucket_name ]] && bucket_subpath="${bucket_name}/"
echo "[bucket_backup] Copying backup file. ${BACKUP_LOCATION} -> ${dest}/${bucket_subpath}${BACKUP_FOLDER}/${TODAY}.${FILE_EXTENSION}"
"${MC_BIN}" cp "${BACKUP_LOCATION}" "${dest}/${bucket_subpath}${BACKUP_FOLDER}/${TODAY}.${FILE_EXTENSION}"
echo "[bucket_backup] Deleting old backup data"
echo "[bucket_backup] Deleting data older than $(date --date="${MAX_RETENTION} days ago" "${DATE_FORMAT}")"
# If you use a scaleway endpoint, there's some specificities to handle
if [[ $endpoint =~ https://.+s3.fr-par.scw.cloud ]]; then
bucket_name="$(echo $endpoint|sed "s/https:\/\/\(.*\)\.s3\.fr\-par\.scw\.cloud/\1/g")"
endpoint="https://s3.fr-par.scw.cloud"
bucket_subpath=""
[[ $bucket_name ]] && bucket_subpath="${bucket_name}/"
fi
"${MC_BIN}" config host add "r${dest}" "${endpoint}" "${access_key}" "${secret_key}"
echo "${MC_BIN}" ls -r "r${dest}/${bucket_subpath}${BACKUP_FOLDER}/"
"${MC_BIN}" ls -r "r${dest}/${bucket_subpath}${BACKUP_FOLDER}/" |
awk '{print $6}' |
grep -v -w "$DATES_TO_KEEP" |
while read backup_file; do
echo "[bucket_backup] Removing r${dest}/${bucket_subpath}${BACKUP_FOLDER}/$backup_file"
"${MC_BIN}" rm "r${dest}/${bucket_subpath}${BACKUP_FOLDER}/$backup_file"
done
}
apply_bucket_backup() {
suffix="${1}"
var_endpoint="BUCKET_ENDPOINT${suffix}"
endpoint="${!var_endpoint}"
var_access_key="BUCKET_ACCESS_KEY${suffix}"
access_key="${!var_access_key}"
var_secret_key="BUCKET_SECRET_KEY${suffix}"
secret_key="${!var_secret_key}"
var_bucket_name="BUCKET_NAME${suffix}"
bucket_name="${!var_bucket_name}"
dest="cmw${suffix//_}"
if [[ $endpoint && $access_key && $secret_key ]]; then
echo "[apply_bucket_backup] endpoint=${endpoint} bucket_name=${bucket_name} dest=${dest}"
bucket_backup "${endpoint}" "${access_key}" "${secret_key}" "${dest}" "${bucket_name}"
return 0
fi
return 1
}
apply_bucket_backup ""
i=0
while true; do
if ! apply_bucket_backup "_${i}"; then
echo "[main] No more backup buckets..."
break
fi
(( i++ ))
done
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment