Pour tout problème contactez-nous par mail : support@froggit.fr | La FAQ :grey_question: | Rejoignez-nous sur le Chat :speech_balloon:

Skip to content
Snippets Groups Projects

Draft: Resolve "fix: export all don't export projects"

Open Draft: Resolve "fix: export all don't export projects"
39-fix-export-all-don-t-export-projects into master
Open Christophe Chaudier requested to merge 39-fix-export-all-don-t-export-projects into master
Compare
and
1 file
+ 63
− 44
Compare changes
  • Side-by-side
  • Inline
mygb.sh
+ 63
− 44
@@ -101,7 +101,7 @@ _help() {
fi
}
_check_dependencies(){
_check_dependencies() {
_all_dependencies_are_installed=true
for dep in ${DEPENDENCIES}; do
@@ -125,11 +125,12 @@ _load_config() {
echo "Use : mygb.sh init"
_fail "File ${config_file} doesn't exist"
fi
backup_dir=${backup_dir:-/tmp/mygb}
mygb_tmp_dir=${backup_dir}/${mygb_date}
mygb_file="${backup_dir}/mygb_${mygb_date}.tgz"
mkdir -p "${mygb_tmp_dir}"
projects_exported_list="${mygb_tmp_dir}/projects_exported_list.txt"
if [[ ${export_archived} == false ]]; then
archived_filter="&archived=false"
@@ -141,7 +142,7 @@ _load_config() {
_info
}
_info(){
_info() {
echo "
GitLab URL [${gitlab_url}]
+ Mode dry_run [${dry_run}]
@@ -194,7 +195,7 @@ _api() {
_nb_of_projects_from_group_page() {
_api GET "groups/${group_url_encoded}/projects?include_subgroups=true&per_page=${API_PER_PAGE}" --head \
| grep 'x-total-pages'| cut -d ':' -f 2 | tr -d "[:space:]"
| grep 'x-total-pages' | cut -d ':' -f 2 | tr -d "[:space:]"
}
_get_projects_from_group() {
@@ -204,6 +205,18 @@ _get_projects_from_group() {
done
}
_nb_of_projects_page() {
_api GET "projects?per_page=${API_PER_PAGE}${archived_filter}${owned_filter}" --head \
| grep 'x-total-pages'| cut -d ':' -f 2 | tr -d "[:space:]"
}
_get_projects() {
total_projects_pages=$(_nb_of_projects_page)
for (( page=1; page<=total_projects_pages; page++ )); do
_api GET "projects?per_page=${API_PER_PAGE}&page=${page}${archived_filter}${owned_filter}" | jq -r '.[].path_with_namespace'
done
}
_get_project_infos() {
project_infos="$(_get_export_filename project "${project}").json"
_api GET "projects/${project_url_encoded}" | jq > "${project_infos}"
@@ -225,7 +238,7 @@ _get_curent_user_email() {
_api GET "user" | jq -r '.email'
}
_dry_run(){
_dry_run() {
cmd="${*}"
if [[ ${dry_run} == false ]]; then
eval "${cmd}"
@@ -267,6 +280,7 @@ _export_project() {
_dry_run _schedule_project_export
_dry_run _wait_export_finished
_dry_run _download_project_export
echo "${project}" >> "${projects_exported_list}"
((nb_projects_exported++))
fi
}
@@ -308,12 +322,10 @@ _export_group() {
_dry_run _download_group_export
((nb_groups_exported++))
if [[ ${export_all} == false ]]; then
projects=$(_get_projects_from_group)
for project in ${projects}; do
_export_project
done
fi
projects=$(_get_projects_from_group)
for project in ${projects}; do
_export_project
done
fi
}
@@ -329,31 +341,22 @@ _get_groups() {
done
}
_nb_of_projects_page() {
_api GET "projects?per_page=${API_PER_PAGE}${archived_filter}${owned_filter}" --head \
| grep 'x-total-pages'| cut -d ':' -f 2 | tr -d "[:space:]"
}
_get_projects() {
total_projects_pages=$(_nb_of_projects_page)
for (( page=1; page<=total_projects_pages; page++ )); do
_api GET "projects?per_page=${API_PER_PAGE}&page=${page}${archived_filter}${owned_filter}" | jq -r '.[].path'
done
}
_export_all(){
_export_all() {
if [[ ${export_all} == true ]]; then
echo "Export all my groups and projects"
for group in $(_get_groups); do
_export_group
done
curent_user_id="$(_get_curent_user_id)"
# on Gitlab.com we want only owned project
if [[ "${gitlab_url}" == "https://gitlab.com/" ]]; then
owned_filter="&owned=true"
fi
for project in $(_get_projects); do
_export_project
# export the project only if is not already exported
grep -q "${project}" "${projects_exported_list}" || _export_project
done
fi
}
@@ -374,7 +377,7 @@ _export() {
# ---[ Import ]---
_import_all(){
_import_all() {
local file_to_import="${1}"
if [[ -n $file_to_import ]]; then
@@ -415,7 +418,7 @@ _import() {
# ---[ Import Group ]---
_import_group(){
_import_group() {
local file_to_import="${1}"
if [[ -n $file_to_import ]]; then
@@ -463,7 +466,7 @@ _schedule_group_import() {
# ---[ Import Project ]---
_import_project(){
_import_project() {
local file_to_import="${1}"
if [[ -n $file_to_import ]]; then
@@ -491,7 +494,6 @@ _import_project(){
echo -e "\nImporting projet : path [${project_path}] in namespace [${project_full_namespace}] ..."
_dry_run _schedule_projet_import
_dry_run _wait_project_import_finished
fi
}
@@ -555,11 +557,11 @@ _report_all() {
local file_to_check="${backup_file}"
report_file="${backup_dir}/mygb_${mygb_date}_report_missing_users.csv"
report_tmp_file="${mygb_tmp_dir}/mygb_${mygb_date}_report_users_to_check.csv"
echo "username;error" > "${report_file}"
report_tmp_users_to_check_file="${mygb_tmp_dir}/mygb_${mygb_date}_report_users_to_check.csv"
echo "Creating report of missing user(s) from file [${file_to_check}] ..."
echo "username;error" > "${report_file}"
if [[ -n $file_to_check ]]; then
import_tmp_dir=$(mktemp -d -p "${mygb_tmp_dir}" tmp_report_all.XXXXX)
@@ -577,17 +579,28 @@ _report_all() {
cd "${import_tmp_dir}" || _fail "Can't move to ${import_tmp_dir}"
done
done
echo "Check users :"
while IFS= read -r line; do
username="$(echo "${line}" | cut -d ";" -f 1)"
public_email="$(echo "${line}" | cut -d ";" -f 2)"
if ! _check_user_exist "${username}"; then
echo "${username};user don't exist" >> "${report_file}"
if [[ -z ${public_email} ]]; then
echo "${username};source user without public_email" >> "${report_file}"
elif ! _check_user_exist "${username}"; then
dest_username=$(_get_user_with_public_email "${public_email}")
if [[ -z ${dest_username} ]]; then
echo "${username};user don't exist" >> "${report_file}"
else
echo "${username};user exist with the same source public_email but with the name ${dest_username}" >> "${report_file}"
fi
elif ! _check_user_public_email "${username}" "${public_email}"; then
echo "${username};as not the source public_email : ${public_email}" >> "${report_file}"
dest_username=$(_get_user_with_public_email "${public_email}")
if [[ -z ${dest_username} ]]; then
echo "${username};exist but do not have the source public_email : ${public_email}" >> "${report_file}"
else
echo "${username};user exist with the same source public_email but with the name ${dest_username}" >> "${report_file}"
fi
fi
done < <(sort -u < "${report_tmp_file}")
done < <(sort -u < "${report_tmp_users_to_check_file}")
fi
echo "----------------------[ Users report ]----------------------"
@@ -597,7 +610,7 @@ _report_all() {
_succed "Report done : ${report_file}"
}
_list_users_group(){
_list_users_group() {
local file_to_check="${1}"
if [[ -n $file_to_check ]]; then
import_group_tmp_dir=$(mktemp -d -p "${mygb_tmp_dir}" tmp_report_group.XXXXX)
@@ -612,11 +625,11 @@ _list_users_group(){
tar zxf "${file_to_extract}" || _fail "Can't extract file ${file_to_extract}"
# shellcheck disable=SC2002
cat ./tree/groups/*/members.ndjson | jq -r '.user.username + ";" + .user.public_email' >> "${report_tmp_file}"
cat ./tree/groups/*/members.ndjson | jq -r '.user.username + ";" + .user.public_email' >> "${report_tmp_users_to_check_file}"
fi
}
_list_users_project(){
_list_users_project() {
local file_to_check="${1}"
if [[ -n $file_to_check ]]; then
import_projet_tmp_dir=$(mktemp -d -p "${mygb_tmp_dir}" tmp_report_projet.XXXXX)
@@ -631,7 +644,7 @@ _list_users_project(){
tar zxf "${file_to_extract}" || _fail "Can't extract file ${file_to_extract}"
# shellcheck disable=SC2002
cat ./tree/project/project_members.ndjson | jq -r '.user.username + ";" + .user.public_email' >> "${report_tmp_file}"
cat ./tree/project/project_members.ndjson | jq -r '.user.username + ";" + .user.public_email' >>"${report_tmp_users_to_check_file}"
fi
}
@@ -652,6 +665,12 @@ _check_user_exist() {
test "$(_api GET "users?username=${username_to_check}" | jq -r '.[].id')"
}
_get_user_with_public_email() {
local public_email_to_search=${1}
_api GET "users?search=${public_email_to_search}" | jq -r '.[].username'
}
# ---[ Main ]---
_main() {

Froggit est fièrement fabriqué avec amour par Lydra avec des Logiciels Libres et hébergé en France chez Scaleway | Status du service