Skip to content
Snippets Groups Projects
Commit 910ef918 authored by Donghee Kang's avatar Donghee Kang
Browse files

Update python approach

parent ce3a9ab5
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
migration_group_export_import.py
+ 62
− 17
View file @ 910ef918
......@@ -12,26 +12,34 @@ Disable ssl warning: suppresses ssl warning for disabling certificate check
"""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
KIT_GITLAB_URL = "https://gitlab.kit.edu"
##########################################################################################################
# Credential
##########################################################################################################
KIT_PILOT_URL = "https://git.scc.kit.edu"
KIT_GITLAB_URL = "https://gitlab.kit.edu"
KIT_GITLAB_TOKEN = "-------------------------" # Personal Access Token
KIT_PILOT_TOKEN = "=========================" # Personal Access Token
KIT_PILOT_TOKEN = "---------------------------------" # Personal Access Token, please change with yours
KIT_GITLAB_TOKEN = "----------------------------------" # Personal Access Token, please change with yours
gl_kit = gitlab.Gitlab(KIT_GITLAB_URL, private_token=KIT_GITLAB_TOKEN, ssl_verify=False, api_version="4")
gl_pilot = gitlab.Gitlab(KIT_PILOT_URL, private_token=KIT_PILOT_TOKEN, ssl_verify=False, api_version="4")
##############################################################################################
# parameter
##############################################################################################
##########################################################################################################
# Parameter
##########################################################################################################
# source group id
group_id_in_source = 123456789 # required parameter
# In destination(gitlab.kit.edu), subgroup "migration" must be existing in advance
# If subgroup "migration" is not existing yet, please create it at the web interface first.
group_namespace_in_target = 'kit/scc/sys/gitlab/migration'
##############################################################################################
group_id_in_source = 19277 # required parameter
# In target gitlab, group path has to define. "KIT / Institute" must be existing.
# If the last subgroup "Migration" is not existing,
# "Migration" will be created first and projects wil be transferred on this subgroup.
# If the last subgroup "Migration" is already existing,
# then projects will be transferred to existing subgroup, directly.
group_namespace_in_target = 'kit/institute/migration'
##########################################################################################################
# Migration
##########################################################################################################
class migration():
def export_projects(self):
......@@ -61,7 +69,7 @@ class migration():
# wait for the preparation shortly
time.sleep(3)
print("Start exporting process for project {} from the source GitLab >>> :^^: <<< ".format(project_name))
print("Exporting process for project {} from the source GitLab start".format(project_name))
# Wait 10sec(small project) or 100sec(big project)
with tqdm(total=100, desc="Exporting {}".format(project_name), bar_format="{l_bar}{bar} [ time left: {remaining} ]") as pbar:
......@@ -76,13 +84,47 @@ class migration():
with open(file_is, 'wb') as f:
project_export.download(streamed=True, action=f.write)
def import_projects(self):
gl_kit.auth()
current_working_directory = os.getcwd()
exported_file_directory = current_working_directory + "/exported_files"
# create last subgroup if it is not existing
structure_of_group = group_namespace_in_target.split('/')
subgroup_id = None
layer = 0
for group_name in structure_of_group:
layer += 1
if layer == 1:
group = gl_kit.groups.get(group_name.upper())
else:
group = gl_kit.groups.get(subgroup_id)
parent_group_id = subgroup_id
subgroups = group.subgroups.list(get_all=True, iterator=True)
for subgroup in subgroups:
subgroup_id = None
if subgroup.attributes.get('path') == structure_of_group[layer]:
subgroup_id = subgroup.attributes.get('id')
break
# create last subgroup, and provide its id
if (layer == (len(structure_of_group)-1)) and (not subgroup_id):
new_subgroup = gl_kit.groups.create(
{
'name': structure_of_group[layer].capitalize(),
'path': structure_of_group[layer],
'parent_id': parent_group_id
}
)
new_subgroup.save()
time.sleep(3)
subgroup_id = new_subgroup.id
print("New subgroup is created on the path : {}".format(group_namespace_in_target))
# prepare the list of exported files under exported_file_directory
list_of_projects = []
list_of_files = os.listdir(exported_file_directory)
......@@ -92,14 +134,14 @@ class migration():
#name_of_project = project.strip("_export.tar.gz") # thjs does not work for a_export.tar.gz
list_of_projects.append(name_of_project)
# importing
# Importing
for project_name in list_of_projects:
file_is = "{}/{}_export.tar.gz".format(exported_file_directory, project_name)
with open(file_is, 'rb') as f:
output = gl_kit.projects.import_project(
f,
path=project_name,
name=project_name.replace('-', ' '),
name=project_name.replace('-', ' ').capitalize(),
namespace=group_namespace_in_target,
override_params={'visibility': 'private'},
)
......@@ -114,6 +156,9 @@ class migration():
pbar.update(1)
project_import.refresh()
print("Exporting and Importing group is completed. Please take a look transferred projects on your group")
print("https://gitlab.kit.edu/{}".format(group_namespace_in_target))
if __name__ == "__main__":
# let's start
......
migration_group_import_only.py
+ 60
− 13
View file @ 910ef918
......@@ -12,24 +12,32 @@ Disable ssl warning: suppresses ssl warning for disabling certificate check
"""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
##########################################################################################################
# Credential
##########################################################################################################
KIT_GITLAB_URL = "https://gitlab.kit.edu"
KIT_TOKEN = "--------------------------" # Personal Access Token
KIT_TOKEN = "---------------------------------" # Personal Access Token, please change with yours
gl = gitlab.Gitlab(KIT_GITLAB_URL, private_token=KIT_TOKEN, ssl_verify=False, api_version="4")
gl.auth()
##############################################################################################
# parameter
##############################################################################################
# In destination(gitlab.kit.edu), subgroup "migration" must be existing in advance
# If subgroup "migration" is not existing yet, please create it at the web interface first.
group_namespace_in_target = 'kit/scc/sys/gitlab/migration'
# Exported files are already available in your local {PWD}/exported_file directory.
list_of_projects = ["my_project_1","my_project_2","my_project_3"]
##########################################################################################################
# Parameter
##########################################################################################################
# Source: Exported files are available in your local {PWD}/exported_file directory.
list_of_projects = ["my_project_1", "my_project_2", "my_project_3"]
##############################################################################################
# In target gitlab, group path has to define. "KIT / Institute" must be existing.
# If the last subgroup "Migration" is not existing,
# "Migration" will be created first and projects wil be transferred on this subgroup.
# If the last subgroup "Migration" is already existing,
# then projects will be transferred to existing subgroup, directly.
group_namespace_in_target = 'kit/institute/migration'
##########################################################################################################
# Migration
##########################################################################################################
class migration():
def import_projects(self):
......@@ -37,20 +45,56 @@ class migration():
current_working_directory = os.getcwd()
exported_file_directory = current_working_directory + "/exported_files"
# create last subgroup if it is not existing
structure_of_group = group_namespace_in_target.split('/')
subgroup_id = None
layer = 0
for group_name in structure_of_group:
layer += 1
if layer == 1:
group = gl.groups.get(group_name.upper())
else:
group = gl.groups.get(subgroup_id)
parent_group_id = subgroup_id
subgroups = group.subgroups.list(get_all=True, iterator=True)
for subgroup in subgroups:
subgroup_id = None
if subgroup.attributes.get('path') == structure_of_group[layer]:
subgroup_id = subgroup.attributes.get('id')
break
# create last subgroup, and provide its id
if (layer == (len(structure_of_group)-1)) and (not subgroup_id):
new_subgroup = gl.groups.create(
{
'name': structure_of_group[layer].capitalize(),
'path': structure_of_group[layer],
'parent_id': parent_group_id
}
)
new_subgroup.save()
time.sleep(3)
subgroup_id = new_subgroup.id
print("New subgroup is created on the path : {}".format(group_namespace_in_target))
for project_name in list_of_projects:
file_is = "{}/{}_export.tar.gz".format(exported_file_directory, project_name)
with open(file_is, 'rb') as f:
output = gl.projects.import_project(
f,
path=project_name,
name=project_name,
name=project_name.capitalize(),
namespace=group_namespace_in_target,
override_params={'visibility': 'private'},
)
# wait for the preparation shortly
time.sleep(3)
print("Start importing process for project {} on target GitLab <<< :^^: >>> ".format(project_name))
print("Start importing process for project {} on target GitLab ".format(project_name))
# Get a ProjectImport object to track the import status
project_import = gl.projects.get(output['id'], lazy=True).imports.get()
......@@ -62,6 +106,9 @@ class migration():
pbar.update(1)
project_import.refresh()
print("Importing projects into the group is completed. Please take a look transferred projects on your group")
print("https://gitlab.kit.edu/{}".format(group_namespace_in_target))
if __name__ == "__main__":
# let's start collecting project information
......
migration_project_export_import.py
+ 24
− 17
View file @ 910ef918
......@@ -12,11 +12,14 @@ Disable ssl warning: suppresses ssl warning for disabling certificate check
"""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
KIT_GITLAB_URL = "https://gitlab.kit.edu"
##########################################################################################################
# Credential
##########################################################################################################
KIT_PILOT_URL = "https://git.scc.kit.edu"
KIT_GITLAB_URL = "https://gitlab.kit.edu"
KIT_GITLAB_TOKEN = "--------------------------" # Personal Access Token
KIT_PILOT_TOKEN = "==========================" # Personal Access Token
KIT_PILOT_TOKEN = "----------------------------------" # Personal Access Token, please change with yours
KIT_GITLAB_TOKEN = "----------------------------------" # Personal Access Token, please change with yours
gl_kit = gitlab.Gitlab(KIT_GITLAB_URL, private_token=KIT_GITLAB_TOKEN, ssl_verify=False, api_version="4")
gl_kit.auth()
......@@ -24,30 +27,32 @@ gl_kit.auth()
gl_pilot = gitlab.Gitlab(KIT_PILOT_URL, private_token=KIT_PILOT_TOKEN, ssl_verify=False, api_version="4")
gl_pilot.auth()
##############################################################################################
# parameter
##############################################################################################
project_id = None # None or 123456789
##########################################################################################################
# Parameter
##########################################################################################################
project_id_in_source = None # (Optional) None or 123456789
source_namespace = "dq1464/migration_project" # user needs to define project_namespace (optional)
source_namespace = "xx1234/migration_project" # user/project namespace in source gitlab
target_namespace = "donghee.kang" # user needs to define user_namespace
target_namespace = "firstname.lastname" # user namespace in target gitlab
##############################################################################################
##########################################################################################################
# Migration
##########################################################################################################
class migration():
def export_import_project(self):
# parameter: project id has a higher priority
if project_id:
project_target = project_id
# parameter: project id has a higher priority
if project_id_in_source:
project_access = project_id_in_source
elif source_namespace:
project_target = source_namespace
project_access = source_namespace
else:
print("Please provide project_id or project_namespace from the source GtiLab")
# export
project = gl_pilot.projects.get(project_target)
project = gl_pilot.projects.get(project_access)
project_export = project.exports.create()
# replace spaces with dash
......@@ -60,7 +65,7 @@ class migration():
# wait for the preparation shortly
time.sleep(3)
print("Start exporting process for project {} from the source GitLab >>> :^^: <<< ".format(project_name))
print("Exporting process for project {} from the source GitLab start".format(project_name))
# Wait 10sec(small project) or 100sec(big project)
with tqdm(total=100, desc="Exporting {}".format(project_name), bar_format="{l_bar}{bar} [ time left: {remaining} ]") as pbar:
......@@ -82,7 +87,7 @@ class migration():
output = gl_kit.projects.import_project(
f,
path=project_name,
name=project.name,
name=project.name.capitalize(),
namespace=target_namespace,
override_params={'visibility': 'private'},
)
......@@ -97,6 +102,8 @@ class migration():
pbar.update(1)
project_import.refresh()
print("Importing project {} to the target GitLab is completed ".format(project_name))
if __name__ == "__main__":
# let's start
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Help | Imprint | Privacy policy | Accessibility | Contact