alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-08 12:10:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: cache submodules

Browse Source
This commit is contained in:
Fu Hanxi
2022-07-12 16:48:54 +08:00
parent 9baa17ac57
commit 62ca8e2fb4
7 changed files with 82 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tools/ci/python_packages/gitlab_api.py
View File

@@ -177,7 +177,7 @@ class Gitlab(object):
return job_id_list
@retry
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None) -> str:
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None) -> str:
"""
Download archive of certain commit of a repository and extract to destination path
@@ -191,6 +191,23 @@ class Gitlab(object):
else:
project = self.gitlab_inst.projects.get(project_id)
if cache_dir:
local_archive_file = os.path.join(cache_dir, f'{ref}.tar.gz')
os.makedirs(os.path.dirname(local_archive_file), exist_ok=True)
if os.path.isfile(local_archive_file):
print('Use cached archive file. Skipping download...')
else:
with open(local_archive_file, 'wb') as fw:
try:
project.repository_archive(sha=ref, streamed=True, action=fw.write)
except gitlab.GitlabGetError as e:
print('Failed to archive from project {}'.format(project_id))
raise e
print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
return self.decompress_archive(local_archive_file, destination)
# no cache
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
try:
project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
@@ -198,9 +215,13 @@ class Gitlab(object):
print('Failed to archive from project {}'.format(project_id))
raise e
print('archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
with tarfile.open(temp_file.name, 'r') as archive_file:
return self.decompress_archive(temp_file.name, destination)
@staticmethod
def decompress_archive(path: str, destination: str) -> str:
with tarfile.open(path, 'r') as archive_file:
root_name = archive_file.getnames()[0]
archive_file.extractall(destination)