action.yml 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237
  1. name: 'Download and configure Kernel Source code'
  2. inputs:
  3. source_location:
  4. description: 'Folder path to save Kernel source'
  5. required: true
  6. type: string
  7. github_token:
  8. description: 'GitHub Token'
  9. required: true
  10. debug:
  11. description: 'Enable Logs'
  12. required: false
  13. type: boolean
  14. default: false
  15. runs:
  16. using: 'composite'
  17. steps:
  18. - name: Download and Prepare Manifest
  19. shell: bash
  20. working-directory: ${{ inputs.source_location }}
  21. run: |
  22. # Download and Prepare Manifest
  23. if [[ "$OP_MANIFEST" == https://* ]]; then
  24. curl --fail --show-error --location --proto '=https' "$OP_MANIFEST" -o manifest.xml
  25. elif [[ "$OP_BRANCH" == wild/* ]]; then
  26. cp "../manifests/$(echo "$OP_OS_VERSION" | tr '[:upper:]' '[:lower:]')/$OP_MANIFEST" manifest.xml
  27. else
  28. curl --fail --show-error --location --proto '=https' "https://raw.githubusercontent.com/OnePlusOSS/kernel_manifest/refs/heads/$OP_BRANCH/$OP_MANIFEST" -o manifest.xml
  29. fi
  30. - name: Download Manifest Archives
  31. shell: python
  32. env:
  33. PYTHONUNBUFFERED: "1"
  34. GITHUB_TOKEN: ${{ inputs.github_token }}
  35. DEBUG: ${{ inputs.debug }}
  36. working-directory: ${{ inputs.source_location }}
  37. run: |
  38. # Download Manifest Archives
  39. import xml.etree.ElementTree as ET
  40. import subprocess
  41. import os, shutil
  42. import time
  43. import glob
  44. from concurrent.futures import ThreadPoolExecutor
  45. import requests
  46. MAX_WORKERS = (os.cpu_count() or 2) * 4
  47. NPROC = int(subprocess.check_output("nproc", shell=True).strip())
  48. TARGET_REPO = "${{ github.repository }}"
  49. TOOLCHAIN_MAP = {
  50. "clang/host/linux-x86": "clang",
  51. "prebuilts/rust": "rust",
  52. "prebuilts/clang-tools": "clang-tools",
  53. "prebuilts/build-tools": "build-tools"
  54. }
  55. DEBUG = os.environ.get("DEBUG", "false").lower() == "true"
  56. aria_quiet_flags = "--quiet --summary-interval=0" if not DEBUG else ""
  57. print("::group::Download Manifest Archives")
  58. def get_release_parts(label, rev):
  59. url = f"https://api.github.com/repos/{TARGET_REPO}/releases?per_page=100"
  60. headers = {
  61. "Authorization": f"token {os.environ['GITHUB_TOKEN']}",
  62. "Accept": "application/vnd.github.v3+json"
  63. }
  64. for attempt in range(3):
  65. try:
  66. response = requests.get(url, headers=headers)
  67. response.raise_for_status()
  68. releases = response.json()
  69. release = next((r for r in releases if r['name'] == "Toolchains Mirror Cache" or r['tag_name'] == "toolchain-cache"), None)
  70. if not release: return []
  71. prefix = f"{label}-{rev}.tar.gz"
  72. return [(a['name'], a['url']) for a in release['assets'] if a['name'].startswith(prefix)]
  73. except Exception as e:
  74. print(f" [RETRY {attempt+1}] API failed: {e}")
  75. time.sleep(2)
  76. return []
  77. def sync_project(task):
  78. name, path, url, strip, rev = task
  79. if path not in ["./", "."]:
  80. os.makedirs(path, exist_ok=True)
  81. headers = (
  82. "-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' "
  83. "-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' "
  84. "-H 'Accept-Encoding: gzip, deflate, br' "
  85. "-H 'Connection: keep-alive' "
  86. "--tcp-fastopen"
  87. )
  88. print(f"Syncing: {name} -> {path}")
  89. start_time = time.time()
  90. print(f" [PENDING] {name}")
  91. try:
  92. label = None
  93. for repo_key, type_label in TOOLCHAIN_MAP.items():
  94. if repo_key in name:
  95. label = type_label
  96. break
  97. if label:
  98. base_filename = f"{label}-{rev}.tar.gz"
  99. asset_data = get_release_parts(label, rev)
  100. if not asset_data:
  101. print(f" [ERROR] No assets found for {base_filename} in release!")
  102. return False
  103. if DEBUG:
  104. print(f" [CACHE] Fetching {label} toolchain: {base_filename}...")
  105. for asset_name, api_url in asset_data:
  106. aria_cmd = (
  107. f"aria2c -x16 -s16 -k1M -j5 --file-allocation=none {aria_quiet_flags} "
  108. f"--header='Authorization: token {os.environ['GITHUB_TOKEN']}' "
  109. f"--header='Accept: application/octet-stream' "
  110. f"-o {asset_name} {api_url}"
  111. )
  112. subprocess.run(aria_cmd, shell=True, check=True, capture_output=not DEBUG)
  113. parts = sorted(glob.glob(f"{base_filename}.part*"))
  114. if parts:
  115. if DEBUG:
  116. print(f" [MERGE] Combining {len(parts)} parts for {rev}...")
  117. subprocess.run(f"cat {base_filename}.part* | tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M --no-same-owner --no-same-permissions -C {path} {strip}", shell=True, check=True)
  118. subprocess.run(f"rm {base_filename}.part*", shell=True, check=True)
  119. else:
  120. if os.path.exists(base_filename):
  121. if DEBUG:
  122. print(f" [EXTRACT] Single file detected...")
  123. subprocess.run(f"tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M --no-same-owner --no-same-permissions -f {base_filename} -C {path} {strip}", shell=True, check=True)
  124. os.remove(base_filename)
  125. else:
  126. print(f" [ERROR] {base_filename} missing after download!")
  127. return False
  128. else:
  129. cmd = f"curl -LfsS {headers} --retry 5 --connect-timeout 30 '{url}' | tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M -C {path} {strip}"
  130. subprocess.run(cmd, shell=True, check=True)
  131. duration = time.time() - start_time
  132. print(f"Synced {name} successfully! ({duration:.2f}s)")
  133. return True
  134. except subprocess.CalledProcessError as e:
  135. print(f" [ERROR] Command failed for {name}: {e.cmd}")
  136. print(f" Stderr: {e.stderr.decode() if e.stderr else 'No stderr'}")
  137. return False
  138. except Exception as e:
  139. print(f" [ERROR] Failed to sync {name}")
  140. return False
  141. global_start = time.time()
  142. with open('manifest.xml', 'r') as f:
  143. manifest_content = f.read()
  144. root = ET.fromstring(manifest_content)
  145. top_dir = os.getcwd()
  146. remotes = {r.get('name'): r.get('fetch').rstrip('/') for r in root.findall('remote')}
  147. default = root.find('default')
  148. def_remote = default.get('remote') if default is not None else None
  149. def_rev = default.get('revision') if default is not None else None
  150. sync_tasks = []
  151. post_process_data = []
  152. for project in root.findall('project'):
  153. name = project.get('name')
  154. path = project.get('path', name)
  155. remote_name = project.get('remote', def_remote)
  156. rev = project.get('revision', def_rev)
  157. base_url = remotes.get(remote_name)
  158. if not base_url: continue
  159. if "github.com" in base_url:
  160. url = f"{base_url}/{name}/archive/{rev}.tar.gz"
  161. strip = "--strip-components=1"
  162. elif "googlesource.com" in base_url:
  163. url = f"{base_url}/{name}/+archive/{rev}.tar.gz"
  164. strip = ""
  165. elif "git.codelinaro.org" in base_url:
  166. url = f"{base_url}/{name}/-/archive/{rev}.tar.gz"
  167. strip = "--strip-components=1"
  168. else:
  169. continue
  170. sync_tasks.append((name, path, url, strip, rev))
  171. for child in project:
  172. if child.tag in ['linkfile', 'copyfile']:
  173. post_process_data.append((path, child))
  174. if DEBUG:
  175. print(f"Starting parallel sync of {len(sync_tasks)} projects...")
  176. with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
  177. success_list = list(executor.map(sync_project, sync_tasks))
  178. if not all(success_list):
  179. print("::error::One or more projects failed to sync!")
  180. print("::endgroup::")
  181. exit(1)
  182. print("Processing linkfiles and copyfiles...")
  183. for path, child in post_process_data:
  184. src_rel = child.get('src')
  185. dest_rel = child.get('dest')
  186. if not src_rel or not dest_rel: continue
  187. src_path = os.path.join(top_dir, path, src_rel)
  188. dest_path = os.path.join(top_dir, dest_rel)
  189. os.makedirs(os.path.dirname(dest_path), exist_ok=True)
  190. if child.tag == 'linkfile':
  191. if os.path.lexists(dest_path): os.remove(dest_path)
  192. rel_target = os.path.relpath(src_path, os.path.dirname(dest_path))
  193. os.symlink(rel_target, dest_path)
  194. print(f" [Link] {dest_rel} -> {src_rel}")
  195. elif child.tag == 'copyfile':
  196. shutil.copy2(src_path, dest_path)
  197. print(f" [Copy] {dest_rel} from {src_rel}")
  198. total_duration = time.time() - global_start
  199. minutes = int(total_duration // 60)
  200. seconds = total_duration % 60
  201. print(f"Kernel Sync completed in {minutes}m {seconds:.2f}s")
  202. print("::endgroup::")