| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237 |
- name: 'Download and configure Kernel Source code'
- inputs:
- source_location:
- description: 'Folder path to save Kernel source'
- required: true
- type: string
- github_token:
- description: 'GitHub Token'
- required: true
- debug:
- description: 'Enable Logs'
- required: false
- type: boolean
- default: false
- runs:
- using: 'composite'
- steps:
- - name: Download and Prepare Manifest
- shell: bash
- working-directory: ${{ inputs.source_location }}
- run: |
- # Download and Prepare Manifest
- if [[ "$OP_MANIFEST" == https://* ]]; then
- curl --fail --show-error --location --proto '=https' "$OP_MANIFEST" -o manifest.xml
- elif [[ "$OP_BRANCH" == wild/* ]]; then
- cp "../manifests/$(echo "$OP_OS_VERSION" | tr '[:upper:]' '[:lower:]')/$OP_MANIFEST" manifest.xml
- else
- curl --fail --show-error --location --proto '=https' "https://raw.githubusercontent.com/OnePlusOSS/kernel_manifest/refs/heads/$OP_BRANCH/$OP_MANIFEST" -o manifest.xml
- fi
- - name: Download Manifest Archives
- shell: python
- env:
- PYTHONUNBUFFERED: "1"
- GITHUB_TOKEN: ${{ inputs.github_token }}
- DEBUG: ${{ inputs.debug }}
- working-directory: ${{ inputs.source_location }}
- run: |
- # Download Manifest Archives
- import xml.etree.ElementTree as ET
- import subprocess
- import os, shutil
- import time
- import glob
- from concurrent.futures import ThreadPoolExecutor
- import requests
-
- MAX_WORKERS = (os.cpu_count() or 2) * 4
- NPROC = int(subprocess.check_output("nproc", shell=True).strip())
- TARGET_REPO = "${{ github.repository }}"
- TOOLCHAIN_MAP = {
- "clang/host/linux-x86": "clang",
- "prebuilts/rust": "rust",
- "prebuilts/clang-tools": "clang-tools",
- "prebuilts/build-tools": "build-tools"
- }
- DEBUG = os.environ.get("DEBUG", "false").lower() == "true"
- aria_quiet_flags = "--quiet --summary-interval=0" if not DEBUG else ""
-
- print("::group::Download Manifest Archives")
-
- def get_release_parts(label, rev):
- url = f"https://api.github.com/repos/{TARGET_REPO}/releases?per_page=100"
- headers = {
- "Authorization": f"token {os.environ['GITHUB_TOKEN']}",
- "Accept": "application/vnd.github.v3+json"
- }
-
- for attempt in range(3):
- try:
- response = requests.get(url, headers=headers)
- response.raise_for_status()
- releases = response.json()
- release = next((r for r in releases if r['name'] == "Toolchains Mirror Cache" or r['tag_name'] == "toolchain-cache"), None)
- if not release: return []
-
- prefix = f"{label}-{rev}.tar.gz"
- return [(a['name'], a['url']) for a in release['assets'] if a['name'].startswith(prefix)]
- except Exception as e:
- print(f" [RETRY {attempt+1}] API failed: {e}")
- time.sleep(2)
- return []
-
- def sync_project(task):
- name, path, url, strip, rev = task
- if path not in ["./", "."]:
- os.makedirs(path, exist_ok=True)
-
- headers = (
- "-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' "
- "-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' "
- "-H 'Accept-Encoding: gzip, deflate, br' "
- "-H 'Connection: keep-alive' "
- "--tcp-fastopen"
- )
-
- print(f"Syncing: {name} -> {path}")
- start_time = time.time()
- print(f" [PENDING] {name}")
- try:
- label = None
- for repo_key, type_label in TOOLCHAIN_MAP.items():
- if repo_key in name:
- label = type_label
- break
-
- if label:
- base_filename = f"{label}-{rev}.tar.gz"
-
- asset_data = get_release_parts(label, rev)
-
- if not asset_data:
- print(f" [ERROR] No assets found for {base_filename} in release!")
- return False
-
- if DEBUG:
- print(f" [CACHE] Fetching {label} toolchain: {base_filename}...")
-
- for asset_name, api_url in asset_data:
- aria_cmd = (
- f"aria2c -x16 -s16 -k1M -j5 --file-allocation=none {aria_quiet_flags} "
- f"--header='Authorization: token {os.environ['GITHUB_TOKEN']}' "
- f"--header='Accept: application/octet-stream' "
- f"-o {asset_name} {api_url}"
- )
- subprocess.run(aria_cmd, shell=True, check=True, capture_output=not DEBUG)
-
- parts = sorted(glob.glob(f"{base_filename}.part*"))
- if parts:
- if DEBUG:
- print(f" [MERGE] Combining {len(parts)} parts for {rev}...")
- subprocess.run(f"cat {base_filename}.part* | tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M --no-same-owner --no-same-permissions -C {path} {strip}", shell=True, check=True)
- subprocess.run(f"rm {base_filename}.part*", shell=True, check=True)
- else:
- if os.path.exists(base_filename):
- if DEBUG:
- print(f" [EXTRACT] Single file detected...")
- subprocess.run(f"tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M --no-same-owner --no-same-permissions -f {base_filename} -C {path} {strip}", shell=True, check=True)
- os.remove(base_filename)
- else:
- print(f" [ERROR] {base_filename} missing after download!")
- return False
- else:
- cmd = f"curl -LfsS {headers} --retry 5 --connect-timeout 30 '{url}' | tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M -C {path} {strip}"
- subprocess.run(cmd, shell=True, check=True)
-
- duration = time.time() - start_time
- print(f"Synced {name} successfully! ({duration:.2f}s)")
- return True
- except subprocess.CalledProcessError as e:
- print(f" [ERROR] Command failed for {name}: {e.cmd}")
- print(f" Stderr: {e.stderr.decode() if e.stderr else 'No stderr'}")
- return False
- except Exception as e:
- print(f" [ERROR] Failed to sync {name}")
- return False
-
- global_start = time.time()
-
- with open('manifest.xml', 'r') as f:
- manifest_content = f.read()
-
- root = ET.fromstring(manifest_content)
- top_dir = os.getcwd()
-
- remotes = {r.get('name'): r.get('fetch').rstrip('/') for r in root.findall('remote')}
- default = root.find('default')
- def_remote = default.get('remote') if default is not None else None
- def_rev = default.get('revision') if default is not None else None
-
- sync_tasks = []
- post_process_data = []
-
- for project in root.findall('project'):
- name = project.get('name')
- path = project.get('path', name)
- remote_name = project.get('remote', def_remote)
- rev = project.get('revision', def_rev)
- base_url = remotes.get(remote_name)
-
- if not base_url: continue
-
- if "github.com" in base_url:
- url = f"{base_url}/{name}/archive/{rev}.tar.gz"
- strip = "--strip-components=1"
- elif "googlesource.com" in base_url:
- url = f"{base_url}/{name}/+archive/{rev}.tar.gz"
- strip = ""
- elif "git.codelinaro.org" in base_url:
- url = f"{base_url}/{name}/-/archive/{rev}.tar.gz"
- strip = "--strip-components=1"
- else:
- continue
-
- sync_tasks.append((name, path, url, strip, rev))
-
- for child in project:
- if child.tag in ['linkfile', 'copyfile']:
- post_process_data.append((path, child))
-
- if DEBUG:
- print(f"Starting parallel sync of {len(sync_tasks)} projects...")
- with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
- success_list = list(executor.map(sync_project, sync_tasks))
-
- if not all(success_list):
- print("::error::One or more projects failed to sync!")
- print("::endgroup::")
- exit(1)
-
- print("Processing linkfiles and copyfiles...")
- for path, child in post_process_data:
- src_rel = child.get('src')
- dest_rel = child.get('dest')
- if not src_rel or not dest_rel: continue
-
- src_path = os.path.join(top_dir, path, src_rel)
- dest_path = os.path.join(top_dir, dest_rel)
- os.makedirs(os.path.dirname(dest_path), exist_ok=True)
-
- if child.tag == 'linkfile':
- if os.path.lexists(dest_path): os.remove(dest_path)
- rel_target = os.path.relpath(src_path, os.path.dirname(dest_path))
- os.symlink(rel_target, dest_path)
- print(f" [Link] {dest_rel} -> {src_rel}")
- elif child.tag == 'copyfile':
- shutil.copy2(src_path, dest_path)
- print(f" [Copy] {dest_rel} from {src_rel}")
-
-
- total_duration = time.time() - global_start
- minutes = int(total_duration // 60)
- seconds = total_duration % 60
- print(f"Kernel Sync completed in {minutes}m {seconds:.2f}s")
- print("::endgroup::")
|