feat(download-kernel): add metadata tracking and support for additional toolchains

- Add metadata JSON file to track archive parts with SHA-256 checksums
- Support additional toolchain paths including kernel-build-tools, bazel, JDK, NDK, and GCC
- Implement robust asset indexing to prevent redundant uploads
- Improve error handling and caching logic for release assets

.github/actions/download-kernel/download_kernel_archives.py

@@ -15,6 +15,7 @@ from typing import Any
 from urllib.error import HTTPError
 from urllib.request import Request
 from urllib.request import urlopen
+import hashlib
 
 
 def parse_args() -> argparse.Namespace:
@@ -49,7 +50,12 @@ def main() -> int:
         "clang/host/linux-x86": "clang",
         "prebuilts/rust": "rust",
         "prebuilts/clang-tools": "clang-tools",
-        "prebuilts/build-tools": "build-tools",
+        "platform/prebuilts/build-tools": "build-tools",
+        "kernel/prebuilts/build-tools": "kernel-build-tools",
+        "platform/prebuilts/bazel/linux-x86_64": "bazel-linux-x86_64",
+        "platform/prebuilts/jdk/jdk11": "jdk11",
+        "toolchain/prebuilts/ndk/r23": "ndk-r23",
+        "platform/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8": "gcc-glibc2.17-4.8",
     }
 
     release_assets_lock = threading.Lock()
@@ -194,15 +200,17 @@ def main() -> int:
         with release_assets_lock:
             if release_assets_cache is not None:
                 return release_assets_cache
-            if not target_repo:
+
+        if not target_repo:
+            with release_assets_lock:
                 release_assets_cache = []
                 return release_assets_cache
-            url = f"https://api.github.com/repos/{target_repo}/releases?per_page=100"
-            try:
-                releases = github_api_get_json(url)
-                if not isinstance(releases, list):
-                    release_assets_cache = []
-                    return release_assets_cache
+
+        assets: list[dict[str, Any]] = []
+        url = f"https://api.github.com/repos/{target_repo}/releases?per_page=100"
+        try:
+            releases = github_api_get_json(url)
+            if isinstance(releases, list):
                 release = next(
                     (
                         r
@@ -212,15 +220,21 @@ def main() -> int:
                     ),
                     None,
                 )
-                if not release:
-                    release_assets_cache = []
-                    return release_assets_cache
-                assets = release.get("assets", [])
-                release_assets_cache = assets if isinstance(assets, list) else []
-                return release_assets_cache
-            except Exception:
-                release_assets_cache = []
-                return release_assets_cache
+                if isinstance(release, dict):
+                    raw_assets = release.get("assets", [])
+                    assets = raw_assets if isinstance(raw_assets, list) else []
+        except Exception:
+            assets = []
+
+        if not assets and github_token:
+            release = get_or_create_toolchain_release()
+            if isinstance(release, dict):
+                raw_assets = release.get("assets", [])
+                assets = raw_assets if isinstance(raw_assets, list) else []
+
+        with release_assets_lock:
+            release_assets_cache = assets
+            return release_assets_cache
 
     def get_or_create_toolchain_release() -> dict[str, Any] | None:
         nonlocal release_info_cache, release_assets_cache
@@ -345,6 +359,21 @@ def main() -> int:
             if os.path.exists(tmp_resp):
                 os.remove(tmp_resp)
 
+    def build_release_asset_index() -> dict[str, dict[str, Any]]:
+        assets = get_toolchain_release_assets()
+        index: dict[str, dict[str, Any]] = {}
+        for a in assets:
+            if isinstance(a, dict) and isinstance(a.get("name"), str):
+                index[a["name"]] = a
+        return index
+
+    def sha256_file(path: str) -> str:
+        h = hashlib.sha256()
+        with open(path, "rb") as f:
+            for chunk in iter(lambda: f.read(1024 * 1024), b""):
+                h.update(chunk)
+        return h.hexdigest()
+
     def ensure_toolchain_cached(label: str, rev: str, src_dir: str) -> None:
         nonlocal release_assets_cache
         if not github_token or not target_repo:
@@ -360,12 +389,10 @@ def main() -> int:
         with lock:
             try:
                 base_filename = f"{label}-{rev}.tar.gz"
-                assets = get_toolchain_release_assets()
-                for a in assets:
-                    if isinstance(a, dict):
-                        asset_name = a.get("name")
-                        if isinstance(asset_name, str) and asset_name.startswith(base_filename):
-                            return
+                meta_name = f"{label}-{rev}.cache.json"
+                asset_index = build_release_asset_index()
+                if meta_name in asset_index:
+                    return
 
                 release = get_or_create_toolchain_release()
                 if not release:
@@ -386,8 +413,23 @@ def main() -> int:
                     size = os.path.getsize(archive_path)
                     max_part = 1900 * 1024 * 1024
 
+                    meta: dict[str, Any] = {
+                        "format": 1,
+                        "label": label,
+                        "rev": rev,
+                        "archive": base_filename,
+                        "parts": [],
+                    }
+
                     if size <= max_part:
                         upload_release_asset(upload_url_template, archive_path, base_filename)
+                        meta["parts"].append(
+                            {
+                                "name": base_filename,
+                                "size": size,
+                                "sha256": sha256_file(archive_path),
+                            }
+                        )
                     else:
                         subprocess.run(
                             ["split", "-b", str(max_part), "-d", "-a", "2", archive_path, f"{archive_path}.part"],
@@ -398,6 +440,18 @@ def main() -> int:
                             if name.startswith(f"{base_filename}.part"):
                                 part_path = os.path.join(tmp_dir, name)
                                 upload_release_asset(upload_url_template, part_path, name)
+                                meta["parts"].append(
+                                    {
+                                        "name": name,
+                                        "size": os.path.getsize(part_path),
+                                        "sha256": sha256_file(part_path),
+                                    }
+                                )
+
+                    meta_path = os.path.join(tmp_dir, meta_name)
+                    with open(meta_path, "w", encoding="utf-8") as f:
+                        json.dump(meta, f, separators=(",", ":"))
+                    upload_release_asset(upload_url_template, meta_path, meta_name)
 
                     with release_assets_lock:
                         release_assets_cache = None
@@ -411,19 +465,38 @@ def main() -> int:
         if not label:
             return False
 
-        assets = get_toolchain_release_assets()
-        if not assets:
-            return False
+        asset_index = build_release_asset_index()
 
         base_filename = f"{label}-{rev}.tar.gz"
+        meta_name = f"{label}-{rev}.cache.json"
         matching: list[tuple[str, str]] = []
-        for a in assets:
-            if not isinstance(a, dict):
-                continue
-            asset_name = a.get("name")
-            asset_url = a.get("url")
-            if isinstance(asset_name, str) and isinstance(asset_url, str) and asset_name.startswith(base_filename):
-                matching.append((asset_name, asset_url))
+
+        if meta_name in asset_index and isinstance(asset_index[meta_name].get("url"), str):
+            tmp_fd, tmp_meta = tempfile.mkstemp(prefix="toolchain-cache-", suffix=".json")
+            os.close(tmp_fd)
+            try:
+                download_github_release_asset(asset_index[meta_name]["url"], tmp_meta)
+                with open(tmp_meta, "r", encoding="utf-8", errors="replace") as f:
+                    meta = json.load(f)
+                parts = meta.get("parts", [])
+                if not isinstance(parts, list) or not parts:
+                    return False
+                for p in parts:
+                    if not isinstance(p, dict):
+                        continue
+                    name = p.get("name")
+                    if isinstance(name, str) and name in asset_index and isinstance(asset_index[name].get("url"), str):
+                        matching.append((name, asset_index[name]["url"]))
+                if not matching:
+                    return False
+            finally:
+                if os.path.exists(tmp_meta):
+                    os.remove(tmp_meta)
+        else:
+            for asset_name, a in asset_index.items():
+                asset_url = a.get("url")
+                if isinstance(asset_url, str) and asset_name.startswith(base_filename):
+                    matching.append((asset_name, asset_url))
 
         if not matching:
             return False

action.yml

@@ -0,0 +1,237 @@
+name: 'Download and configure Kernel Source code'
+
+inputs:
+  source_location:
+    description: 'Folder path to save Kernel source'
+    required: true
+    type: string
+  github_token:
+    description: 'GitHub Token'
+    required: true
+  debug:
+    description: 'Enable Logs'
+    required: false
+    type: boolean
+    default: false
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Download and Prepare Manifest
+      shell: bash
+      working-directory: ${{ inputs.source_location }}
+      run: |
+        # Download and Prepare Manifest
+        if [[ "$OP_MANIFEST" == https://* ]]; then
+          curl --fail --show-error --location --proto '=https' "$OP_MANIFEST" -o manifest.xml
+        elif [[ "$OP_BRANCH" == wild/* ]]; then
+          cp "../manifests/$(echo "$OP_OS_VERSION" | tr '[:upper:]' '[:lower:]')/$OP_MANIFEST" manifest.xml
+        else
+          curl --fail --show-error --location --proto '=https' "https://raw.githubusercontent.com/OnePlusOSS/kernel_manifest/refs/heads/$OP_BRANCH/$OP_MANIFEST" -o manifest.xml
+        fi
+
+    - name: Download Manifest Archives
+      shell: python
+      env:
+        PYTHONUNBUFFERED: "1"
+        GITHUB_TOKEN: ${{ inputs.github_token }}
+        DEBUG: ${{ inputs.debug }}
+      working-directory: ${{ inputs.source_location }}
+      run: |
+        # Download Manifest Archives
+        import xml.etree.ElementTree as ET
+        import subprocess
+        import os, shutil
+        import time
+        import glob
+        from concurrent.futures import ThreadPoolExecutor
+        import requests
+        
+        MAX_WORKERS = (os.cpu_count() or 2) * 4
+        NPROC = int(subprocess.check_output("nproc", shell=True).strip())
+        TARGET_REPO = "${{ github.repository }}"
+        TOOLCHAIN_MAP = {
+            "clang/host/linux-x86": "clang",
+            "prebuilts/rust": "rust",
+            "prebuilts/clang-tools": "clang-tools",
+            "prebuilts/build-tools": "build-tools"
+        }
+        DEBUG = os.environ.get("DEBUG", "false").lower() == "true"
+        aria_quiet_flags = "--quiet --summary-interval=0" if not DEBUG else ""
+        
+        print("::group::Download Manifest Archives")
+        
+        def get_release_parts(label, rev):
+            url = f"https://api.github.com/repos/{TARGET_REPO}/releases?per_page=100"
+            headers = {
+                "Authorization": f"token {os.environ['GITHUB_TOKEN']}",
+                "Accept": "application/vnd.github.v3+json"
+            }
+            
+            for attempt in range(3):
+                try:
+                    response = requests.get(url, headers=headers)
+                    response.raise_for_status()
+                    releases = response.json()
+                    release = next((r for r in releases if r['name'] == "Toolchains Mirror Cache" or r['tag_name'] == "toolchain-cache"), None)
+                    if not release: return []
+                    
+                    prefix = f"{label}-{rev}.tar.gz"
+                    return [(a['name'], a['url']) for a in release['assets'] if a['name'].startswith(prefix)]
+                except Exception as e:
+                    print(f"  [RETRY {attempt+1}] API failed: {e}")
+                    time.sleep(2)
+            return []
+        
+        def sync_project(task):
+            name, path, url, strip, rev = task
+            if path not in ["./", "."]:
+                os.makedirs(path, exist_ok=True)
+            
+            headers = (
+                "-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' "
+                "-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' "
+                "-H 'Accept-Encoding: gzip, deflate, br' "
+                "-H 'Connection: keep-alive' "
+                "--tcp-fastopen"
+            )
+            
+            print(f"Syncing: {name} -> {path}")
+            start_time = time.time()
+            print(f"  [PENDING] {name}")
+            try:
+                label = None
+                for repo_key, type_label in TOOLCHAIN_MAP.items():
+                    if repo_key in name:
+                        label = type_label
+                        break
+                
+                if label:
+                    base_filename = f"{label}-{rev}.tar.gz"
+                    
+                    asset_data = get_release_parts(label, rev)
+                    
+                    if not asset_data:
+                        print(f"  [ERROR] No assets found for {base_filename} in release!")
+                        return False
+                    
+                    if DEBUG:
+                        print(f"  [CACHE] Fetching {label} toolchain: {base_filename}...")
+                    
+                    for asset_name, api_url in asset_data:
+                        aria_cmd = (
+                            f"aria2c -x16 -s16 -k1M -j5 --file-allocation=none {aria_quiet_flags} "
+                            f"--header='Authorization: token {os.environ['GITHUB_TOKEN']}' "
+                            f"--header='Accept: application/octet-stream' "
+                            f"-o {asset_name} {api_url}"
+                        )
+                        subprocess.run(aria_cmd, shell=True, check=True, capture_output=not DEBUG)
+                    
+                    parts = sorted(glob.glob(f"{base_filename}.part*"))
+                    if parts:
+                        if DEBUG:
+                            print(f"  [MERGE] Combining {len(parts)} parts for {rev}...")
+                        subprocess.run(f"cat {base_filename}.part* | tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M --no-same-owner --no-same-permissions -C {path} {strip}", shell=True, check=True)
+                        subprocess.run(f"rm {base_filename}.part*", shell=True, check=True)
+                    else:
+                        if os.path.exists(base_filename):
+                            if DEBUG:
+                                print(f"  [EXTRACT] Single file detected...")
+                            subprocess.run(f"tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M --no-same-owner --no-same-permissions -f {base_filename} -C {path} {strip}", shell=True, check=True)
+                            os.remove(base_filename)
+                        else:
+                            print(f"  [ERROR] {base_filename} missing after download!")
+                            return False
+                else:
+                    cmd = f"curl -LfsS {headers} --retry 5 --connect-timeout 30 '{url}' | tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M -C {path} {strip}"
+                    subprocess.run(cmd, shell=True, check=True)
+                
+                duration = time.time() - start_time
+                print(f"Synced {name} successfully! ({duration:.2f}s)")
+                return True
+            except subprocess.CalledProcessError as e:
+                print(f"  [ERROR] Command failed for {name}: {e.cmd}")
+                print(f"   Stderr: {e.stderr.decode() if e.stderr else 'No stderr'}")
+                return False
+            except Exception as e:
+                print(f"  [ERROR] Failed to sync {name}")
+                return False
+        
+        global_start = time.time()
+        
+        with open('manifest.xml', 'r') as f:
+            manifest_content = f.read()
+        
+        root = ET.fromstring(manifest_content)
+        top_dir = os.getcwd()
+        
+        remotes = {r.get('name'): r.get('fetch').rstrip('/') for r in root.findall('remote')}
+        default = root.find('default')
+        def_remote = default.get('remote') if default is not None else None
+        def_rev = default.get('revision') if default is not None else None
+        
+        sync_tasks = []
+        post_process_data = []
+        
+        for project in root.findall('project'):
+            name = project.get('name')
+            path = project.get('path', name)
+            remote_name = project.get('remote', def_remote)
+            rev = project.get('revision', def_rev)
+            base_url = remotes.get(remote_name)
+            
+            if not base_url: continue
+            
+            if "github.com" in base_url:
+                url = f"{base_url}/{name}/archive/{rev}.tar.gz"
+                strip = "--strip-components=1"
+            elif "googlesource.com" in base_url:
+                url = f"{base_url}/{name}/+archive/{rev}.tar.gz"
+                strip = ""
+            elif "git.codelinaro.org" in base_url:
+                url = f"{base_url}/{name}/-/archive/{rev}.tar.gz"
+                strip = "--strip-components=1"
+            else:
+                continue
+            
+            sync_tasks.append((name, path, url, strip, rev))
+            
+            for child in project:
+                if child.tag in ['linkfile', 'copyfile']:
+                    post_process_data.append((path, child))
+        
+        if DEBUG:
+            print(f"Starting parallel sync of {len(sync_tasks)} projects...")
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            success_list = list(executor.map(sync_project, sync_tasks))
+            
+            if not all(success_list):
+                print("::error::One or more projects failed to sync!")
+                print("::endgroup::")
+                exit(1)
+        
+        print("Processing linkfiles and copyfiles...")
+        for path, child in post_process_data:
+            src_rel = child.get('src')
+            dest_rel = child.get('dest')
+            if not src_rel or not dest_rel: continue
+            
+            src_path = os.path.join(top_dir, path, src_rel)
+            dest_path = os.path.join(top_dir, dest_rel)
+            os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+            
+            if child.tag == 'linkfile':
+                if os.path.lexists(dest_path): os.remove(dest_path)
+                rel_target = os.path.relpath(src_path, os.path.dirname(dest_path))
+                os.symlink(rel_target, dest_path)
+                print(f"  [Link] {dest_rel} -> {src_rel}")
+            elif child.tag == 'copyfile':
+                shutil.copy2(src_path, dest_path)
+                print(f"  [Copy] {dest_rel} from {src_rel}")
+        
+        
+        total_duration = time.time() - global_start
+        minutes = int(total_duration // 60)
+        seconds = total_duration % 60
+        print(f"Kernel Sync completed in {minutes}m {seconds:.2f}s")
+        print("::endgroup::")