action.yml 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386
  1. name: 'Restore Cache'
  2. description: 'Restores cache from GitHub Releases. Supports both single and multiple cache paths/keys/buckets.'
  3. inputs:
  4. cache_path:
  5. description: 'Single path where cache should be restored (use cache_paths for multiple)'
  6. required: false
  7. type: string
  8. cache_paths:
  9. description: 'Multiline list of paths where caches should be restored (one per line)'
  10. required: false
  11. type: string
  12. default: ""
  13. cache_key:
  14. description: 'Primary key to restore cache (use cache_keys for multiple)'
  15. required: false
  16. type: string
  17. cache_keys:
  18. description: 'Multiline list of primary keys to restore caches (one per line, must match cache_paths)'
  19. required: false
  20. type: string
  21. default: ""
  22. restore_keys:
  23. description: 'Multiline list of keys (fallback prefixes)'
  24. required: false
  25. type: string
  26. default: ""
  27. cache_bucket:
  28. description: 'Single tag name for the release (use cache_buckets for multiple)'
  29. required: false
  30. default: 'general-cache'
  31. cache_buckets:
  32. description: 'Multiline list of tag names for releases (one per line, must match cache_paths)'
  33. required: false
  34. type: string
  35. default: ""
  36. compression_level:
  37. description: 'Compression level (0-9). 0 = No compression, 1 = Fast, 19 = Best.'
  38. required: false
  39. type: number
  40. default: 6
  41. outputs:
  42. cache-hit:
  43. description: 'Returns "true" if at least one cache was found and restored, "false" otherwise'
  44. value: ${{ steps.restore.outputs.cache-hit }}
  45. cache-key:
  46. description: 'The key that matched (primary or fallback). Multiple entries joined by comma.'
  47. value: ${{ steps.restore.outputs.cache-key }}
  48. cache-size:
  49. description: 'Size of restored cache in human-readable format. Multiple entries joined by comma.'
  50. value: ${{ steps.restore.outputs.cache-size }}
  51. runs:
  52. using: 'composite'
  53. steps:
  54. - name: Detect and Download Cache
  55. id: restore
  56. shell: bash
  57. working-directory: ${{ github.workspace }}
  58. env:
  59. TARGET_REPO: "${{ github.repository }}"
  60. run: |
  61. set -euo pipefail
  62. # Validate and parse inputs
  63. PATHS_INPUT="${{ inputs.cache_paths }}"
  64. KEYS_INPUT="${{ inputs.cache_keys }}"
  65. BUCKETS_INPUT="${{ inputs.cache_buckets }}"
  66. # If multiple inputs are empty, fall back to single inputs
  67. if [ -z "$PATHS_INPUT" ]; then
  68. PATHS_INPUT="${{ inputs.cache_path }}"
  69. fi
  70. if [ -z "$KEYS_INPUT" ]; then
  71. KEYS_INPUT="${{ inputs.cache_key }}"
  72. fi
  73. if [ -z "$BUCKETS_INPUT" ]; then
  74. BUCKETS_INPUT="${{ inputs.cache_bucket }}"
  75. fi
  76. # Split inputs into arrays
  77. IFS=$'\n' read -rd '' -a PATHS_ARRAY <<<"$PATHS_INPUT" || true
  78. IFS=$'\n' read -rd '' -a KEYS_ARRAY <<<"$KEYS_INPUT" || true
  79. IFS=$'\n' read -rd '' -a BUCKETS_ARRAY <<<"$BUCKETS_INPUT" || true
  80. # Validate array lengths match
  81. if [[ ${#PATHS_ARRAY[@]} -ne ${#KEYS_ARRAY[@]} ]] || [[ ${#PATHS_ARRAY[@]} -ne ${#BUCKETS_ARRAY[@]} ]]; then
  82. echo "::error::Mismatch in array lengths: paths=${#PATHS_ARRAY[@]}, keys=${#KEYS_ARRAY[@]}, buckets=${#BUCKETS_ARRAY[@]}"
  83. exit 1
  84. fi
  85. ARIA2_OPTS=(
  86. "-x16" "-s16" "-k1M" "-j5" "--file-allocation=none"
  87. "--header=User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
  88. "--header=Accept: */*"
  89. "--header=Connection: keep-alive"
  90. )
  91. ARIA2_OPTS+=("--quiet" "--summary-interval=0" "--console-log-level=error")
  92. # Collect results
  93. OVERALL_HIT=false
  94. MATCHED_KEYS=()
  95. CACHE_SIZES=()
  96. # Process each cache entry
  97. for ((idx=0; idx<${#PATHS_ARRAY[@]}; idx++)); do
  98. CACHE_PATH="${PATHS_ARRAY[$idx]}"
  99. CACHE_KEY="${KEYS_ARRAY[$idx]}"
  100. CACHE_BUCKET="${BUCKETS_ARRAY[$idx]}"
  101. [ -z "$CACHE_PATH" ] && continue
  102. TAG_NAME="$CACHE_BUCKET"
  103. BASE_URL="https://github.com/${TARGET_REPO}/releases/download/$TAG_NAME"
  104. ASSETS_URL="https://github.com/${TARGET_REPO}/releases/expanded_assets/$TAG_NAME"
  105. echo "::group:: [$((idx+1))/${#PATHS_ARRAY[@]}] Cache: $CACHE_BUCKET - Fetching Asset List"
  106. HTTP_RESPONSE=$(curl -sL -A "Mozilla/5.0" -w "%{http_code}" "$ASSETS_URL" -o assets.html || echo "404")
  107. if [ "$HTTP_RESPONSE" -eq 200 ]; then
  108. ALL_ASSETS=$(grep -oP "download/$TAG_NAME/\K[^\"' ]+" assets.html | sort -u || echo "")
  109. else
  110. echo "Status $HTTP_RESPONSE: Failed to fetch assets from scraper endpoint."
  111. ALL_ASSETS=""
  112. fi
  113. rm -f assets.html
  114. if [ -z "$ALL_ASSETS" ]; then
  115. echo "No assets found in bucket '$TAG_NAME'. Skipping."
  116. echo "::endgroup::"
  117. continue
  118. fi
  119. echo "Successfully fetched asset list. Has $(echo "$ALL_ASSETS" | wc -l) assets."
  120. echo "::endgroup::"
  121. SEARCH_LIST=$(printf "%s\n%s" "$CACHE_KEY" "${{ inputs.restore_keys }}" | sed '/^$/d')
  122. FOUND=false
  123. while read -r KEY; do
  124. [ -z "$KEY" ] && continue
  125. echo "::group:: [$((idx+1))/${#PATHS_ARRAY[@]}] Searching Prefix: $KEY"
  126. MATCH=$(echo "$ALL_ASSETS" | grep "^cache-$KEY" | sort -r | head -n 1 || true)
  127. if [ -z "$MATCH" ]; then
  128. echo "Not found."
  129. echo "::endgroup::"
  130. continue
  131. fi
  132. BASE_FILENAME=$(echo "$MATCH" | sed -E 's/\.(tzst|tar)(\.part[a-z]{2})?$//')
  133. EXT=$(echo "$MATCH" | grep -oP '\.(tzst|tar)' | head -n 1)
  134. [[ "$MATCH" == *".part"* ]] && IS_SPLIT=true || IS_SPLIT=false
  135. [ "$EXT" = ".tzst" ] && COMPRESSED=true || COMPRESSED=false
  136. FOUND=true
  137. echo "✅ Hit! Restoring $BASE_FILENAME$EXT"
  138. echo "::endgroup::"
  139. echo "::group:: [$((idx+1))/${#PATHS_ARRAY[@]}] Downloading Cache"
  140. download_asset() {
  141. local filename="$1"
  142. local url="$2"
  143. local max_retries=3
  144. local attempt=1
  145. while [ $attempt -le $max_retries ]; do
  146. echo " Attempt $attempt: Downloading $filename..."
  147. # Prefer gh release download (reliable and atomic) when available
  148. rm -f "$filename"
  149. if command -v gh >/dev/null 2>&1; then
  150. if timeout 10m gh release download "$TAG_NAME" --repo "$TARGET_REPO" --pattern "$filename" -D . >/dev/null 2>&1; then
  151. # gh writes the file with the same name into the current directory
  152. if [ -s "$filename" ]; then
  153. return 0
  154. fi
  155. fi
  156. fi
  157. # Fallback to aria2c
  158. if timeout 10m aria2c "${ARIA2_OPTS[@]}" --retry-wait=10 --max-tries=10 -o "$filename" "$url"; then
  159. # ensure file is non-empty
  160. if [ -s "$filename" ]; then
  161. return 0
  162. else
  163. echo " ⚠️ Downloaded file is empty. Retrying..."
  164. rm -f "$filename"
  165. fi
  166. fi
  167. echo " ⚠️ Download failed or timed out. Retrying in 5s..."
  168. sleep 5
  169. attempt=$((attempt + 1))
  170. done
  171. return 1
  172. }
  173. if [ "$IS_SPLIT" = "true" ]; then
  174. for part in {a..z}{a..z}; do
  175. PART_NAME="$BASE_FILENAME$EXT.part$part"
  176. if echo "$ALL_ASSETS" | grep -q "^$PART_NAME$"; then
  177. if ! download_asset "$PART_NAME" "$BASE_URL/$PART_NAME"; then
  178. echo "::error::Failed to download $PART_NAME after multiple retries."
  179. exit 1
  180. fi
  181. else break; fi
  182. done
  183. else
  184. if ! download_asset "$BASE_FILENAME$EXT" "$BASE_URL/$BASE_FILENAME$EXT"; then
  185. echo "::error::Failed to download $BASE_FILENAME$EXT after multiple retries."
  186. exit 1
  187. fi
  188. fi
  189. echo "::endgroup::"
  190. echo "::group:: [$((idx+1))/${#PATHS_ARRAY[@]}] Extracting Cache"
  191. mkdir -p "$CACHE_PATH"
  192. FINAL_ARCHIVE="$BASE_FILENAME$EXT"
  193. EXTRACT_DIR="$(dirname "$CACHE_PATH")"
  194. if [ "$IS_SPLIT" = "true" ]; then
  195. mapfile -t PARTS_ARRAY < <(printf '%s\n' "$FINAL_ARCHIVE".part* | sort)
  196. if [ ${#PARTS_ARRAY[@]} -eq 0 ] || [ ! -e "${PARTS_ARRAY[0]}" ]; then
  197. echo "::error::Split archive parts not found for $FINAL_ARCHIVE"
  198. exit 1
  199. fi
  200. for part_file in "${PARTS_ARRAY[@]}"; do
  201. if [ ! -s "$part_file" ]; then
  202. echo "::error::Split archive part is missing or empty: $part_file"
  203. exit 1
  204. fi
  205. done
  206. # Create a concatenated archive file but keep parts until extraction succeeds
  207. cat "${PARTS_ARRAY[@]}" > "$FINAL_ARCHIVE"
  208. # Attempt to fetch checksum asset (FILENAME.sha256) and verify before extracting
  209. CHECKSUM_FILE="$FINAL_ARCHIVE.sha256"
  210. EXPECTED_SUM=""
  211. if command -v gh >/dev/null 2>&1; then
  212. if gh release view "$TAG_NAME" --repo "$TARGET_REPO" >/dev/null 2>&1; then
  213. if gh release download "$TAG_NAME" --repo "$TARGET_REPO" --pattern "$CHECKSUM_FILE" -D . >/dev/null 2>&1; then
  214. if [ -s "$CHECKSUM_FILE" ]; then
  215. EXPECTED_SUM=$(awk '{print $1}' "$CHECKSUM_FILE" || true)
  216. echo "Found checksum for $FINAL_ARCHIVE: $EXPECTED_SUM"
  217. fi
  218. fi
  219. fi
  220. fi
  221. MAX_EXTRACT_ATTEMPTS=3
  222. attempt=1
  223. extracted=false
  224. while [ $attempt -le $MAX_EXTRACT_ATTEMPTS ]; do
  225. echo "Attempting extraction (attempt $attempt/$MAX_EXTRACT_ATTEMPTS)"
  226. # If we have an expected checksum, verify it
  227. if [ -n "$EXPECTED_SUM" ]; then
  228. if command -v sha256sum >/dev/null 2>&1; then
  229. ACTUAL_SUM=$(sha256sum "$FINAL_ARCHIVE" | awk '{print $1}') || ACTUAL_SUM=""
  230. else
  231. ACTUAL_SUM=$(openssl dgst -sha256 "$FINAL_ARCHIVE" | awk '{print $2}') || ACTUAL_SUM=""
  232. fi
  233. echo " actual=$ACTUAL_SUM expected=$EXPECTED_SUM"
  234. if [ -n "$ACTUAL_SUM" ] && [ "$ACTUAL_SUM" != "$EXPECTED_SUM" ]; then
  235. echo " ⚠️ Checksum mismatch (will re-download parts)"
  236. extracted=false
  237. else
  238. echo " Checksum OK or not provided, proceeding to extract"
  239. extracted=true
  240. fi
  241. else
  242. echo " No checksum available, proceeding to extraction"
  243. extracted=true
  244. fi
  245. if [ "$extracted" = "true" ]; then
  246. if [ "$COMPRESSED" = "true" ]; then
  247. if tar -I "zstd -d -T0" -xf "$FINAL_ARCHIVE" -C "$EXTRACT_DIR"; then
  248. extracted=true
  249. break
  250. else
  251. extracted=false
  252. fi
  253. else
  254. if tar -xf "$FINAL_ARCHIVE" -C "$EXTRACT_DIR"; then
  255. extracted=true
  256. break
  257. else
  258. extracted=false
  259. fi
  260. fi
  261. fi
  262. echo "Extraction failed (possible truncated download or checksum mismatch)."
  263. if [ $attempt -lt $MAX_EXTRACT_ATTEMPTS ]; then
  264. echo "Re-downloading split parts and retrying..."
  265. # remove concatenated before re-download
  266. rm -f "$FINAL_ARCHIVE"
  267. for part_file in "${PARTS_ARRAY[@]}"; do
  268. echo " Re-downloading $part_file"
  269. rm -f "$part_file"
  270. if ! download_asset "$part_file" "$BASE_URL/$part_file"; then
  271. echo "::error::Failed to re-download $part_file"
  272. exit 1
  273. fi
  274. done
  275. # rebuild concatenated archive
  276. cat "${PARTS_ARRAY[@]}" > "$FINAL_ARCHIVE"
  277. # re-fetch checksum if available
  278. if command -v gh >/dev/null 2>&1; then
  279. rm -f "$CHECKSUM_FILE"
  280. gh release download "$TAG_NAME" --repo "$TARGET_REPO" --pattern "$CHECKSUM_FILE" -D . >/dev/null 2>&1 || true
  281. if [ -s "$CHECKSUM_FILE" ]; then
  282. EXPECTED_SUM=$(awk '{print $1}' "$CHECKSUM_FILE" || true)
  283. else
  284. EXPECTED_SUM=""
  285. fi
  286. fi
  287. fi
  288. attempt=$((attempt+1))
  289. done
  290. if [ "$extracted" = "false" ]; then
  291. echo "::error::Extraction failed after $MAX_EXTRACT_ATTEMPTS attempts. Aborting."
  292. exit 1
  293. fi
  294. # cleanup parts and final archive after successful extraction
  295. rm -f "${PARTS_ARRAY[@]}"
  296. rm -f "$FINAL_ARCHIVE"
  297. else
  298. if [ "$COMPRESSED" = "true" ]; then
  299. tar -I "zstd -d -T0" -xf "$FINAL_ARCHIVE" -C "$EXTRACT_DIR"
  300. else
  301. tar -xf "$FINAL_ARCHIVE" -C "$EXTRACT_DIR"
  302. fi
  303. rm -f "$FINAL_ARCHIVE"
  304. fi
  305. echo "✅ Restore complete."
  306. echo "::endgroup::"
  307. # Track results for this entry
  308. OVERALL_HIT=true
  309. MATCHED_KEYS+=("$KEY")
  310. CACHE_SIZE=$(du -sh "$CACHE_PATH" 2>/dev/null | awk '{print $1}' || echo "unknown")
  311. CACHE_SIZES+=("$CACHE_SIZE")
  312. # Provide a concise summary of restored path (top-level + one level deep)
  313. echo "--- Cache restore summary for: $CACHE_PATH ---"
  314. if [ -e "$CACHE_PATH" ]; then
  315. du -sh "$CACHE_PATH" || true
  316. du -h --max-depth=1 "$CACHE_PATH" 2>/dev/null | sort -hr | head -n 10 || true
  317. echo "Showing one-level-deep details for top entries (up to 5 each):"
  318. while read -r _entry; do
  319. entry_path=$(echo "$_entry" | awk '{print $2}')
  320. [ -z "$entry_path" ] && continue
  321. if [ "$entry_path" = "$CACHE_PATH" ]; then
  322. continue
  323. fi
  324. echo "-> $entry_path :"
  325. du -h --max-depth=1 "$entry_path" 2>/dev/null | sort -hr | head -n 5 || true
  326. done < <(du -h --max-depth=1 "$CACHE_PATH" 2>/dev/null | sort -hr | head -n 10)
  327. fi
  328. break
  329. done <<< "$SEARCH_LIST"
  330. if [ "$FOUND" = "false" ]; then
  331. echo "⚠️ No cache matches found for bucket '$CACHE_BUCKET'. Proceeding with fresh run."
  332. MATCHED_KEYS+=("none")
  333. CACHE_SIZES+=("0B")
  334. fi
  335. done
  336. # Output results
  337. if [ "$OVERALL_HIT" = "true" ]; then
  338. echo "cache-hit=true" >> $GITHUB_OUTPUT
  339. else
  340. echo "cache-hit=false" >> $GITHUB_OUTPUT
  341. fi
  342. echo "cache-key=$(IFS=,; echo "${MATCHED_KEYS[*]}")" >> $GITHUB_OUTPUT
  343. echo "cache-size=$(IFS=,; echo "${CACHE_SIZES[*]}")" >> $GITHUB_OUTPUT