From e70a4f7e1ec2630c2764dbc5a24e2b4221c3eac7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= <jakub@status.im>
Date: Thu, 31 Aug 2023 15:54:41 +0200
Subject: [PATCH] nix: make fetching Gradle dependencies more generic
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Before what we did was essentially guess what files might exist for any
given package. This approach mostly works, but not entirely.
This is especially problematic when dealing with weird edge case
packages like `react-native`, which you can read about here:

https://github.com/react-native-community/discussions-and-proposals/pull/508
https://github.com/react-native-community/discussions-and-proposals/blob/4a06fc64/proposals/0508-out-of-npm-artifacts.md#the-react-native-android-archive

In order to avoid as much the guessing aspect of fetching Gradle
dependencies we are using both HTML listsings of files and
`artifact-metadata.json` files that exist for more recent packages.

This way we can avoid having to add special edge cases that have been
found out when working on React Native 72 upgrade in:
https://github.com/status-im/status-mobile/pull/17062

Signed-off-by: Jakub Sokołowski <jakub@status.im>
---
 nix/deps/gradle/url2json.sh | 61 +++++++++++++++++++++++++++++--------
 nix/shells.nix              |  2 +-
 2 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/nix/deps/gradle/url2json.sh b/nix/deps/gradle/url2json.sh
index b56f3ff41a89..356436b62026 100755
--- a/nix/deps/gradle/url2json.sh
+++ b/nix/deps/gradle/url2json.sh
@@ -14,6 +14,10 @@ declare -a REPOS=(
   "https://jitpack.io"
 )
 
+# Ignore files which filenames contain these elements:
+FILENAMES_BLACKLIST='-(javadoc|sources|src|tests|adapters|bin|api)'
+FILETYPES_BLACKLIST='\.(md5|sha|pom|json|zip|module)$'
+
 function nix_prefetch() {
     nix store prefetch-file --json "${1}" 2>/dev/null
 }
@@ -39,6 +43,47 @@ function pom_has_nodeps_jar() {
         >/dev/null 2>&1 
 }
 
+function guess_pkg_files() {
+    # Some deps are just POMs, in which case there is no JAR to fetch.
+    [[ "${OBJ_TYPE}" == "" ]]        && echo "${PKG_NAME}.jar"
+    [[ "${OBJ_TYPE}" == "jar" ]]     && echo "${PKG_NAME}.jar"
+    [[ "${OBJ_TYPE}" == "bundle" ]]  && echo "${PKG_NAME}.jar"
+    [[ "${OBJ_TYPE}" =~ aar* ]]      && echo "${PKG_NAME}.aar"
+    [[ "${OBJ_TYPE}" == "aar.asc" ]] && echo "${PKG_NAME}.${OBJ_TYPE}"
+    pom_has_nodeps_jar "${POM_PATH}" && echo "${PKG_NAME}-nodeps.jar"
+}
+
+function get_pkg_files() {
+    REPO_URL="${1}"
+    PKG_PATH="${2}"
+    PKG_NAME="${3}"
+    # Google Maven repo doesn't have normal HTML directory listing.
+    if [[ "${REPO_URL}" == "https://dl.google.com/dl/android/maven2" ]]; then
+        FOUND=$(curl --fail -s "${REPO_URL}/${PKG_PATH}/artifact-metadata.json")
+        # Some older packages do not have artifacts-metadata.json.
+        if [[ "$?" -eq 0 ]]; then
+            FOUND=$(echo "${FOUND}" | jq -r '.artifacts[].name')
+        else
+            FOUND=''
+        fi
+    else
+        FOUND=$(
+            curl -s "${REPO_URL}/${PKG_PATH}/" \
+                | htmlq a -a href \
+                | grep -e "^${PKG_NAME}"
+        )
+    fi
+    if [[ "${FOUND}" == '' ]]; then
+        guess_pkg_files
+    else
+        # Filter out files we don't actually need for builds.
+        echo "${FOUND}" \
+            | grep -v -E \
+                -e "${FILENAMES_BLACKLIST}" \
+                -e "${FILETYPES_BLACKLIST}"
+    fi
+}
+
 function fetch_and_template_file() {
     local FILENAME="${1}"
     local OBJ_URL OBJ_NIX_FETCH_OUT OBJ_NAME OBJ_PATH
@@ -58,10 +103,6 @@ function fetch_and_template_file() {
       }"
 }
 
-function fetch_and_template_file_no_fail() {
-    fetch_and_template_file "${1}" 2>/dev/null || true
-}
-
 if [[ -z "${1}" ]]; then
     echo "Required POM URL argument not given!" >&2
     exit 1
@@ -77,7 +118,7 @@ echo -en "${CLR} - Nix entry for: ${1##*/}\r" >&2
 REPO_URL=$(match_repo_url "${PKG_URL_NO_EXT}")
 
 if [[ -z "${REPO_URL}" ]]; then
-    echo " ! Repo URL not found: %s" "${REPO_URL}" >&2
+    echo " ! Repo URL not found for: ${POM_URL}" >&2
     exit 1
 fi
 # Get the relative path without full URL
@@ -114,12 +155,8 @@ echo -ne "
         \"sha256\": \"${POM_SHA256}\"
       }"
 
-# Some deps are just POMs, in which case there is no JAR to fetch.
-[[ "${OBJ_TYPE}" == "" ]]        && fetch_and_template_file_no_fail "${PKG_NAME}.jar"
-[[ "${OBJ_TYPE}" == "jar" ]]     && fetch_and_template_file "${PKG_NAME}.jar"
-[[ "${OBJ_TYPE}" == "bundle" ]]  && fetch_and_template_file "${PKG_NAME}.jar"
-[[ "${OBJ_TYPE}" =~ aar* ]]      && fetch_and_template_file "${PKG_NAME}.aar"
-[[ "${OBJ_TYPE}" == "aar.asc" ]] && fetch_and_template_file "${PKG_NAME}.${OBJ_TYPE}"
-pom_has_nodeps_jar "${POM_PATH}" && fetch_and_template_file "${PKG_NAME}-nodeps.jar"
+for FILE in $(get_pkg_files "${REPO_URL}" "${PKG_PATH}" "${PKG_NAME}"); do
+    fetch_and_template_file "${FILE}"
+done
 
 echo -e '\n    }\n  },'
diff --git a/nix/shells.nix b/nix/shells.nix
index 031ec7da65d3..97fa148119af 100644
--- a/nix/shells.nix
+++ b/nix/shells.nix
@@ -38,7 +38,7 @@ let
 
     # for running gradle by hand
     gradle = mkShell {
-      buildInputs = with pkgs; [ gradle maven goMavenResolver ];
+      buildInputs = with pkgs; [ gradle maven goMavenResolver htmlq ];
       shellHook = ''
         export STATUS_GO_ANDROID_LIBDIR="DUMMY"
         export STATUS_NIX_MAVEN_REPO="${pkgs.deps.gradle}"