Merge branch 'main' into PHP-8-1

addons/locate_mirror.sh

@@ -10,12 +10,6 @@ source /var/scripts/fetch_lib.sh
 # Must be root
 root_check
 
-# Use another method if the new one doesn't work
-if [ -z "$REPO" ]
-then
-    REPO=$(apt-get update -q4 && apt-cache policy | grep http | tail -1 | awk '{print $2}')
-fi
-
 # Check where the best mirrors are and update
 msg_box "To make downloads as fast as possible when updating Ubuntu \
 you should download mirrors that are as geographically close to you as possible.
@@ -24,34 +18,43 @@ Please note that there are no guarantees that the download mirrors \
 this script finds will remain for the lifetime of this server.
 Because of this, we don't recommend that you change the mirror unless you live far away from the default.
 
-This is the method used: https://github.com/jblakeman/apt-select"
-msg_box "Your current server repository is: $REPO"
+This is the method used: https://github.com/vegardit/fast-apt-mirror.sh"
+
+# Install
+install_if_not bash
+install_if_not curl
+install_if_not apt-transport-https
+install_if_not ca-certificates
+curl_to_dir https://raw.githubusercontent.com/vegardit/fast-apt-mirror.sh/v1/ fast-apt-mirror.sh /usr/local/bin
+mv /usr/local/bin/fast-apt-mirror.sh /usr/local/bin/fast-apt-mirror
+chmod 755 /usr/local/bin/fast-apt-mirror
 
+# Variables
+CURRENT_MIRROR=$(fast-apt-mirror current)
+FIND_MIRROR=$(fast-apt-mirror find -v --healthchecks 100)
+msg_box "Current mirror is $CURRENT_MIRROR"
+
+# Ask
 if ! yesno_box_no "Do you want to try to find a better mirror?"
 then
-    print_text_in_color "$ICyan" "Keeping $REPO as mirror..."
+    print_text_in_color "$ICyan" "Keeping $CURRENT_MIRROR as mirror..."
     sleep 1
 else
-    if [[ "$KEYBOARD_LAYOUT" =~ ,|/|_ ]]
-    then
-        msg_box "Your keymap contains more than one language, or a special character. ($KEYBOARD_LAYOUT)
-This script can only handle one keymap at the time.\nThe default mirror ($REPO) will be kept."
-        exit 1
-    fi
+    # Find
     print_text_in_color "$ICyan" "Locating the best mirrors..."
-    curl_to_dir https://bootstrap.pypa.io get-pip.py /tmp
-    install_if_not python3
-    install_if_not python3-testresources
-    install_if_not python3-distutils
-    cd /tmp && python3 get-pip.py
-    pip install \
-        --upgrade pip \
-        apt-select
-    check_command apt-select -m up-to-date -t 4 -c -C "$KEYBOARD_LAYOUT"
-    sudo cp /etc/apt/sources.list /etc/apt/sources.list.backup && \
-    if [ -f sources.list ]
+    if [ "$CURRENT_MIRROR/" != "$FIND_MIRROR" ]
     then
-        sudo mv sources.list /etc/apt/
+        if yesno_box_yes "Do you want to replace the $CURRENT_MIRROR with $FIND_MIRROR?"
+        then
+            # Backup
+            cp -f /etc/apt/sources.list /etc/apt/sources.list.backup
+            # Replace
+            if fast-apt-mirror current --apply # TODO is fast-apt-mirror.sh set better here?
+            then
+                msg_box "Your Ubuntu repo was successfully changed to $FASTEST_MIRROR"
+            fi
+        fi
+    else
+        msg_box "You already have the fastest mirror available, congrats!"
     fi
-    msg_box "The apt-mirror was successfully changed."
 fi

lib.sh

@@ -134,6 +134,10 @@ nc_update() {
     NCBAD=$((NCMAJOR-2))
     NCNEXT="$((${CURRENTVERSION%%.*}+1))"
 }
+maxmind_geoip() {
+    # shellcheck source=/dev/null
+    source <(curl -sL https://shortio.hanssonit.se/t3vm7ro4CP)
+}
 # Set the hour for automatic updates. This would be 18:00 as only the hour is configurable.
 AUT_UPDATES_TIME="18"
 # Keys
@@ -149,6 +153,9 @@ HTTP_CONF="nextcloud_http_domain_self_signed.conf"
 # Collabora App
 HTTPS_CONF="$SITES_AVAILABLE/$SUBDOMAIN.conf"
 HTTP2_CONF="/etc/apache2/mods-available/http2.conf"
+# GeoBlock
+GEOBLOCK_MOD_CONF="/etc/apache2/conf-available/geoblock.conf"
+GEOBLOCK_MOD="/etc/apache2/mods-available/maxminddb.load"
 # PHP-FPM
 PHPVER=8.3
 PHP_FPM_DIR=/etc/php/$PHPVER/fpm
@@ -179,7 +186,7 @@ fulltextsearch_install() {
     FULLTEXTSEARCH_SERVICE=nextcloud-fulltext-elasticsearch-worker.service
     # Supports 0-9.0-99.0-9. Max supprted version with this function is 9.99.9. When ES 10.0.0 is out we have a problem.
     # Maybe "10\\.[[:digit:]][[:digit:]]\\.[[:digit:]]" will work?
-    FULLTEXTSEARCH_IMAGE_NAME_LATEST_TAG="$(curl -s -m 900 https://www.docker.elastic.co/r/elasticsearch | grep -Eo "[[:digit:]]\\.[[:digit:]][[:digit:]]\\.[[:digit:]]" | sort --version-sort | tail -1)"
+    FULLTEXTSEARCH_IMAGE_NAME_LATEST_TAG="$(curl -s -m 900 https://www.docker.elastic.co/r/elasticsearch?limit=500 | grep -Eo "[[:digit:]]\\.[[:digit:]][[:digit:]]\\.[[:digit:]]" | sort --version-sort | tail -1)"
     # Legacy, changed 2023-09-21
     DOCKER_IMAGE_NAME=es01
     # Legacy, not used at all
@@ -394,55 +401,31 @@ curl "https://api.metadefender.com/v4/hash/$hash" -H "apikey: $apikey"
 }
 
 # Used in geoblock.sh
-download_geoip_dat() {
-# 1 = IP version 4 or 6
-# 2 = v4 or v6
-if site_200 https://dl.miyuru.lk/geoip/maxmind/country/maxmind"$1".dat.gz
-then
-    curl_to_dir https://dl.miyuru.lk/geoip/maxmind/country maxmind"$1".dat.gz /tmp
-    # Scan file for virus
-    if ! metadefender-scan /tmp/maxmind"$1".dat.gz | grep '"scan_all_result_a":"No Threat Detected","current_av_result_a":"No Threat Detected"'
+download_geoip_mmdb() {
+    maxmind_geoip
+    export MwKfcYATm43NMT
+    export i9HL69SLnp4ymy 
+    {
+    echo "GEOIPUPDATE_ACCOUNT_ID=$MwKfcYATm43NMT"
+    echo "GEOIPUPDATE_LICENSE_KEY=$i9HL69SLnp4ymy"
+    echo "GEOIPUPDATE_EDITION_IDS=GeoLite2-City GeoLite2-Country"
+    echo "GEOIPUPDATE_FREQUENCY=0"
+    echo "GEOIPUPDATE_PRESERVE_FILE_TIMES=1"
+    echo "GEOIPUPDATE_VERBOSE=1"
+    } > /tmp/dockerenv
+    unset MwKfcYATm43NMT
+    unset i9HL69SLnp4ymy 
+    install_docker
+    if docker run --name maxmind --env-file /tmp/dockerenv -v /usr/share/GeoIP:/usr/share/GeoIP ghcr.io/maxmind/geoipupdate
     then
-        msg_box "Potential threat found in /tmp/maxmind$1.dat.gz! Please report this to $ISSUES. We will now delete the file!"
-        rm -f /tmp/maxmind"$1".dat.gz
+        docker rm -f maxmind
+        rm -f /tmp/dockerenv
     else
-        install_if_not gzip
-        gzip -d /tmp/maxmind"$1".dat.gz
-        mv /tmp/maxmind"$1".dat /usr/share/GeoIP/GeoIP"$2".dat
-        chown root:root /usr/share/GeoIP/GeoIP"$2".dat
-        chmod 644 /usr/share/GeoIP/GeoIP"$2".dat
-        find "$SCRIPTS" -type f -regex "$SCRIPTS/202[0-9]-[01][0-9]-Maxmind-Country-IP$2\.dat" -delete
-        rm -f /usr/share/GeoIP/GeoIP.dat
-    fi
-fi
-}
-
-get_newest_dat_files() {
-# Check current month and year
-CURR_MONTH="$(date +%B)"
-# https://stackoverflow.com/a/12487455
-CURR_MONTH="${CURR_MONTH^}"
-CURR_YEAR="$(date +%Y)"
-
-# Check latest updated
-if site_200 https://www.miyuru.lk/geoiplegacy
-then
-    if curl -s https://www.miyuru.lk/geoiplegacy | grep -q "$CURR_MONTH $CURR_YEAR"
-    then
-        # DIFF local file with month from curl
-        # This is to know if the online file is the same month as the local file
-        LOCAL_FILE_TIMESTAMP=$(date -r /usr/share/GeoIP/GeoIPv4.dat "+%B %Y")
-        LOCAL_FILE_TIMESTAMP="${LOCAL_FILE_TIMESTAMP^}"
-        ONLINE_FILE_TIMESTAMP="$CURR_MONTH $CURR_YEAR"
-        if [ "$ONLINE_FILE_TIMESTAMP" != "$LOCAL_FILE_TIMESTAMP" ]
-        then
-            # IPv4
-            download_geoip_dat "4" "v4"
-            # IPv6
-            download_geoip_dat "6" "v6"
-        fi
+        docker rm -f maxmind
+        rm -f /tmp/dockerenv
+        msg_box "Update limit for Maxmind GeoDatabase reached! Please try again tomorrow."
+        return 1
     fi
-fi
 }
 
 # Check if process is runnnig: is_process_running dpkg
@@ -1503,7 +1486,7 @@ any_key() {
 
 lowest_compatible_nc() {
 # .ocdata needs to exist to be able to check version, occ relies on everytihgn working
-until [ -f "$NCDATA"/.ocdata ]
+until [ -f "$NCDATA"/.ocdata ] || [ -f "$NCDATA"/.ncdata ]
 do
         # SUPPORT LEGACY: If it's not in the standard path, check for existing datadir in config.php
         if [ -f "$NCPATH"/config/config.php ]
@@ -1516,7 +1499,7 @@ do
 If you think this is a bug, please report it to $ISSUES"
             else
                 # Check again an break if found
-                if [ -f "$NCDATA"/.ocdata ]
+                if [ -f "$NCDATA"/.ocdata ] || [ -f "$NCDATA"/.ncdata ] 
                 then
                     break
                 fi

network/geoblock.sh

@@ -11,6 +11,7 @@ Geoblock can break the certificate renewal via \"Let's encrypt!\" if done too st
 If you have problems with \"Let's encrypt!\", please uninstall geoblock first to see if that fixes those issues!"
 # shellcheck source=lib.sh
 source /var/scripts/fetch_lib.sh
+# source <(curl -sL https://raw.githubusercontent.com/nextcloud/vm/geoblock-v2/lib.sh) # TODO, remove after testing
 
 # Check for errors + debug code and abort if something isn't right
 # 1 = ON
@@ -22,41 +23,89 @@ debug_mode
 root_check
 
 # Check if it is already configured
-if ! grep -q "^#Geoip-block" /etc/apache2/apache2.conf
+if [ ! -f "$GEOBLOCK_MOD_CONF" ] || [ ! -f "$GEOBLOCK_MOD" ]
 then
     # Ask for installing
     install_popup "$SCRIPT_NAME"
 else
     # Ask for removal or reinstallation
     reinstall_remove_menu "$SCRIPT_NAME"
-    # Removal
+    # Remove Apache mod config
+    rm -f "$GEOBLOCK_MOD_CONF"
+    # Remove old database files
     find /var/scripts -type f -regex \
 "$SCRIPTS/202[0-9]-[01][0-9]-Maxmind-Country-IPv[46]\.dat" -delete
+    # Remove Apache2 mod
+    if [ -f "$GEOBLOCK_MOD" ]
+    then
+        a2dismod maxminddb
+        rm -f "$GEOBLOCK_MOD"
+        rm -f /usr/lib/apache2/modules/mod_maxminddb.so
+    fi
     if is_this_installed libapache2-mod-geoip
     then
         a2dismod geoip
         apt-get purge libapache2-mod-geoip -y
-        rm -rf /usr/share/GeoIP
     fi
-    apt-get autoremove -y
-    sed -i "/^#Geoip-block-start/,/^#Geoip-block-end/d" /etc/apache2/apache2.conf
-    check_command systemctl restart apache2
+    # Remove PPA
+    if grep ^ /etc/apt/sources.list /etc/apt/sources.list.d/* | grep maxmind-ubuntu-ppa
+    then
+        install_if_not ppa-purge
+        yes | ppa-purge maxmind/ppa
+        rm -f /etc/apt/sources.list.d/maxmind*
+    fi
+    # Remove  Apache config
+    if grep "Geoip-block-start" /etc/apache2/apache2.conf
+    then
+        sed -i "/^#Geoip-block-start/,/^#Geoip-block-end/d" /etc/apache2/apache2.conf
+    fi
+    if [ -f "$GEOBLOCK_MOD_CONF" ]
+    then
+        a2disconf geoblock
+        rm -f "$GEOBLOCK_MOD_CONF"
+    fi
     # Show successful uninstall if applicable
     removal_popup "$SCRIPT_NAME"
+    # Make sure it's clean from unused packages and files
+    apt purge libmaxminddb0* libmaxminddb-dev* mmdb-bin* apache2-dev* -y
+    apt autoremove -y
+    #rm -rf /usr/share/GeoIP keep these to save downloads...
+    check_command systemctl restart apache2
 fi
 
-# Install needed tools
-install_if_not libapache2-mod-geoip
+# Download GeoIP Databases
+if ! download_geoip_mmdb
+then
+   exit 1
+fi
 
-# Enable apache mod
-check_command a2enmod geoip rewrite
-check_command systemctl restart apache2
+##### GeoIP script (Apache Setup)
+# Install  requirements
+yes | add-apt-repository ppa:maxmind/ppa
+install_if_not libmaxminddb0
+install_if_not libmaxminddb-dev
+install_if_not mmdb-bin
+install_if_not apache2-dev
 
-# Download newest dat files
-# IPv4
-download_geoip_dat "4" "v4"
-# IPv6
-download_geoip_dat "6" "v6"
+# maxminddb_module https://github.com/maxmind/mod_maxminddb
+cd /tmp
+curl_to_dir https://github.com/maxmind/mod_maxminddb/releases/download/1.2.0/ mod_maxminddb-1.2.0.tar.gz /tmp
+tar -xzf mod_maxminddb-1.2.0.tar.gz
+cd mod_maxminddb-1.2.0
+if ./configure
+then
+    make install
+    if ! apachectl -M | grep -i "maxminddb"
+    then
+       msg_box "Couldn't install the Apache module for MaxMind. Please report this to $ISSUES"
+       exit 1
+    fi
+    # Cleanup
+    rm -rf mod_maxminddb-1.2.0 mod_maxminddb-1.2.0.tar.gz
+fi
+
+check_command a2enmod rewrite remoteip maxminddb
+check_command systemctl restart apache2
 
 # Restrict to countries and/or continents
 choice=$(whiptail --title "$TITLE"  --checklist \
@@ -160,24 +209,35 @@ then
     mapfile -t choice <<< "$choice"
 fi
 
-GEOIP_CONF="#Geoip-block-start - Please don't remove or change this line
-<IfModule mod_geoip.c>
-  GeoIPEnable On
-  GeoIPDBFile /usr/share/GeoIP/GeoIPv4.dat
-  GeoIPDBFile /usr/share/GeoIP/GeoIPv6.dat
+# Create conff
+cat << GEOBLOCKCONF_CREATE > "$GEOBLOCK_MOD_CONF"
+<IfModule mod_maxminddb.c>
+  MaxMindDBEnable On
+  MaxMindDBFile DB /usr/share/GeoIP/GeoLite2-Country.mmdb
+
+  MaxMindDBEnv MM_CONTINENT_CODE DB/continent/code
+  MaxMindDBEnv MM_COUNTRY_CODE DB/country/iso_code
 </IfModule>
-<Location />\n"
+
+  # Geoblock rules
+GEOBLOCKCONF_CREATE
+
+# Add <Location> parameters to maxmind conf
+echo "<Location />" >> "$GEOBLOCK_MOD_CONF"
 for continent in "${choice[@]}"
 do
-    GEOIP_CONF+="  SetEnvIf GEOIP_CONTINENT_CODE    $continent AllowCountryOrContinent\n"
-    GEOIP_CONF+="  SetEnvIf GEOIP_CONTINENT_CODE_V6 $continent AllowCountryOrContinent\n"
+    echo "  SetEnvIf MM_CONTINENT_CODE    $continent AllowCountryOrContinent" >> "$GEOBLOCK_MOD_CONF"
 done
 for country in "${selected_options[@]}"
 do
-    GEOIP_CONF+="  SetEnvIf GEOIP_COUNTRY_CODE    $country AllowCountryOrContinent\n"
-    GEOIP_CONF+="  SetEnvIf GEOIP_COUNTRY_CODE_V6 $country AllowCountryOrContinent\n"
+    echo "  SetEnvIf MM_COUNTRY_CODE    $country AllowCountryOrContinent" >> "$GEOBLOCK_MOD_CONF"
 done
-GEOIP_CONF+="  Allow from env=AllowCountryOrContinent
+echo "  Allow from env=AllowCountryOrContinent" >> "$GEOBLOCK_MOD_CONF"
+
+# Add allow rules to maxmind conf
+cat << GEOBLOCKALLOW_CREATE >> "$GEOBLOCK_MOD_CONF"
+
+  # Specifically allow this
   Allow from 127.0.0.1/8
   Allow from 192.168.0.0/16
   Allow from 172.16.0.0/12
@@ -188,13 +248,18 @@ GEOIP_CONF+="  Allow from env=AllowCountryOrContinent
   Order Deny,Allow
   Deny from all
 </Location>
-#Geoip-block-end - Please don't remove or change this line"
 
-# Write everything to the file
-echo -e "$GEOIP_CONF" >> /etc/apache2/apache2.conf
-
-check_command systemctl restart apache2
+  # Logs
+  LogLevel info
+  CustomLog "$VMLOGS/geoblock_access.log" common
+GEOBLOCKALLOW_CREATE
 
-msg_box "GeoBlock was successfully configured"
+# Enable config
+check_command a2enconf geoblock
 
-exit
+if check_command systemctl restart apache2
+then
+    msg_box "GeoBlock was successfully configured"
+else
+    msg_box "Something went wrong, please check Apache error logs."
+fi