Add nextcloud (#96)

* sort and dedupe cspell

* init commit

* update ix_values

* initial changes

* conditions

* try curl

* add configs

* add bits for installing stuff in NC image

* make build expandable

* remove unused directive

* add some todos

* another todoo

* couple more

* add cron

* remove todo

* log failure

* fix healthcheck

* nginx

* fix nginx and its warnings

* better portal

* add  old storage layout support

* add migrations

* fix

* jinja's default :D

* expand description

* cleaner

* update readme

README.md

@@ -21,7 +21,7 @@ Thank you for your understanding.
 | ix-chart             | charts     | -     | -        |
 | minio                | charts     | ✅    | ✅       |
 | netdata              | charts     | ✅    | -        |
-| nextcloud            | charts     | -     | -        |
+| nextcloud            | charts     | ✅    | ✅       |
 | photoprism           | charts     | ✅    | -        |
 | plex                 | charts     | ✅    | ✅       |
 | pihole               | charts     | ✅    | ✅       |

cspell.config.yaml

@@ -13,19 +13,6 @@ words:
   - cifs
   - clamav
   - clamd
-  - consts
-  - cpus
-  - cuda
-  - dnsmasq
-  - fscrawler
-  - graalvm
-  - hexparrot
-  - homarr
-  - freshclamd
-  - hostable
-  - htpasswd
-  - immich
-  - ipaddr
   - cloudflared
   - collabora
   - consts
@@ -51,10 +38,12 @@ words:
   - filebrowser
   - freedns
   - freshclamd
+  - fscrawler
   - gandi
   - godaddy
   - goip
   - goipde
+  - graalvm
   - hetzner
   - hexparrot
   - homarr
@@ -76,6 +65,7 @@ words:
   - jellyseerr
   - kavita
   - komga
+  - libsmbclient
   - lidarr
   - logsearch
   - logseq
@@ -100,13 +90,14 @@ words:
   - nocopy
   - noip
   - nowdns
-  - openj
   - omada
   - opendns
+  - openj
   - openvino
   - organizr
   - overseerr
   - palworld
+  - pecl
   - pgvecto
   - photoprism
   - pihole
@@ -129,6 +120,7 @@ words:
   - servercow
   - shoutrrr
   - sigdb
+  - smbclient
   - sonarr
   - spdyn
   - startpage

ix-dev/stable/nextcloud/README.md

@@ -0,0 +1,3 @@
+# Nextcloud
+
+[Nextcloud](https://nextcloud.com/) is a file sharing server that puts the control and security of your own data back into your hands.

ix-dev/stable/nextcloud/app.yaml

@@ -0,0 +1,70 @@
+app_version: 29.0.4
+capabilities:
+- description: Nextcloud, Nginx and Postgres are able to chown files.
+  name: CHOWN
+- description: Nextcloud, Nginx and Postgres are able to bypass permission checks
+    for it's sub-processes.
+  name: FOWNER
+- description: Nextcloud, Nginx and Postgres are able to bypass permission checks.
+  name: DAC_OVERRIDE
+- description: Nextcloud, Nginx and Postgres are able to set group ID for it's sub-processes.
+  name: SETGID
+- description: Nextcloud, Nginx and Postgres are able to set user ID for it's sub-processes.
+  name: SETUID
+- description: Nextcloud, Nginx and Postgres are able to bind to privileged ports.
+  name: NET_BIND_SERVICE
+- description: Nextcloud, Nginx and Postgres are able to use raw sockets.
+  name: NET_RAW
+categories:
+- productivity
+description: A file sharing server that puts the control and security of your own
+  data back into your hands.
+home: https://nextcloud.com/
+host_mounts: []
+icon: https://media.sys.truenas.net/apps/nextcloud/icons/icon.svg
+keywords:
+- nextcloud
+- storage
+- sync
+- http
+- web
+- php
+lib_version: 1.0.0
+lib_version_hash: 909c14fa5586b5738632ff5ec8489eb26af143ecc601cf199df0b8b9e6d6a10c
+maintainers:
+- email: dev@ixsystems.com
+  name: truenas
+  url: https://www.truenas.com/
+name: nextcloud
+run_as_context:
+- description: Nextcloud runs as root user.
+  gid: 0
+  group_name: root
+  uid: 0
+  user_name: root
+- description: Postgres runs as non-root user.
+  gid: 999
+  group_name: postgres
+  uid: 999
+  user_name: postgres
+- description: Redis runs as non-root user and root group.
+  gid: 1000
+  group_name: redis
+  uid: 0
+  user_name: redis
+- description: Nginx runs as root user.
+  gid: 0
+  group_name: root
+  uid: 0
+  user_name: root
+screenshots:
+- https://media.sys.truenas.net/apps/nextcloud/screenshots/screenshot1.png
+- https://media.sys.truenas.net/apps/nextcloud/screenshots/screenshot2.png
+- https://media.sys.truenas.net/apps/nextcloud/screenshots/screenshot3.png
+sources:
+- https://github.com/nextcloud/docker
+- https://github.com/nextcloud/helm
+- https://github.com/truenas/charts/tree/master/charts/nextcloud
+title: Nextcloud
+train: stable
+version: 1.0.0

ix-dev/stable/nextcloud/item.yaml

@@ -0,0 +1,14 @@
+categories:
+- productivity
+icon_url: https://media.sys.truenas.net/apps/nextcloud/icons/icon.svg
+screenshots:
+- https://media.sys.truenas.net/apps/nextcloud/screenshots/screenshot1.png
+- https://media.sys.truenas.net/apps/nextcloud/screenshots/screenshot2.png
+- https://media.sys.truenas.net/apps/nextcloud/screenshots/screenshot3.png
+tags:
+- nextcloud
+- storage
+- sync
+- http
+- web
+- php

ix-dev/stable/nextcloud/ix_values.yaml

@@ -0,0 +1,35 @@
+images:
+  image:
+    repository: nextcloud
+    tag: 29.0.4
+  nginx_image:
+    repository: nginx
+    tag: 1.25.4
+  postgres_image:
+    repository: postgres
+    tag: 13.1
+  redis_image:
+    repository: redis
+    tag: 7.0.11
+
+consts:
+  nextcloud_container_name: nextcloud
+  cron_container_name: cron
+  perms_container_name: permissions
+  redis_container_name: redis
+  postgres_container_name: postgres
+  nginx_container_name: nginx
+  pg_run_user: 999
+  pg_run_group: 999
+  redis_run_user: 1001
+  redis_run_group: 0
+  db_name: nextcloud
+  ssl_key_path: /etc/nginx-certs/private.key
+  ssl_cert_path: /etc/nginx-certs/public.crt
+
+  packages:
+    smbclient:
+      additional_apt: ["libsmbclient-dev"]
+      pecl: ["smbclient"]
+      docker_php_ext: ["smbclient"]
+      ldd: ["smbclient.so"]

ix-dev/stable/nextcloud/migrations/migrate_from_kubernetes

@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import yaml
+
+from migration_helpers.resources import migrate_resources
+from migration_helpers.dns_config import migrate_dns_config
+from migration_helpers.storage import migrate_storage_item
+from migration_helpers.kubernetes_secrets import get_value_from_secret
+
+
+def migrate(values):
+    config = values.get("helm_secret", {}).get("config", {})
+    k8s_secrets = values.get("release_secrets", {})
+    if not config:
+        raise ValueError("No config found in values")
+
+    new_values = {
+        "nextcloud": {
+            "admin_user": config["ncConfig"]["adminUser"],
+            "admin_password": config["ncConfig"]["adminPassword"],
+            "host": config["ncConfig"].get("host", ""),
+            "data_dir_path": config["ncConfig"]["dataDir"],
+            "apt_packages": config["ncConfig"].get("commands", []),
+            "redis_password": get_value_from_secret(
+                k8s_secrets, "redis-creds", "REDIS_PASSWORD"
+            ),
+            "db_password": get_value_from_secret(
+                k8s_secrets, "postgres-creds", "POSTGRES_PASSWORD"
+            ),
+            "db_user": get_value_from_secret(
+                k8s_secrets, "postgres-creds", "POSTGRES_USER"
+            ),
+            "php_upload_limit": config["ncConfig"]["maxUploadLimit"],
+            "php_memory_limit": config["ncConfig"]["phpMemoryLimit"],
+            "max_execution_time": config["ncConfig"]["maxExecutionTime"],
+            "op_cache_memory_consumption": config["ncConfig"]["opCacheMemoryConsumption"],
+            "cron": {
+                "enabled": config["ncConfig"]["cron"]["enabled"],
+                **(
+                    {"schedule": config["ncConfig"]["cron"]["schedule"]}
+                    if config["ncConfig"]["cron"]["enabled"]
+                    else {}
+                ),
+            },
+            "additional_envs": config["ncConfig"].get("additionalEnvs", []),
+        },
+        "network": {
+            "web_port": config["ncNetwork"].get("webPort", 9001),
+            "certificate_id": config["ncNetwork"].get("certificateID", None),
+            "dns_opts": migrate_dns_config(
+                config.get("podOptions", {}).get("dnsConfig", {})
+            ),
+            "nginx": {
+                "proxy_timeout": config["ncNetwork"]["nginx"]["proxyTimeouts"],
+                "use_different_port": config["ncNetwork"]["nginx"][
+                    "useDifferentAccessPort"
+                ],
+                **(
+                    {
+                        "external_access_port": config["ncNetwork"]["nginx"][
+                            "externalAccessPort"
+                        ]
+                    }
+                    if config["ncNetwork"]["nginx"]["useDifferentAccessPort"]
+                    else {}
+                ),
+            },
+        },
+        "storage": {
+            "is_data_in_the_same_volume": config["ncStorage"].get(
+                "isDataInTheSameVolume", False
+            ),
+            "html": migrate_storage_item(config["ncStorage"]["html"]),
+            "data": migrate_storage_item(config["ncStorage"]["data"]),
+            "postgres_data": migrate_storage_item(config["ncStorage"]["pgData"]),
+            "additional_storage": [
+                migrate_storage_item(item, include_read_only=True)
+                for item in config["ncStorage"]["additionalStorages"]
+            ],
+        },
+        "resources": migrate_resources(config["resources"]),
+    }
+
+    return new_values
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        exit(1)
+
+    if os.path.exists(sys.argv[1]):
+        with open(sys.argv[1], "r") as f:
+            print(yaml.dump(migrate(yaml.safe_load(f.read()))))

ix-dev/stable/nextcloud/migrations/migration_helpers/cpu.py

@@ -0,0 +1,27 @@
+import math
+import re
+import os
+
+CPU_COUNT = os.cpu_count()
+
+NUMBER_REGEX = re.compile(r"^[1-9][0-9]$")
+FLOAT_REGEX = re.compile(r"^[0-9]+\.[0-9]+$")
+MILI_CPU_REGEX = re.compile(r"^[0-9]+m$")
+
+
+def transform_cpu(cpu) -> int:
+    result = 2
+    if NUMBER_REGEX.match(cpu):
+        result = int(cpu)
+    elif FLOAT_REGEX.match(cpu):
+        result = int(math.ceil(float(cpu)))
+    elif MILI_CPU_REGEX.match(cpu):
+        num = int(cpu[:-1])
+        num = num / 1000
+        result = int(math.ceil(num))
+
+    if CPU_COUNT is not None:
+        # Do not exceed the actual CPU count
+        result = min(result, CPU_COUNT)
+
+    return result

ix-dev/stable/nextcloud/migrations/migration_helpers/dns_config.py

@@ -0,0 +1,9 @@
+def migrate_dns_config(dns_config):
+    if not dns_config:
+        return []
+
+    dns_opts = []
+    for opt in dns_config.get("options", []):
+        dns_opts.append(f"{opt['name']}:{opt['value']}")
+
+    return dns_opts

ix-dev/stable/nextcloud/migrations/migration_helpers/kubernetes_secrets.py

@@ -0,0 +1,15 @@
+def get_value_from_secret(secrets={}, secret_name="", key=""):
+    if not secrets or not secret_name or not key:
+        raise ValueError("Expected [secrets], [secret_name] and [key] to be set")
+    for secret in secrets.items():
+        curr_secret_name = secret[0]
+        curr_data = secret[1]
+
+        if curr_secret_name.endswith(secret_name):
+            if not curr_data.get(key, None):
+                raise ValueError(
+                    f"Expected [{key}] to be set in secret [{curr_secret_name}]"
+                )
+            return curr_data[key]
+
+    raise ValueError(f"Secret [{secret_name}] not found")

ix-dev/stable/nextcloud/migrations/migration_helpers/memory.py

@@ -0,0 +1,49 @@
+import re
+import math
+import psutil
+
+TOTAL_MEM = psutil.virtual_memory().total
+
+SINGLE_SUFFIX_REGEX = re.compile(r"^[1-9][0-9]*([EPTGMK])$")
+DOUBLE_SUFFIX_REGEX = re.compile(r"^[1-9][0-9]*([EPTGMK])i$")
+BYTES_INTEGER_REGEX = re.compile(r"^[1-9][0-9]*$")
+EXPONENT_REGEX = re.compile(r"^[1-9][0-9]*e[0-9]+$")
+
+SUFFIX_MULTIPLIERS = {
+    "K": 10**3,
+    "M": 10**6,
+    "G": 10**9,
+    "T": 10**12,
+    "P": 10**15,
+    "E": 10**18,
+}
+
+DOUBLE_SUFFIX_MULTIPLIERS = {
+    "Ki": 2**10,
+    "Mi": 2**20,
+    "Gi": 2**30,
+    "Ti": 2**40,
+    "Pi": 2**50,
+    "Ei": 2**60,
+}
+
+
+def transform_memory(memory):
+    result = 4096  # Default to 4GB
+
+    if re.match(SINGLE_SUFFIX_REGEX, memory):
+        suffix = memory[-1]
+        result = int(memory[:-1]) * SUFFIX_MULTIPLIERS[suffix]
+    elif re.match(DOUBLE_SUFFIX_REGEX, memory):
+        suffix = memory[-2:]
+        result = int(memory[:-2]) * DOUBLE_SUFFIX_MULTIPLIERS[suffix]
+    elif re.match(BYTES_INTEGER_REGEX, memory):
+        result = int(memory)
+    elif re.match(EXPONENT_REGEX, memory):
+        result = int(float(memory))
+
+    result = math.ceil(result)
+    result = min(result, TOTAL_MEM)
+    # Convert to Megabytes
+    result = result / 1024 / 1024
+    return int(result)