Codebase git mirroring

Makefile

@@ -12,7 +12,7 @@ SECRETS_DIR=${BUILD_DIR}/secrets
 DB_PASSWORD_PATH=${SECRETS_DIR}/db_password
 PGPASS_PATH=${SECRETS_DIR}/.pgpass
 SECRET_KEY_PATH=${SECRETS_DIR}/django_secret_key
-EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password
+EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password github_integration_app_private_key github_integration_app_client_secret
 GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH)
 
 ENVREPLACE := deploy/scripts/envreplace

base.yml

@@ -67,6 +67,8 @@ services:
       - django_secret_key
       - github_client_secret
       - orcid_client_secret
+      - github_integration_app_private_key
+      - github_integration_app_client_secret
       - hcaptcha_secret
       - mail_api_key
     volumes:
@@ -98,6 +100,10 @@ secrets:
     file: ./build/secrets/django_secret_key
   github_client_secret:
     file: ./build/secrets/github_client_secret
+  github_integration_app_private_key:
+    file: ./build/secrets/github_integration_app_private_key
+  github_integration_app_client_secret:
+    file: ./build/secrets/github_integration_app_client_secret
   hcaptcha_secret:
     file: ./build/secrets/hcaptcha_secret
   mail_api_key:

deploy/conf/.env.template

@@ -41,6 +41,12 @@ ORCID_CLIENT_ID=
 DATACITE_API_USERNAME=
 DATACITE_DRY_RUN="true" # allowed values: "true" or "false"
 
+# github integration app
+GITHUB_INTEGRATION_APP_ID=
+GITHUB_INTEGRATION_APP_INSTALLATION_ID=
+GITHUB_INTEGRATION_APP_CLIENT_ID=
+GITHUB_MODEL_LIBRARY_ORG_NAME=
+
 # test
 TEST_USER_ID=10000000
 TEST_USERNAME=__test_user__

django/Dockerfile

@@ -40,9 +40,9 @@ RUN --mount=type=cache,target=/var/lib/apt,sharing=locked \
         unzip \
         && update-alternatives --install /usr/bin/python python /usr/bin/python3 1000 \
         && apt-get upgrade -q -y -o Dpkg::Options::="--force-confold" \
-        && mkdir -p /etc/service/django \
-        && touch /etc/service/django/run /etc/postgresql-backup-pre \
-        && chmod a+x /etc/service/django/run /etc/postgresql-backup-pre \
+        && mkdir -p /etc/service/django /etc/service/huey \
+        && touch /etc/service/django/run /etc/service/huey/run /etc/postgresql-backup-pre \
+        && chmod a+x /etc/service/django/run /etc/service/huey/run /etc/postgresql-backup-pre \
         && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 WORKDIR /code
@@ -58,5 +58,13 @@ COPY ./deploy/cron.weekly/* /etc/cron.weekly/
 COPY ./deploy/db/autopostgresqlbackup.conf /etc/default/autopostgresqlbackup
 COPY ./deploy/db/postgresql-backup-pre /etc/
 COPY ${RUN_SCRIPT} /etc/service/django/run
+COPY ./deploy/huey.sh /etc/service/huey/run
 COPY . /code
+
+# FIXME: replace with install from pypi
+# upgrading pip because of some bug with the debian patched version
+RUN python3 -m pip install --upgrade pip
+RUN pip3 install git+https://github.com/sgfost/codemeticulous.git
+RUN pip3 install -r /tmp/requirements.txt
+
 CMD ["/sbin/my_init"]

django/core/huey.py

@@ -0,0 +1,13 @@
+from django_redis import get_redis_connection
+from huey import RedisHuey
+
+
+class DjangoRedisHuey(RedisHuey):
+    """Huey subclass that uses the existing connection pool
+    from the django-redis cache backend
+    """
+
+    def __init__(self, *args, **kwargs):
+        connection = get_redis_connection("default")
+        kwargs["connection_pool"] = connection.connection_pool
+        super().__init__(*args, **kwargs)

django/core/models.py

@@ -464,6 +464,14 @@ def github_url(self):
         """
         return self.get_social_account_profile_url("github")
 
+    @property
+    def github_username(self):
+        github_account = self.get_social_account("github")
+        if github_account:
+            return github_account.extra_data.get("login")
+        else:
+            return None
+
     def get_social_account_profile_url(self, provider_name):
         social_acct = self.get_social_account(provider_name)
         if social_acct:

django/core/serializers.py

@@ -69,10 +69,8 @@ def create(model_cls, validated_data, context):
 
 def update(serializer_update, instance, validated_data):
     tags = TagSerializer(many=True, data=validated_data.pop("tags"))
-    instance = serializer_update(instance, validated_data)
     set_tags(instance, tags)
-    instance.save()
-    return instance
+    return serializer_update(instance, validated_data)
 
 
 class EditableSerializerMixin(serializers.Serializer):

django/core/settings/defaults.py

@@ -118,6 +118,7 @@ def is_test(self):
     "django_extensions",
     "django_vite",
     "guardian",
+    "huey.contrib.djhuey",
     "rest_framework",
     "rest_framework_swagger",
     "robots",
@@ -396,6 +397,11 @@ def is_test(self):
             "handlers": ["console", "comsesfile"],
             "propagate": False,
         },
+        "huey": {
+            "level": "INFO",
+            "handlers": ["console", "comsesfile"],
+            "propagate": False,
+        },
     },
 }
 
@@ -479,10 +485,18 @@ def is_test(self):
         "LOCATION": "unix:///shared/redis/redis.sock",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            "CONNECTION_POOL_KWARGS": {"max_connections": 20},
         },
     }
 }
 
+HUEY = {
+    "name": "comses",
+    "huey_class": "core.huey.DjangoRedisHuey",
+    "immediate": False,  # always run tasks in the background, even in dev (for now)
+    # if removed here, it will default to DEBUG
+}
+
 # SSO, user registration, and django-allauth configuration, see
 # https://django-allauth.readthedocs.io/en/latest/configuration.html
 # ACCOUNT_ADAPTER = 'core.adapter.AccountAdapter'
@@ -501,12 +515,27 @@ def is_test(self):
 ACCOUNT_CHANGE_EMAIL = True
 
 ORCID_CLIENT_ID = os.getenv("ORCID_CLIENT_ID", "")
-
 ORCID_CLIENT_SECRET = read_secret("orcid_client_secret")
-
 GITHUB_CLIENT_ID = os.getenv("GITHUB_CLIENT_ID", "")
 GITHUB_CLIENT_SECRET = read_secret("github_client_secret")
 
+GITHUB_INTEGRATION_APP_ID = int(os.getenv("GITHUB_INTEGRATION_APP_ID") or 0)
+GITHUB_INTEGRATION_APP_PRIVATE_KEY = read_secret("github_integration_app_private_key")
+GITHUB_INTEGRATION_APP_INSTALLATION_ID = int(
+    os.getenv("GITHUB_INTEGRATION_APP_INSTALLATION_ID") or 0
+)
+# client id and secret are only used for getting user access tokens to be able to push
+# to the user's repositories. We are not re-using the regular oauth app in order to
+# keep minimal permissions
+GITHUB_INTEGRATION_APP_CLIENT_ID = os.getenv("GITHUB_INTEGRATION_APP_ID", "")
+GITHUB_INTEGRATION_APP_CLIENT_SECRET = read_secret(
+    "github_integration_app_client_secret"
+)
+GITHUB_MODEL_LIBRARY_ORG_NAME = os.getenv("GITHUB_MODEL_LIBRARY_ORG_NAME", "")
+GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT = os.getenv(
+    "GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT", 100 * 1024 * 1024
+)
+
 TEST_BASIC_AUTH_PASSWORD = os.getenv("TEST_BASIC_AUTH_PASSWORD", "test password")
 TEST_USER_ID = os.getenv("TEST_USER_ID", 1000000)
 TEST_USERNAME = os.getenv("TEST_USERNAME", "__test_user__")

django/core/settings/staging.py

@@ -181,5 +181,10 @@
             "handlers": ["comsesfile"],
             "propagate": False,
         },
+        "huey": {
+            "level": "WARNING",
+            "handlers": ["comsesfile"],
+            "propagate": False,
+        },
     },
 }

django/core/settings/test.py

@@ -18,7 +18,7 @@
 SHARE_DIR = path.realpath("library/tests/tmp")
 LIBRARY_ROOT = path.join(SHARE_DIR, "library")
 LIBRARY_PREVIOUS_ROOT = path.join(SHARE_DIR, ".latest")
-REPOSITORY_ROOT = path.join(BASE_DIR, "repository")
+REPOSITORY_ROOT = path.join(SHARE_DIR, "repository")
 BACKUP_ROOT = path.join(SHARE_DIR, "backups")
 BORG_ROOT = path.join(BACKUP_ROOT, "repo")
 EXTRACT_ROOT = path.join(SHARE_DIR, "extract")

django/core/tests/base.py

@@ -173,5 +173,13 @@ def initialize_test_shared_folders():
     )
 
 
+def clear_test_shared_folder(dir=settings.REPOSITORY_ROOT):
+    for fs in os.scandir(dir):
+        if fs.is_dir():
+            shutil.rmtree(os.path.join(dir, fs.name), ignore_errors=True)
+        elif fs.is_file():
+            os.remove(os.path.join(dir, fs.name))
+
+
 def destroy_test_shared_folders():
     shutil.rmtree(settings.SHARE_DIR, ignore_errors=True)

django/curator/tests/test_dump_restore.py

@@ -19,7 +19,7 @@
 from core.tests.base import EventFactory, JobFactory
 from library.fs import import_archive
 from library.models import Codebase
-from library.tests.base import CodebaseFactory
+from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ def setUp(self):
         fs_api = self.release.get_fs_api()
         import_archive(
             codebase_release=self.release,
-            nested_code_folder_name="library/tests/archives/nestedcode",
+            nested_code_folder_name=TEST_SAMPLES_DIR / "archives" / "nestedcode",
             fs_api=fs_api,
         )
 

django/deploy/huey.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec /code/manage.py run_huey

django/home/apps.py

@@ -17,4 +17,4 @@ def ready(self):
         """
         from . import signals
 
-        logger.debug("fully loaded signals: %s", signals)
+        logger.debug("fully loaded signals: %s for app: %s", signals, self.name)