-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #6 from ckan/solr-9
Build images for Solr 9
- Loading branch information
Showing
11 changed files
with
245 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| name: CKAN 2.10 with Solr 8 | ||
|
|
||
| on: push | ||
|
|
||
| jobs: | ||
| call-reusable-workflow: | ||
| uses: ./.github/workflows/reusable.yml | ||
| with: | ||
| ckan_version: "2.10" | ||
| ckan_branch: "dev-v2.10" | ||
| solr_version: "8" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| name: CKAN 2.10 with Solr 9 | ||
|
|
||
| on: push | ||
|
|
||
| jobs: | ||
| call-reusable-workflow: | ||
| uses: ./.github/workflows/reusable.yml | ||
| with: | ||
| ckan_version: "2.10" | ||
| ckan_branch: "dev-v2.10" | ||
| solr_version: "9" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| name: CKAN 2.9 with Solr 8 | ||
|
|
||
| on: push | ||
|
|
||
| jobs: | ||
| call-reusable-workflow: | ||
| uses: ./.github/workflows/reusable.yml | ||
| with: | ||
| ckan_version: "2.9" | ||
| ckan_branch: "dev-v2.9" | ||
| solr_version: "8" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| name: CKAN 2.9 with Solr 9 | ||
|
|
||
| on: push | ||
|
|
||
| jobs: | ||
| call-reusable-workflow: | ||
| uses: ./.github/workflows/reusable.yml | ||
| with: | ||
| ckan_version: "2.9" | ||
| ckan_branch: "dev-v2.9" | ||
| solr_version: "9" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| name: Reusable CKAN Test workflow | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| ckan_version: | ||
| required: true | ||
| type: string | ||
| ckan_branch: | ||
| required: true | ||
| type: string | ||
| solr_version: | ||
| required: true | ||
| type: string | ||
|
|
||
|
|
||
| jobs: | ||
| build: | ||
| runs-on: ubuntu-20.04 | ||
| services: | ||
| postgres: | ||
| image: ckan/ckan-postgres-dev:master | ||
| env: | ||
| POSTGRES_USER: postgres | ||
| POSTGRES_PASSWORD: postgres | ||
| POSTGRES_DB: postgres | ||
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | ||
| ports: | ||
| - 5432:5432 | ||
| redis: | ||
| image: redis:3 | ||
| ports: | ||
| - 6379:6379 | ||
| env: | ||
| # Given that CKAN is executed in the runner, we need to map service's port and set URL to localhost. | ||
| # We are avoiding the CKAN_ patterns since test should be run in strict mode | ||
| SQLALCHEMY_URL: postgresql://ckan_default:pass@localhost/ckan_test | ||
| DATASTORE_WRITE_URL: postgresql://datastore_write:pass@localhost/datastore_test | ||
| DATASTORE_READ_URL: postgresql://datastore_read:pass@localhost/datastore_test | ||
| SOLR_URL: http://localhost:8983/solr/ckan | ||
| REDIS_URL: redis://localhost:6379/1 | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Build Solr image | ||
| run: | | ||
| cd solr-${{ inputs.solr_version }} | ||
| make build CKAN_VERSION=${{ inputs.ckan_version }} | ||
| docker run --name ckan-solr -p 8983:8983 -d ckan/ckan-solr:${{ inputs.ckan_version }}-solr${{ inputs.solr_version }} | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: 3.9 | ||
| - name: Clone CKAN repositry | ||
| run: git clone https://github.com/ckan/ckan.git | ||
| - name: Checkout CKAN ${{ inputs.ckan_branch }} | ||
| run: | | ||
| cd ckan | ||
| git checkout ${{ inputs.ckan_branch }} | ||
| - name: Install requirements-setuptools.txt (CKAN 2.9 only) | ||
| if: ${{ inputs.ckan_branch == 'dev-v2.9'}} | ||
| run: | | ||
| cd ckan | ||
| pip install -r requirement-setuptools.txt | ||
| pip install wheel | ||
| - name: Install requirements and CKAN | ||
| run: | | ||
| cd ckan | ||
| pip install -U pip | ||
| pip install -r requirements.txt | ||
| pip install -r dev-requirements.txt | ||
| # Change to `pip install .` when #7221 is merged | ||
| pip install -e . | ||
| pip check | ||
| - name: Set config file | ||
| run: | | ||
| cd ckan | ||
| ckan config-tool test-core.ini sqlalchemy.url=${SQLALCHEMY_URL} | ||
| ckan config-tool test-core.ini ckan.datastore.write_url=${DATASTORE_WRITE_URL} | ||
| ckan config-tool test-core.ini ckan.datastore.read_url=${DATASTORE_READ_URL} | ||
| ckan config-tool test-core.ini solr_url=${SOLR_URL} | ||
| ckan config-tool test-core.ini ckan.redis.url=${REDIS_URL} | ||
| - name: Execute tests | ||
| run: | | ||
| cd ckan | ||
| ckan -c test-core.ini db init | ||
| pytest --ckan-ini=test-core.ini ckan/tests ckanext |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| FROM solr:9 | ||
|
|
||
| EXPOSE 8983 | ||
|
|
||
| ARG CKAN_BRANCH="dev-v2.10" | ||
|
|
||
| ENV SOLR_CONFIG_DIR="/opt/solr/server/solr/configsets" | ||
| ENV SOLR_SCHEMA_FILE="$SOLR_CONFIG_DIR/ckan/conf/managed-schema" | ||
|
|
||
| USER root | ||
|
|
||
| # Create a CKAN configset by copying the default one | ||
| RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan | ||
|
|
||
| # Update the schema | ||
| ADD https://raw.githubusercontent.com/ckan/ckan/$CKAN_BRANCH/ckan/config/solr/schema.xml $SOLR_SCHEMA_FILE | ||
| RUN chmod 644 $SOLR_SCHEMA_FILE | ||
|
|
||
| USER solr | ||
|
|
||
| CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| FROM solr:9 | ||
|
|
||
| EXPOSE 8983 | ||
|
|
||
| ARG CKAN_BRANCH="dev-v2.10" | ||
|
|
||
| ENV SOLR_INSTALL="/opt/solr/" | ||
| ENV SOLR_CONFIG_DIR="$SOLR_INSTALL/server/solr/configsets" | ||
| ENV SOLR_SCHEMA_FILE="$SOLR_CONFIG_DIR/ckan/conf/managed-schema" | ||
|
|
||
| ARG JTS_VERSION="1.19.0" | ||
| ARG JTS_JAR_FILE="$SOLR_INSTALL/server/solr-webapp/webapp/WEB-INF/lib/jts-core-$JTS_VERSION.jar" | ||
|
|
||
| USER root | ||
|
|
||
| # Create a CKAN configset by copying the default one | ||
| RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan | ||
|
|
||
| # Update the schema | ||
| ADD https://raw.githubusercontent.com/ckan/ckan/$CKAN_BRANCH/ckan/config/solr/schema.xml $SOLR_SCHEMA_FILE | ||
|
|
||
| # Install JTS JAR file | ||
| ADD https://repo1.maven.org/maven2/org/locationtech/jts/jts-core/$JTS_VERSION/jts-core-$JTS_VERSION.jar \ | ||
| $JTS_JAR_FILE | ||
| RUN chmod 644 $JTS_JAR_FILE | ||
|
|
||
| # Add the spatial field type definitions and fields | ||
|
|
||
| ## RPT | ||
| ENV SOLR_RPT_FIELD_DEFINITION '<fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType" \ | ||
| spatialContextFactory="JTS" \ | ||
| autoIndex="true" \ | ||
| validationRule="repairBuffer0" \ | ||
| distErrPct="0.025" \ | ||
| maxDistErr="0.001" \ | ||
| distanceUnits="kilometers" />' | ||
|
|
||
| ENV SOLR_RPT_FIELD '<field name="spatial_geom" type="location_rpt" indexed="true" multiValued="true" />' | ||
|
|
||
| RUN sed -i "/<types>/a $SOLR_RPT_FIELD_DEFINITION" $SOLR_SCHEMA_FILE | ||
| RUN sed -i "/<fields>/a $SOLR_RPT_FIELD" $SOLR_SCHEMA_FILE | ||
|
|
||
| ## BBox | ||
| ENV SOLR_BBOX_FIELDS '<field name="bbox_area" type="float" indexed="true" stored="true" /> \ | ||
| <field name="maxx" type="float" indexed="true" stored="true" /> \ | ||
| <field name="maxy" type="float" indexed="true" stored="true" /> \ | ||
| <field name="minx" type="float" indexed="true" stored="true" /> \ | ||
| <field name="miny" type="float" indexed="true" stored="true" />' | ||
|
|
||
| RUN sed -i "/<fields>/a $SOLR_BBOX_FIELDS" $SOLR_SCHEMA_FILE | ||
|
|
||
|
|
||
| RUN chmod 644 $SOLR_SCHEMA_FILE | ||
|
|
||
| USER solr | ||
|
|
||
| CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| .PHONY: all help build build-all push | ||
| SHELL := /bin/bash | ||
| CKAN_VERSION=2.10 | ||
| TAG_NAME="ckan/ckan-solr:$(CKAN_VERSION)-solr9" | ||
| TAG_NAME_SPATIAL="ckan/ckan-solr:$(CKAN_VERSION)-solr9-spatial" | ||
|
|
||
| all: help | ||
| help: | ||
| @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' | ||
|
|
||
|
|
||
| build: ## Build a Solr 9 image for a specific CKAN version, `make build CKAN_VERSION=2.9` (default is 2.10) | ||
| docker build -t $(TAG_NAME) . | ||
| docker build -t $(TAG_NAME_SPATIAL) -f Dockerfile.spatial . | ||
|
|
||
| push: ## Push an image to the Docker registry, `make push CKAN_VERSION=2.9` (default is 2.10) | ||
| docker push $(TAG_NAME) | ||
| docker push $(TAG_NAME_SPATIAL) |