diff --git a/.github/workflows/javadocs.yml b/.github/workflows/javadocs.yml deleted file mode 100644 index eb1960999..000000000 --- a/.github/workflows/javadocs.yml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright 2017-2023 O2 Czech Republic, a.s. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -name: Upload javadocs - -on: - push: - branches: - - master - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: "adopt" - java-version: 11 - - name: Generate and upload javadocs - env: - PROXIMA_DOC_GC_STORAGE: ${{ secrets.JAVADOC_GOOGLE_STORAGE }} - GOOGLE_CREDENTIALS: ${{ secrets.JAVADOC_GOOGLE_CREDENTIALS }} - run: ./.github/workflows/upload-javadocs.sh - diff --git a/.github/workflows/mvn.sh b/.github/workflows/mvn.sh deleted file mode 100755 index ff33aa32a..000000000 --- a/.github/workflows/mvn.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Copyright 2017-2023 O2 Czech Republic, a.s. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -set -e - -# FIXME; remove this after fix to maven.java.net certificate. -# auto disable after 2023/04/30 -if [[ $(date +%Y-%m-%d) < "2023-05-01" ]]; then - mvn install -Pallow-snaphots -DskipTests -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.resolver.transport=wagon -pl :proxima-direct-io-hbase-bindings -am -fi - -IS_PR=$([[ ! -z $GITHUB_HEAD_REF ]] && echo ${GITHUB_HEAD_REF} || echo false) -BRANCH=${GITHUB_REF##*/} - -MVN_OPTS=$(.github/mvn-build-changed-modules.sh ${BRANCH} ${IS_PR}) -echo "${BRANCH} ${IS_PR} ${MVN_OPTS}" - -mvn spotless:check -B -V && mvn install -B -V -Pallow-snapshots,with-coverage,ci -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 ${MVN_OPTS} || (sleep 5; exit 1) - -if [[ $1 == "11" ]]; then - if [ "${IS_PR}" != "false" ] || [ "${BRANCH}" == "master" ]; then - mvn org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -B -V -Pallow-snapshots,with-coverage,ci $(.github/mvn-build-changed-modules.sh sonar ${BRANCH} ${IS_PR}); - fi -fi - diff --git a/.github/workflows/upload-javadocs.sh b/.github/workflows/upload-javadocs.sh deleted file mode 100755 index 36d485870..000000000 --- a/.github/workflows/upload-javadocs.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Copyright 2017-2023 O2 Czech Republic, a.s. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -set -eu - -source $(dirname $0)/functions.sh - -if [ "$(is_datadriven_repo ${GITHUB_REPOSITORY})" == "1" ]; then - VERSION=$(proxima_version) - - ./gradlew javadocAggregate -x test - echo ${GOOGLE_CREDENTIALS} > /tmp/google-credentials.json - gcloud auth activate-service-account --key-file /tmp/google-credentials.json - - gsutil -m cp -r build/docs/apidocs gs://${PROXIMA_DOC_GC_STORAGE}/${VERSION}/ -fi diff --git a/README.md b/README.md index 14a976871..5fd102cfc 100644 --- a/README.md +++ b/README.md @@ -346,9 +346,7 @@ First, let's introduce some glossary: ``` ## Online Java docs - * [Latest](https://proxima.datadriven.cz/javadoc/) - * [0.13-SNAPSHOT](https://proxima.datadriven.cz/javadoc/0.13-SNAPSHOT/index.html) - * [0.12.0](https://proxima.datadriven.cz/javadoc/0.12.0/index.html) + * [Latest](https://datadrivencz.github.io/proxima-platform/apidoc/) ## Build notes CI is run only against changed modules (and its dependents) in pull requests. To completely rebuild the whole project in a PR push a commit with commit message 'rebuild'. After the build, you can squash and remove the commit. diff --git a/docs/src/content/book/datamodel.md b/docs/src/content/book/datamodel.md index 9700aa850..7d3cdc058 100644 --- a/docs/src/content/book/datamodel.md +++ b/docs/src/content/book/datamodel.md @@ -19,7 +19,7 @@ A _key_ is unique string identification of any particular instance of an entity. Here we see two instances of a particular entity with attribute `Attribute1` and `Attribute2` with type `int`. -The type and serialization of all attributes is defined by a _scheme_ and associated [ValueSerializer](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/scheme/ValueSerializer.html). Let's assume we have an eshop which sells some goods. A typical eshop will have a database of products and users and will want to track behavior of users on the website in order to provide some level of personification. Given such use case, we would define entities in HOCON configuration as follows: +The type and serialization of all attributes is defined by a _scheme_ and associated [ValueSerializer](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/scheme/ValueSerializer.html). Let's assume we have an eshop which sells some goods. A typical eshop will have a database of products and users and will want to track behavior of users on the website in order to provide some level of personification. Given such use case, we would define entities in HOCON configuration as follows: ``` entities { @@ -79,9 +79,9 @@ Each _attribute_ is of two possible types: ##### Scalar attributes -`Details` and `preferences` are examples of a _scalar attribute_ of entity _user_. Such attribute may be present or missing (be null), but if present it can have only single value of given type. The type of the attribute is given by its _scheme_. _Scheme_ is [URI](https://en.wikipedia.org/wiki/Uniform_Resource_Identifier) which points to a [ValueSerializerFactory](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/scheme/ValueSerializerFactory.html), which creates instances of [ValueSerializer](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/scheme/ValueSerializer.html). This serializer is then used whenever the platform needs to convert the object representing the attribute's value to bytes and back. +`Details` and `preferences` are examples of a _scalar attribute_ of entity _user_. Such attribute may be present or missing (be null), but if present it can have only single value of given type. The type of the attribute is given by its _scheme_. _Scheme_ is [URI](https://en.wikipedia.org/wiki/Uniform_Resource_Identifier) which points to a [ValueSerializerFactory](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/scheme/ValueSerializerFactory.html), which creates instances of [ValueSerializer](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/scheme/ValueSerializer.html). This serializer is then used whenever the platform needs to convert the object representing the attribute's value to bytes and back. -The scheme `proto:` used in the example above declares that the attribute will be serialized using [ProtoValueSerializer](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/scheme/proto/ProtoSerializerFactory.ProtoValueSerializer.html) and hold a corresponding class that was generated using `protoc` (and extends [Message](https://www.javadoc.io/doc/com.google.protobuf/protobuf-java/latest/com/google/protobuf/Message.html)). For details refer to [protocol buffers](https://developers.google.com/protocol-buffers) documentation. +The scheme `proto:` used in the example above declares that the attribute will be serialized using [ProtoValueSerializer](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/scheme/proto/ProtoSerializerFactory.ProtoValueSerializer.html) and hold a corresponding class that was generated using `protoc` (and extends [Message](https://www.javadoc.io/doc/com.google.protobuf/protobuf-java/latest/com/google/protobuf/Message.html)). For details refer to [protocol buffers](https://developers.google.com/protocol-buffers) documentation. ##### Wildcard attributes @@ -95,12 +95,12 @@ We will see examples of all these uses throughout this book. ### StreamElement -The platform handles all data as _data streams_ consisting of _upserts_ and _deletions_ of data. Each _upsert_ or _delete_ is an immutable event describing that a new data element was added, updated or removed. Every [StreamElement](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/storage/StreamElement.html) consists of the following parts: +The platform handles all data as _data streams_ consisting of _upserts_ and _deletions_ of data. Each _upsert_ or _delete_ is an immutable event describing that a new data element was added, updated or removed. Every [StreamElement](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/storage/StreamElement.html) consists of the following parts: | entity | attribute | key | timestamp | value | delete wildcard flag | |-----------|-------------|-------|------------|-------|----------------------| -Entity is represented by [EntityDescriptor](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/repository/EntityDescriptor.html), attribute is represented by its name (which is especially needed for wildcard attributes, because name of the attribute does not represent a specific instance of the attribute) and [AttributeDescriptor](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/repository/AttributeDescriptor.html). +Entity is represented by [EntityDescriptor](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/repository/EntityDescriptor.html), attribute is represented by its name (which is especially needed for wildcard attributes, because name of the attribute does not represent a specific instance of the attribute) and [AttributeDescriptor](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/repository/AttributeDescriptor.html). Key, timestamp and value are the key of the entity (representing a "row"), timestamp is epoch timestamp in millis representing the instant at which the particular change (upsert or delete) happened and value is the new updated value (for upserts) or null (for deletes). diff --git a/docs/src/content/book/generator.md b/docs/src/content/book/generator.md index 1d5d5f422..5e395dcaf 100644 --- a/docs/src/content/book/generator.md +++ b/docs/src/content/book/generator.md @@ -97,7 +97,7 @@ This adds `cz.o2.proxima:compiler-maven-plugin` as a plugin to the build and inv If you cannot use the maven plugin, you can incorporate generation of the model into your build using direct invocation of the generator. -The generator can be invoked from `cz.o2.proxima:proxima-compiler-java-cli` artifact using class [ModelGenerator](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/generator/ModelGenerator.html). +The generator can be invoked from `cz.o2.proxima:proxima-compiler-java-cli` artifact using class [ModelGenerator](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/generator/ModelGenerator.html). Downloading the artifact and running the class, we get: ```shell diff --git a/docs/src/content/book/storages.md b/docs/src/content/book/storages.md index 0b3662381..314f51b14 100644 --- a/docs/src/content/book/storages.md +++ b/docs/src/content/book/storages.md @@ -15,7 +15,7 @@ We will describe the key properties of these three type below. A commit log is a type of write-once-read-many storage. The key properties are, that it consists of immutable records, that can only be appended and have associated timestamp. Such records are called _events_. A commit log is thus an **unordered** sequence of events and thus forms a _partitioned event stream_. Proxima platform further restricts this to _partitioned event stream of upserts_. -This is due to the requirement that every event in the event stream refers to a particular _(key, attribute)_ pair, thus every event describes a change of value (or insertion of new value) of particular attribute of particular entity at given timestamp. These proparties were defined in [Data model]({{< relref "/book/datamodel" >}}) and are constituents of **StreamElement** (see [javadoc](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/storage/StreamElement.html)), a commit log is therefore a stream of StreamElements. +This is due to the requirement that every event in the event stream refers to a particular _(key, attribute)_ pair, thus every event describes a change of value (or insertion of new value) of particular attribute of particular entity at given timestamp. These proparties were defined in [Data model]({{< relref "/book/datamodel" >}}) and are constituents of **StreamElement** (see [javadoc](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/storage/StreamElement.html)), a commit log is therefore a stream of StreamElements. A commit log, being a stream of upserts, can be reduced to table, via the _table-stream duality_ (see [Data model]({{< relref "/book/datamodel" >}}). We can create a table from the stream of upserts by _applying_ the upserts to the initial state of the table. Provided the commit log implementation (e.g. [Apache Kafka](https://kafka.apache.org/)) is able to hold all most recent updates to any key-attributes, then we can reconstruct a table from the stream by reading all the updates and applying each individual update to initially empty table. A commit log that has this property is called a _state commit log_, because it is able to hold the state (and modifications over time) of attributes. @@ -135,6 +135,6 @@ A replica attribute family might also specify an optional `filter`, which will t } ``` -The filter must implement [StorageFilter](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/storage/StorageFilter.html) interface. +The filter must implement [StorageFilter](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/storage/StorageFilter.html) interface. With this high-level description of entities and attribute families, we are ready to see how will this definition by actually used to access the data and to ensure eventual consistency between primary and replica storages. Let's first see the latter described in [Replication]({{< relref "/book/replication" >}}). diff --git a/docs/src/content/book/transformations.md b/docs/src/content/book/transformations.md index 39ddde9da..5145e2137 100644 --- a/docs/src/content/book/transformations.md +++ b/docs/src/content/book/transformations.md @@ -15,6 +15,6 @@ transformations { } ``` -This definition declares a transformation of attribute `data` of entity `event` using a user-defined transformation implemented in `cz.o2.proxima.example.EventDataToUserHistory` (see [here](https://github.com/datadrivencz/proxima-platform/blob/master/example/model/src/main/java/cz/o2/proxima/example/EventDataToUserHistory.java)). The transformation is implementation of [ElementWiseTransfomation](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/transform/ElementWiseTransformation.html), which receives input `StreamElement` for updates to the input attribute(s) and emits transformed elements through [Collector](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/transform/ElementWiseTransformation.Collector.html). The `apply` method returns number of outputs that will be written through the collector, because the collection might be asynchronous. +This definition declares a transformation of attribute `data` of entity `event` using a user-defined transformation implemented in `cz.o2.proxima.example.EventDataToUserHistory` (see [here](https://github.com/datadrivencz/proxima-platform/blob/master/example/model/src/main/java/cz/o2/proxima/example/EventDataToUserHistory.java)). The transformation is implementation of [ElementWiseTransfomation](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/transform/ElementWiseTransformation.html), which receives input `StreamElement` for updates to the input attribute(s) and emits transformed elements through [Collector](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/transform/ElementWiseTransformation.Collector.html). The `apply` method returns number of outputs that will be written through the collector, because the collection might be asynchronous. -As in the case of attribute families (see [here]({{< relref "/book/storages#selecting-what-to-replicate" >}})), transformations might be invoked for all updates (upserts) to the source attribute(s), or the definition can contain optional `filter` implementation of [StorageFilter](https://proxima.datadriven.cz/javadoc/latest/cz/o2/proxima/core/storage/StorageFilter.html) interface. +As in the case of attribute families (see [here]({{< relref "/book/storages#selecting-what-to-replicate" >}})), transformations might be invoked for all updates (upserts) to the source attribute(s), or the definition can contain optional `filter` implementation of [StorageFilter](https://datadrivencz.github.io/proxima-platform/apidocs/cz/o2/proxima/core/storage/StorageFilter.html) interface.