diff --git a/.github/secret_scanning.yml b/.github/secret_scanning.yml index 3d036d4d4..3e87bf78b 100644 --- a/.github/secret_scanning.yml +++ b/.github/secret_scanning.yml @@ -1,4 +1,4 @@ paths-ignore: - - "quickstart/exampledata/config/credentials.yml" - - "quickstart/exampledata/config/nginx/mtls.conf.d/*.key" - - "quickstart/exampledata/config/nginx/mtls.conf.d/*.crt" + - "examples/exampledata/config/credentials.yml" + - "examples/exampledata/config/nginx/mtls.conf.d/*.key" + - "examples/exampledata/config/nginx/mtls.conf.d/*.crt" diff --git a/CHANGELOG.md b/CHANGELOG.md index aa09e0a35..176afad7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -315,7 +315,7 @@ * add possibility to convert hex to int in `calculator` processor with new added function `from_hex` * add metrics on rule level -* add grafana example dashboards under `quickstart/exampledata/config/grafana/dashboards` +* add grafana example dashboards under `examples/exampledata/config/grafana/dashboards` * add new configuration field `id` for all rules to identify rules in metrics and logs - if no `id` is given, the `id` will be generated in a stable way - add verification of rule `id` uniqueness on processor level over both rule trees to ensure metrics are counted correctly on rule level diff --git a/README.md b/README.md index 8e33a94b9..bf7d2371c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ allowing further applications besides log handling. This readme provides basic information about the following topics: - [About Logprep](#about-logprep) - [Getting Started](https://logprep.readthedocs.io/en/latest/getting_started.html) -- [Docker Quickstart](https://logprep.readthedocs.io/en/latest/getting_started.html#logprep-quickstart-environment) +- [Deployment Examples](https://logprep.readthedocs.io/en/latest/examples/index.html) - [Event Generation](https://logprep.readthedocs.io/en/latest/user_manual/execution.html#event-generation) - [Documentation](https://logprep.readthedocs.io/en/latest) - [Contributing](#contributing) diff --git a/doc/source/development/programaticly_start_logprep.rst b/doc/source/development/programaticly_start_logprep.rst index 4c1426ab7..51de7c9a4 100644 --- a/doc/source/development/programaticly_start_logprep.rst +++ b/doc/source/development/programaticly_start_logprep.rst @@ -23,10 +23,10 @@ An example with input connector and preprocessors could look like this: "predetector": { "type": "pre_detector", "specific_rules": [ - "quickstart/exampledata/rules/pre_detector/specific" + "examples/exampledata/rules/pre_detector/specific" ], "generic_rules": [ - "quickstart/exampledata/rules/pre_detector/generic" + "examples/exampledata/rules/pre_detector/generic" ], "pre_detector_topic": "output_topic" } @@ -61,10 +61,10 @@ An example without input connector and preprocessors could look like this: "predetector": { "type": "pre_detector", "specific_rules": [ - "quickstart/exampledata/rules/pre_detector/specific" + "examples/exampledata/rules/pre_detector/specific" ], "generic_rules": [ - "quickstart/exampledata/rules/pre_detector/generic" + "examples/exampledata/rules/pre_detector/generic" ], "pre_detector_topic": "output_topic" } diff --git a/doc/source/examples/compose.rst b/doc/source/examples/compose.rst index 9e1747f80..98c9769bc 100644 --- a/doc/source/examples/compose.rst +++ b/doc/source/examples/compose.rst @@ -5,7 +5,7 @@ Docker Compose Example Deployment To demonstrate the functionality of logprep this repo comes with a complete `kafka`, `logprep` and `opensearch` stack. To get it running `docker` with compose support must be first installed. -The docker compose file is located in the directory `quickstart`. +The docker compose file is located in the directory `examples/compose`. A prerequisite is to run `sysctl -w vm.max_map_count=262144`, otherwise Opensearch might not properly start. @@ -14,7 +14,7 @@ The environment can either be started with a Logprep container or without one: Run without Logprep Container (default) --------------------------------------- - 1. Run from within the `quickstart` directory: + 1. Run from within the `examples/compose` directory: .. code-block:: bash @@ -25,14 +25,14 @@ Run without Logprep Container (default) .. code-block:: bash - logprep run quickstart/exampledata/config/pipeline.yml + logprep run examples/exampledata/config/pipeline.yml If logprep is run with the metrics enabled, the necessary environment variable has to be set first: .. code-block:: bash export PROMETHEUS_MULTIPROC_DIR="tmp/logprep" - logprep run quickstart/exampledata/config/pipeline.yml + logprep run examples/exampledata/config/pipeline.yml @@ -40,7 +40,7 @@ Run without Logprep Container (default) Run with Logprep Container -------------------------- - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -50,7 +50,7 @@ Run with Logprep Container Run with getting config from http server with basic authentication ------------------------------------------------------------------ - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -60,14 +60,14 @@ Run with getting config from http server with basic authentication .. code-block:: bash - export LOGPREP_CREDENTIALS_FILE="quickstart/exampledata/config/credentials.yml" + export LOGPREP_CREDENTIALS_FILE="examples/exampledata/config/credentials.yml" logprep run http://localhost:8081/config/pipeline.yml Run with getting config from http server with mTLS authentication ----------------------------------------------------------------- - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -77,12 +77,12 @@ Run with getting config from http server with mTLS authentication .. code-block:: bash - export LOGPREP_CREDENTIALS_FILE="quickstart/exampledata/config/credentials.yml" + export LOGPREP_CREDENTIALS_FILE="examples/exampledata/config/credentials.yml" logprep run https://localhost:8082/config/pipeline.yml -Interacting with the Quickstart Environment -------------------------------------------- +Interacting with the Compose Environment +---------------------------------------- The start up takes a few seconds to complete, but once everything is up and running it is possible to write JSON events into Kafka and read the processed events in @@ -111,9 +111,9 @@ UCL Postgres: `localhost:5432` ucl ucl ====================== ================= ======================== ======================= The example rules that are used in the docker instance of Logprep can be found -in `quickstart/exampledata/rules`. +in `examples/exampledata/rules`. Example events that trigger for the example rules can be found in -`quickstart/exampledata/input_logdata/logclass/test_input.jsonl`. +`examples/exampledata/input_logdata/logclass/test_input.jsonl`. These events can be added to Kafka with the following command: .. code-block:: bash @@ -133,12 +133,12 @@ Utilizing FDA and UCL If you want to try out the FDA and UCL you first have to do some preparations. -0. Run the quickstart setup with the :code:`oauth2` profile: +0. Run the example compose setup with the :code:`oauth2` profile: :code:`docker compose --profile oauth2 up -d`. 1. Sign into the keycloak admin panel and create a logprep user in the :code:`logprep` realm. Make sure that the user is part of the :code:`logprep-admin` group and has a password. If you choose a password other than :code:`logprep` you have to update the credentials file - :code:`quickstart/exampledata/config/credentials.yml`, such that the password of + :code:`examples/exampledata/config/credentials.yml`, such that the password of :code:`http://localhost:3001` and :code:`http://localhost:3002` reflects your choice. 2. You have to login to the FDA with the previously created user and create a release, as well as your first logclass. It is also necessary to add an example event to this logclass in order @@ -148,7 +148,7 @@ If you want to try out the FDA and UCL you first have to do some preparations. your created logprep user and then configure required Use-Cases. At the current moment these configuration are not yet processed by logprep though, as the ucl only provides a mock endpoint which doesn't contain your Use-Case configurations. -4. Set the env :code:`LOGPREP_CREDENTIALS_FILE` to :code:`quickstart/exampledata/config/credentials.yml` +4. Set the env :code:`LOGPREP_CREDENTIALS_FILE` to :code:`examples/exampledata/config/credentials.yml` Once you have set everything up you can run logprep with the following command. Just consider that the first :code:`pipeline.yml` argument is used to define a proper :code:`input` @@ -157,10 +157,10 @@ you should ensure that the :code:`stage` and :code:`loglcass` are set properly. .. code-block:: bash - logprep run quickstart/exampledata/config/pipeline.yml "http://localhost:3002/api/v1/pipelines?stage=prod&logclass=ExampleClass" "http://localhost:3001/api/v1/general-predetection" + logprep run examples/exampledata/config/pipeline.yml "http://localhost:3002/api/v1/pipelines?stage=prod&logclass=ExampleClass" "http://localhost:3001/api/v1/general-predetection" .. note:: - If you did use the quickstart setup before and run into problems it is advised to first pull + If you did use the example compose setup before and run into problems it is advised to first pull all images again to update them to the latest version: - :code:`docker compose -f ./quickstart/docker-compose.yml pull`. + :code:`docker compose -f ./example/compose/docker-compose.yml pull`. diff --git a/quickstart/docker-compose.yml b/examples/compose/docker-compose.yml similarity index 86% rename from quickstart/docker-compose.yml rename to examples/compose/docker-compose.yml index 63f893e0d..4cb8e7b4d 100644 --- a/quickstart/docker-compose.yml +++ b/examples/compose/docker-compose.yml @@ -100,21 +100,21 @@ services: - kafka - opensearch volumes: - - ../quickstart/:/home/logprep/quickstart/ + - ../../examples/:/home/logprep/examples/ tmpfs: - /tmp/logprep/prometheus_multiproc entrypoint: - logprep - run - - /home/logprep/quickstart/exampledata/config/pipeline.yml + - /home/logprep/examples/exampledata/config/pipeline.yml grafana: image: bitnami/grafana:latest container_name: grafana ports: - "3000:3000" volumes: - - ../quickstart/exampledata/config/grafana/datasources:/opt/bitnami/grafana/conf/provisioning/datasources - - ../quickstart/exampledata/config/grafana/dashboards:/opt/bitnami/grafana/conf/provisioning/dashboards + - ../../examples/exampledata/config/grafana/datasources:/opt/bitnami/grafana/conf/provisioning/datasources + - ../../examples/exampledata/config/grafana/dashboards:/opt/bitnami/grafana/conf/provisioning/dashboards networks: - prometheus prometheus: @@ -123,7 +123,7 @@ services: ports: - "9090:9090" volumes: - - ../quickstart/exampledata/config/prometheus/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml + - ../../examples/exampledata/config/prometheus/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml networks: - kafka - prometheus @@ -138,9 +138,9 @@ services: ports: - "8081:8081" volumes: - - ../quickstart/exampledata:/usr/share/nginx/html:ro - - ../quickstart/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ../quickstart/exampledata/config/nginx/conf.d:/etc/nginx/conf.d:ro + - ../../examples/exampledata:/usr/share/nginx/html:ro + - ../../examples/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ../../examples/exampledata/config/nginx/conf.d:/etc/nginx/conf.d:ro mtls-config: image: nginx:latest container_name: mtls-config @@ -149,9 +149,9 @@ services: ports: - "8082:8082" volumes: - - ../quickstart/exampledata:/usr/share/nginx/html:ro - - ../quickstart/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ../quickstart/exampledata/config/nginx/mtls.conf.d:/etc/nginx/conf.d:ro + - ../../examples/exampledata:/usr/share/nginx/html:ro + - ../../examples/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ../../examples/exampledata/config/nginx/mtls.conf.d:/etc/nginx/conf.d:ro keycloak: image: bitnami/keycloak:latest container_name: keycloak @@ -160,7 +160,7 @@ services: profiles: - oauth2 volumes: - - ../quickstart/exampledata/config/keycloak:/opt/keycloak/data/import + - ../../examples/exampledata/config/keycloak:/opt/keycloak/data/import environment: KEYCLOAK_ADMIN_USER: admin KEYCLOAK_ADMIN_PASSWORD: admin @@ -263,8 +263,8 @@ services: ports: - "3002:80" volumes: - - ../quickstart/exampledata/config/fda/config.json:/usr/share/nginx/html/assets/config/keycloak/config.json - - ../quickstart/exampledata/config/fda/default.conf:/etc/nginx/conf.d/default.conf + - ../../examples/exampledata/config/fda/config.json:/usr/share/nginx/html/assets/config/keycloak/config.json + - ../../examples/exampledata/config/fda/default.conf:/etc/nginx/conf.d/default.conf networks: - fda ucl-api: @@ -338,9 +338,9 @@ services: ports: - "3001:80" volumes: - - ../quickstart/exampledata/config/ucl/nginx.example.conf:/etc/nginx/conf.d/default.conf - - ../quickstart/exampledata/config/ucl/general.example.json:/usr/share/nginx/html/assets/config/general/config.json - - ../quickstart/exampledata/config/ucl/keycloak.example.json:/usr/share/nginx/html/assets/config/keycloak/config.json + - ../../examples/exampledata/config/ucl/nginx.example.conf:/etc/nginx/conf.d/default.conf + - ../../examples/exampledata/config/ucl/general.example.json:/usr/share/nginx/html/assets/config/general/config.json + - ../../examples/exampledata/config/ucl/keycloak.example.json:/usr/share/nginx/html/assets/config/keycloak/config.json networks: - ucl diff --git a/quickstart/exampledata/config/credentials.yml b/examples/exampledata/config/credentials.yml similarity index 65% rename from quickstart/exampledata/config/credentials.yml rename to examples/exampledata/config/credentials.yml index 6a8c59496..8ca296a8a 100644 --- a/quickstart/exampledata/config/credentials.yml +++ b/examples/exampledata/config/credentials.yml @@ -15,14 +15,14 @@ getter: username: user password: password "https://localhost:8082": - client_key: quickstart/exampledata/config/nginx/mtls.conf.d/client.key - cert: quickstart/exampledata/config/nginx/mtls.conf.d/client.crt - ca_cert: quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt + client_key: examples/exampledata/config/nginx/mtls.conf.d/client.key + cert: examples/exampledata/config/nginx/mtls.conf.d/client.crt + ca_cert: examples/exampledata/config/nginx/mtls.conf.d/ca.crt input: endpoints: /auth-json: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt /lab/123/ABC/auditlog: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt diff --git a/quickstart/exampledata/config/dummy-output.yml b/examples/exampledata/config/dummy-output.yml similarity index 100% rename from quickstart/exampledata/config/dummy-output.yml rename to examples/exampledata/config/dummy-output.yml diff --git a/quickstart/exampledata/config/fda/config.json b/examples/exampledata/config/fda/config.json similarity index 100% rename from quickstart/exampledata/config/fda/config.json rename to examples/exampledata/config/fda/config.json diff --git a/quickstart/exampledata/config/fda/default.conf b/examples/exampledata/config/fda/default.conf similarity index 100% rename from quickstart/exampledata/config/fda/default.conf rename to examples/exampledata/config/fda/default.conf diff --git a/quickstart/exampledata/config/grafana/dashboards/kafka.json b/examples/exampledata/config/grafana/dashboards/kafka.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/kafka.json rename to examples/exampledata/config/grafana/dashboards/kafka.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json b/examples/exampledata/config/grafana/dashboards/logprep-dashboard.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json rename to examples/exampledata/config/grafana/dashboards/logprep-dashboard.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-processors.json b/examples/exampledata/config/grafana/dashboards/logprep-processors.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-processors.json rename to examples/exampledata/config/grafana/dashboards/logprep-processors.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-provider.yml b/examples/exampledata/config/grafana/dashboards/logprep-provider.yml similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-provider.yml rename to examples/exampledata/config/grafana/dashboards/logprep-provider.yml diff --git a/quickstart/exampledata/config/grafana/dashboards/sample.yaml b/examples/exampledata/config/grafana/dashboards/sample.yaml similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/sample.yaml rename to examples/exampledata/config/grafana/dashboards/sample.yaml diff --git a/quickstart/exampledata/config/grafana/datasources/prometheus.yml b/examples/exampledata/config/grafana/datasources/prometheus.yml similarity index 100% rename from quickstart/exampledata/config/grafana/datasources/prometheus.yml rename to examples/exampledata/config/grafana/datasources/prometheus.yml diff --git a/quickstart/exampledata/config/grafana/datasources/sample.yaml b/examples/exampledata/config/grafana/datasources/sample.yaml similarity index 100% rename from quickstart/exampledata/config/grafana/datasources/sample.yaml rename to examples/exampledata/config/grafana/datasources/sample.yaml diff --git a/quickstart/exampledata/config/http_pipeline.yml b/examples/exampledata/config/http_pipeline.yml similarity index 100% rename from quickstart/exampledata/config/http_pipeline.yml rename to examples/exampledata/config/http_pipeline.yml diff --git a/quickstart/exampledata/config/keycloak/logprep_realm_export.json b/examples/exampledata/config/keycloak/logprep_realm_export.json similarity index 100% rename from quickstart/exampledata/config/keycloak/logprep_realm_export.json rename to examples/exampledata/config/keycloak/logprep_realm_export.json diff --git a/quickstart/exampledata/config/nginx/conf.d/.htpasswd b/examples/exampledata/config/nginx/conf.d/.htpasswd similarity index 100% rename from quickstart/exampledata/config/nginx/conf.d/.htpasswd rename to examples/exampledata/config/nginx/conf.d/.htpasswd diff --git a/quickstart/exampledata/config/nginx/conf.d/default.conf b/examples/exampledata/config/nginx/conf.d/default.conf similarity index 100% rename from quickstart/exampledata/config/nginx/conf.d/default.conf rename to examples/exampledata/config/nginx/conf.d/default.conf diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt b/examples/exampledata/config/nginx/mtls.conf.d/ca.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt rename to examples/exampledata/config/nginx/mtls.conf.d/ca.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/ca.key b/examples/exampledata/config/nginx/mtls.conf.d/ca.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/ca.key rename to examples/exampledata/config/nginx/mtls.conf.d/ca.key diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.crt b/examples/exampledata/config/nginx/mtls.conf.d/client.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.crt rename to examples/exampledata/config/nginx/mtls.conf.d/client.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.csr b/examples/exampledata/config/nginx/mtls.conf.d/client.csr similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.csr rename to examples/exampledata/config/nginx/mtls.conf.d/client.csr diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.key b/examples/exampledata/config/nginx/mtls.conf.d/client.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.key rename to examples/exampledata/config/nginx/mtls.conf.d/client.key diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/default.conf b/examples/exampledata/config/nginx/mtls.conf.d/default.conf similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/default.conf rename to examples/exampledata/config/nginx/mtls.conf.d/default.conf diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.crt b/examples/exampledata/config/nginx/mtls.conf.d/server.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.crt rename to examples/exampledata/config/nginx/mtls.conf.d/server.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.csr b/examples/exampledata/config/nginx/mtls.conf.d/server.csr similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.csr rename to examples/exampledata/config/nginx/mtls.conf.d/server.csr diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.key b/examples/exampledata/config/nginx/mtls.conf.d/server.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.key rename to examples/exampledata/config/nginx/mtls.conf.d/server.key diff --git a/quickstart/exampledata/config/nginx/nginx.conf b/examples/exampledata/config/nginx/nginx.conf similarity index 100% rename from quickstart/exampledata/config/nginx/nginx.conf rename to examples/exampledata/config/nginx/nginx.conf diff --git a/quickstart/exampledata/config/pipeline.yml b/examples/exampledata/config/pipeline.yml similarity index 67% rename from quickstart/exampledata/config/pipeline.yml rename to examples/exampledata/config/pipeline.yml index 71fd7e70b..b45ff72f2 100644 --- a/quickstart/exampledata/config/pipeline.yml +++ b/examples/exampledata/config/pipeline.yml @@ -15,26 +15,26 @@ metrics: pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic - dissector: type: dissector specific_rules: - - quickstart/exampledata/rules/dissector/specific/ + - examples/exampledata/rules/dissector/specific/ generic_rules: - - quickstart/exampledata/rules/dissector/generic/ + - examples/exampledata/rules/dissector/generic/ - dropper: type: dropper specific_rules: - - quickstart/exampledata/rules/dropper/specific + - examples/exampledata/rules/dropper/specific generic_rules: - - quickstart/exampledata/rules/dropper/generic + - examples/exampledata/rules/dropper/generic - filter: "test_dropper" dropper: drop: @@ -44,37 +44,37 @@ pipeline: - pre_detector: type: pre_detector specific_rules: - - quickstart/exampledata/rules/pre_detector/specific + - examples/exampledata/rules/pre_detector/specific generic_rules: - - quickstart/exampledata/rules/pre_detector/generic + - examples/exampledata/rules/pre_detector/generic outputs: - opensearch: sre - tree_config: quickstart/exampledata/rules/pre_detector/tree_config.json - alert_ip_list_path: quickstart/exampledata/rules/pre_detector/alert_ips.yml + tree_config: examples/exampledata/rules/pre_detector/tree_config.json + alert_ip_list_path: examples/exampledata/rules/pre_detector/alert_ips.yml - amides: type: amides specific_rules: - - quickstart/exampledata/rules/amides/specific + - examples/exampledata/rules/amides/specific generic_rules: - - quickstart/exampledata/rules/amides/generic - models_path: quickstart/exampledata/models/model.zip + - examples/exampledata/rules/amides/generic + models_path: examples/exampledata/models/model.zip num_rule_attributions: 10 max_cache_entries: 1000000 decision_threshold: 0.32 - pseudonymizer: type: pseudonymizer - pubkey_analyst: quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem - pubkey_depseudo: quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem - regex_mapping: quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml + pubkey_analyst: examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem + pubkey_depseudo: examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem + regex_mapping: examples/exampledata/rules/pseudonymizer/regex_mapping.yml hash_salt: a_secret_tasty_ingredient outputs: - opensearch: pseudonyms specific_rules: - - quickstart/exampledata/rules/pseudonymizer/specific/ + - examples/exampledata/rules/pseudonymizer/specific/ generic_rules: - - quickstart/exampledata/rules/pseudonymizer/generic/ + - examples/exampledata/rules/pseudonymizer/generic/ max_cached_pseudonyms: 1000000 - calculator: diff --git a/quickstart/exampledata/config/postgresql/keycloak_db.sql b/examples/exampledata/config/postgresql/keycloak_db.sql similarity index 100% rename from quickstart/exampledata/config/postgresql/keycloak_db.sql rename to examples/exampledata/config/postgresql/keycloak_db.sql diff --git a/quickstart/exampledata/config/prometheus/prometheus.yml b/examples/exampledata/config/prometheus/prometheus.yml similarity index 100% rename from quickstart/exampledata/config/prometheus/prometheus.yml rename to examples/exampledata/config/prometheus/prometheus.yml diff --git a/quickstart/exampledata/config/ucl/general.example.json b/examples/exampledata/config/ucl/general.example.json similarity index 100% rename from quickstart/exampledata/config/ucl/general.example.json rename to examples/exampledata/config/ucl/general.example.json diff --git a/quickstart/exampledata/config/ucl/keycloak.example.json b/examples/exampledata/config/ucl/keycloak.example.json similarity index 100% rename from quickstart/exampledata/config/ucl/keycloak.example.json rename to examples/exampledata/config/ucl/keycloak.example.json diff --git a/quickstart/exampledata/config/ucl/nginx.example.conf b/examples/exampledata/config/ucl/nginx.example.conf similarity index 100% rename from quickstart/exampledata/config/ucl/nginx.example.conf rename to examples/exampledata/config/ucl/nginx.example.conf diff --git a/quickstart/exampledata/config/user_password.txt b/examples/exampledata/config/user_password.txt similarity index 100% rename from quickstart/exampledata/config/user_password.txt rename to examples/exampledata/config/user_password.txt diff --git a/quickstart/exampledata/input_logdata/logclass/config.yaml b/examples/exampledata/input_logdata/logclass/config.yaml similarity index 100% rename from quickstart/exampledata/input_logdata/logclass/config.yaml rename to examples/exampledata/input_logdata/logclass/config.yaml diff --git a/quickstart/exampledata/input_logdata/logclass/test_input.jsonl b/examples/exampledata/input_logdata/logclass/test_input.jsonl similarity index 100% rename from quickstart/exampledata/input_logdata/logclass/test_input.jsonl rename to examples/exampledata/input_logdata/logclass/test_input.jsonl diff --git a/quickstart/exampledata/models/model.zip b/examples/exampledata/models/model.zip similarity index 100% rename from quickstart/exampledata/models/model.zip rename to examples/exampledata/models/model.zip diff --git a/quickstart/exampledata/rules/amides/generic/amides_generic.yml b/examples/exampledata/rules/amides/generic/amides_generic.yml similarity index 100% rename from quickstart/exampledata/rules/amides/generic/amides_generic.yml rename to examples/exampledata/rules/amides/generic/amides_generic.yml diff --git a/quickstart/exampledata/rules/amides/specific/amides_specific.yml b/examples/exampledata/rules/amides/specific/amides_specific.yml similarity index 100% rename from quickstart/exampledata/rules/amides/specific/amides_specific.yml rename to examples/exampledata/rules/amides/specific/amides_specific.yml diff --git a/quickstart/exampledata/rules/dissector/generic/example_rule.yml b/examples/exampledata/rules/dissector/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dissector/generic/example_rule.yml rename to examples/exampledata/rules/dissector/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/dissector/specific/example_rule.yml b/examples/exampledata/rules/dissector/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dissector/specific/example_rule.yml rename to examples/exampledata/rules/dissector/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/dropper/generic/example_rule.yml b/examples/exampledata/rules/dropper/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dropper/generic/example_rule.yml rename to examples/exampledata/rules/dropper/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/dropper/specific/example_rule.yml b/examples/exampledata/rules/dropper/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dropper/specific/example_rule.yml rename to examples/exampledata/rules/dropper/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/labeler/generic/example_rule.yml b/examples/exampledata/rules/labeler/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/labeler/generic/example_rule.yml rename to examples/exampledata/rules/labeler/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/labeler/schema.json b/examples/exampledata/rules/labeler/schema.json similarity index 100% rename from quickstart/exampledata/rules/labeler/schema.json rename to examples/exampledata/rules/labeler/schema.json diff --git a/quickstart/exampledata/rules/labeler/specific/example_rule.yml b/examples/exampledata/rules/labeler/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/labeler/specific/example_rule.yml rename to examples/exampledata/rules/labeler/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/alert_ips.yml b/examples/exampledata/rules/pre_detector/alert_ips.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/alert_ips.yml rename to examples/exampledata/rules/pre_detector/alert_ips.yml diff --git a/quickstart/exampledata/rules/pre_detector/generic/example_rule.yml b/examples/exampledata/rules/pre_detector/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/generic/example_rule.yml rename to examples/exampledata/rules/pre_detector/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/specific/example_rule.yml b/examples/exampledata/rules/pre_detector/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/specific/example_rule.yml rename to examples/exampledata/rules/pre_detector/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/tree_config.json b/examples/exampledata/rules/pre_detector/tree_config.json similarity index 100% rename from quickstart/exampledata/rules/pre_detector/tree_config.json rename to examples/exampledata/rules/pre_detector/tree_config.json diff --git a/quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem b/examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem rename to examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem diff --git a/quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem b/examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem rename to examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem diff --git a/quickstart/exampledata/rules/pseudonymizer/generic/example_rule.yml b/examples/exampledata/rules/pseudonymizer/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/generic/example_rule.yml rename to examples/exampledata/rules/pseudonymizer/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml b/examples/exampledata/rules/pseudonymizer/regex_mapping.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml rename to examples/exampledata/rules/pseudonymizer/regex_mapping.yml diff --git a/quickstart/exampledata/rules/pseudonymizer/specific/example_rule.yml b/examples/exampledata/rules/pseudonymizer/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/specific/example_rule.yml rename to examples/exampledata/rules/pseudonymizer/specific/example_rule.yml diff --git a/logprep/connector/http/input.py b/logprep/connector/http/input.py index 5f412d568..55690affd 100644 --- a/logprep/connector/http/input.py +++ b/logprep/connector/http/input.py @@ -45,7 +45,7 @@ endpoints: /firstendpoint: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt /second*: username: user password: secret_password diff --git a/logprep/metrics/metrics.py b/logprep/metrics/metrics.py index 99ae80813..caf12bcf7 100644 --- a/logprep/metrics/metrics.py +++ b/logprep/metrics/metrics.py @@ -3,7 +3,7 @@ :code:`logprep_number_of_processed_events_total` or :code:`logprep_processing_time_per_event_sum`. Examples of grafana dashboards can be found in `the logprep github repo -`_ +`_ Configuration ============= diff --git a/logprep/processor/amides/processor.py b/logprep/processor/amides/processor.py index d6652d105..5e0a6de87 100644 --- a/logprep/processor/amides/processor.py +++ b/logprep/processor/amides/processor.py @@ -28,7 +28,7 @@ Since there is a plethora of different SIEM event types, the current implementation focuses on events that provide process command lines. Command lines are most commonly targeted by SIEM rules while they are also highly vulnerable to evasions. The rules and models for AMIDES provided in the -quickstart example are for Sysmon Process Creation events. In general, the Amides rule format allows +deployment examples are for Sysmon Process Creation events. In general, the Amides rule format allows to create rules for other event types that provide process command lines, e.g. Process Creation events generated by Windows Security Auditing. diff --git a/logprep/util/configuration.py b/logprep/util/configuration.py index f6f8f09c8..3dc5a6a90 100644 --- a/logprep/util/configuration.py +++ b/logprep/util/configuration.py @@ -53,26 +53,26 @@ pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic - dissectorname: type: dissector specific_rules: - - quickstart/exampledata/rules/dissector/specific/ + - examples/exampledata/rules/dissector/specific/ generic_rules: - - quickstart/exampledata/rules/dissector/generic/ + - examples/exampledata/rules/dissector/generic/ - dropper: type: dropper specific_rules: - - quickstart/exampledata/rules/dropper/specific + - examples/exampledata/rules/dropper/specific generic_rules: - - quickstart/exampledata/rules/dropper/generic + - examples/exampledata/rules/dropper/generic - filter: "test_dropper" dropper: drop: @@ -82,37 +82,37 @@ - pre_detector: type: pre_detector specific_rules: - - quickstart/exampledata/rules/pre_detector/specific + - examples/exampledata/rules/pre_detector/specific generic_rules: - - quickstart/exampledata/rules/pre_detector/generic + - examples/exampledata/rules/pre_detector/generic outputs: - opensearch: sre - tree_config: quickstart/exampledata/rules/pre_detector/tree_config.json - alert_ip_list_path: quickstart/exampledata/rules/pre_detector/alert_ips.yml + tree_config: examples/exampledata/rules/pre_detector/tree_config.json + alert_ip_list_path: examples/exampledata/rules/pre_detector/alert_ips.yml - amides: type: amides specific_rules: - - quickstart/exampledata/rules/amides/specific + - examples/exampledata/rules/amides/specific generic_rules: - - quickstart/exampledata/rules/amides/generic - models_path: quickstart/exampledata/models/model.zip + - examples/exampledata/rules/amides/generic + models_path: examples/exampledata/models/model.zip num_rule_attributions: 10 max_cache_entries: 1000000 decision_threshold: 0.32 - pseudonymizer: type: pseudonymizer - pubkey_analyst: quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem - pubkey_depseudo: quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem - regex_mapping: quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml + pubkey_analyst: examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem + pubkey_depseudo: examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem + regex_mapping: examples/exampledata/rules/pseudonymizer/regex_mapping.yml hash_salt: a_secret_tasty_ingredient outputs: - opensearch: pseudonyms specific_rules: - - quickstart/exampledata/rules/pseudonymizer/specific/ + - examples/exampledata/rules/pseudonymizer/specific/ generic_rules: - - quickstart/exampledata/rules/pseudonymizer/generic/ + - examples/exampledata/rules/pseudonymizer/generic/ max_cached_pseudonyms: 1000000 - calculator: @@ -173,12 +173,12 @@ pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic" + - examples/exampledata/rules/labeler/generic" export LOGPREP_OUTPUT=" output: kafka: diff --git a/tests/acceptance/test_full_configuration.py b/tests/acceptance/test_full_configuration.py index a09dd3fb6..5175c31ab 100644 --- a/tests/acceptance/test_full_configuration.py +++ b/tests/acceptance/test_full_configuration.py @@ -87,12 +87,12 @@ def test_start_of_logprep_from_http_with_templated_url_and_config(): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic """, "LOGPREP_OUTPUT": """ output: diff --git a/tests/testdata/unit/requester/generic_rules/requester.json b/tests/testdata/unit/requester/generic_rules/requester.json index 2564b1e24..d7154a894 100644 --- a/tests/testdata/unit/requester/generic_rules/requester.json +++ b/tests/testdata/unit/requester/generic_rules/requester.json @@ -2,7 +2,7 @@ { "filter": "message1", "requester": { - "url": "http://localhost:32000/quickstart/exampledata/config/pipeline.yml", + "url": "http://localhost:32000/examples/exampledata/config/pipeline.yml", "method": "GET" } } diff --git a/tests/testdata/unit/requester/specific_rules/requester.json b/tests/testdata/unit/requester/specific_rules/requester.json index 1ae41c251..701c4e95e 100644 --- a/tests/testdata/unit/requester/specific_rules/requester.json +++ b/tests/testdata/unit/requester/specific_rules/requester.json @@ -2,7 +2,7 @@ { "filter": "message", "requester": { - "url": "http://localhost:32000/quickstart/exampledata/config/pipeline.yml", + "url": "http://localhost:32000/examples/exampledata/config/pipeline.yml", "method": "GET" } } diff --git a/tests/unit/connector/test_real_kafka.py b/tests/unit/connector/test_real_kafka.py index 25eb6a909..a35d00f8b 100644 --- a/tests/unit/connector/test_real_kafka.py +++ b/tests/unit/connector/test_real_kafka.py @@ -24,7 +24,7 @@ def setup_module(): if not in_ci: subprocess.run( - ["docker", "compose", "-f", "quickstart/docker-compose.yml", "up", "-d", "kafka"] + ["docker", "compose", "-f", "example/compose/docker-compose.yml", "up", "-d", "kafka"] ) diff --git a/tests/unit/test_quickstart.py b/tests/unit/test_quickstart.py index 18be79ee7..fbd686676 100644 --- a/tests/unit/test_quickstart.py +++ b/tests/unit/test_quickstart.py @@ -5,19 +5,19 @@ from logprep import run_logprep -class TestQuickstart: - QUICKSTART_CONFIG_PATH = "quickstart/exampledata/config/pipeline.yml" +class TestExampleCompose: + EXAMPLE_CONFIG_PATH = "examples/exampledata/config/pipeline.yml" @mock.patch("os.environ", new={"PROMETHEUS_MULTIPROC_DIR": "/tmp"}) - def test_quickstart_setup_is_valid(self): - """ensures the quickstart rules are valid""" + def test_example_compose_setup_is_valid(self): + """ensures the example rules are valid""" with mock.patch( "sys.argv", [ "logprep", "test", "config", - self.QUICKSTART_CONFIG_PATH, + self.EXAMPLE_CONFIG_PATH, ], ): with pytest.raises(SystemExit) as e_info: diff --git a/tests/unit/test_run_logprep.py b/tests/unit/test_run_logprep.py index 7f61b55d0..cf4e4870a 100644 --- a/tests/unit/test_run_logprep.py +++ b/tests/unit/test_run_logprep.py @@ -55,7 +55,7 @@ def setup_method(self): "logprep.util.configuration.Configuration.as_yaml", ), ( - "test dry-run tests/testdata/config/config.yml quickstart/exampledata/input_logdata/test_input.jsonl", + "test dry-run tests/testdata/config/config.yml examples/exampledata/input_logdata/test_input.jsonl", "logprep.util.rule_dry_runner.DryRunner.run", ), ( @@ -255,7 +255,7 @@ def test_logprep_exits_on_request_exception(self): @mock.patch("logprep.util.rule_dry_runner.DryRunner.run") def test_test_dry_run_starts_dry_runner(self, mock_dry_runner): config_path = ("tests/testdata/config/config.yml",) - events_path = "quickstart/exampledata/input_logdata/test_input.jsonl" + events_path = "examples/exampledata/input_logdata/test_input.jsonl" result = self.cli_runner.invoke(cli, ["test", "dry-run", *config_path, events_path]) assert result.exit_code == 0 mock_dry_runner.assert_called() diff --git a/tests/unit/util/test_configuration.py b/tests/unit/util/test_configuration.py index 3c067d58a..4a027b339 100644 --- a/tests/unit/util/test_configuration.py +++ b/tests/unit/util/test_configuration.py @@ -196,7 +196,7 @@ def test_pipeline_property_is_merged_from_configs(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -377,9 +377,9 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], } } ] @@ -393,10 +393,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME_UNKNOWN_OPTION": "FOO", } } @@ -411,10 +411,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME UNKNOWN OPTION": "FOO", } } @@ -444,10 +444,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME UNKNOWN OPTION": "FOO", } }, @@ -523,12 +523,12 @@ def test_verify_verifies_config(self, tmp_path, test_case, test_config, error_co pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic """, "LOGPREP_OUTPUT": """ output: @@ -780,7 +780,7 @@ def test_reload_raises_on_invalid_processor_config(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -805,7 +805,7 @@ def test_reload_raises_on_invalid_processor_config(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -854,7 +854,7 @@ def test_reload_raises_on_same_version(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] diff --git a/tests/unit/util/test_getter.py b/tests/unit/util/test_getter.py index c64d51d39..ddbeb2c27 100644 --- a/tests/unit/util/test_getter.py +++ b/tests/unit/util/test_getter.py @@ -545,9 +545,7 @@ def test_get_raw_raises_if_credential_file_env_set_and_unauthorizes(self): http_getter: HttpGetter = GetterFactory.from_string(f"https://{domain}/bar") with mock.patch.dict( "os.environ", - { - ENV_NAME_LOGPREP_CREDENTIALS_FILE: "quickstart/exampledata/config/credentials.yml" - }, + {ENV_NAME_LOGPREP_CREDENTIALS_FILE: "examples/exampledata/config/credentials.yml"}, ): http_getter.get_json() assert error.value.response.status_code == 401