From d0fe7272e11d77a4939d8584672ffd0cd2e6374e Mon Sep 17 00:00:00 2001 From: Oleg Kopysov Date: Thu, 5 Sep 2024 18:14:56 +0300 Subject: [PATCH 1/5] docs: Add pages for Single scan for PR and local files Signed-off-by: Oleg Kopysov --- doc/docs/user-guide/cli/cli_file.md | 57 +++++++++++++++++++++++++++++ doc/docs/user-guide/cli/cli_pr.md | 57 +++++++++++++++++++++++++++++ doc/mkdocs.yml | 6 +-- 3 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 doc/docs/user-guide/cli/cli_file.md create mode 100644 doc/docs/user-guide/cli/cli_pr.md diff --git a/doc/docs/user-guide/cli/cli_file.md b/doc/docs/user-guide/cli/cli_file.md new file mode 100644 index 00000000..af642931 --- /dev/null +++ b/doc/docs/user-guide/cli/cli_file.md @@ -0,0 +1,57 @@ +# Single scan of local files + +One-time scan on a specific local files from the command line. + +--- + +You can perform a one-time scan on a specific local file or folder using the single scan mode. +LPVS should be compiled from the source code. Please refer to the [detailed guideline](../../user-guide/service/source_code.md). + +Follow the next steps: + +* Install LPVS and navigate to the target directory. + +```bash +mvn clean install +cd target/ +``` + +* Choose a specific local file or folder which is a target for the scan and run the command with flag `--local.path`: + +```bash +java -jar lpvs-*.jar --local.path= +``` + +Example: + +```bash +-jar lpvs-*.jar --local.path=test.java +``` + +!!! note + + By default, the above commands require a pre-configured MySQL database. Use the "singlescan" profile to skip + setting up a pre-configured MySQL database: + + ```bash + java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path= + ``` + +* Optionally, generate an HTML report and save it in a specified folder using flag `--build.html.report`. + Replace `path/to/your/folder` with the full path to the folder where you want to save the HTML report, + and `your_report_filename.html` with the desired filename for the report. + +```bash +java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path= --build.html.report= +``` + +!!! warning + + Ensure that the specified folder exists before generating the HTML report. + +Examples of the command: + +```bash +java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path=test.c +java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path=test_folder --build.html.report=test/report.html +``` diff --git a/doc/docs/user-guide/cli/cli_pr.md b/doc/docs/user-guide/cli/cli_pr.md new file mode 100644 index 00000000..1a6073e1 --- /dev/null +++ b/doc/docs/user-guide/cli/cli_pr.md @@ -0,0 +1,57 @@ +# Single scan of the pull request + +One-time scan on a specific pull request from the command line. + +--- + +You can perform a one-time scan on a specific pull request using the single scan mode. +LPVS should be compiled from the source code. Please refer to the [detailed guideline](../../user-guide/service/source_code.md). + +Follow the next steps: + +* Install LPVS and navigate to the target directory. + +```bash +mvn clean install +cd target/ +``` + +* Choose a specific pull request from GitHub which is a target for the scan and run the command with flag `--github.pull.request`: + +```bash +java -jar -Dgithub.token= lpvs-*.jar --github.pull.request= +``` + +Example: + +```bash +-jar -Dgithub.token=your_personal_token lpvs-*.jar --github.pull.request=https://github.com/Samsung/LPVS/pull/594 +``` + +!!! note + + By default, the above commands require a pre-configured MySQL database. Use the "singlescan" profile to skip + setting up a pre-configured MySQL database: + + ```bash + java -jar -Dspring.profiles.active=singlescan -Dgithub.token= lpvs-*.jar --github.pull.request= + ``` + +* Optionally, generate an HTML report and save it in a specified folder using flag `--build.html.report`. +Replace `path/to/your/folder` with the full path to the folder where you want to save the HTML report, +and `your_report_filename.html` with the desired filename for the report. + +```bash +java -jar -Dspring.profiles.active=singlescan -Dgithub.token= lpvs-*.jar --github.pull.request= --build.html.report= +``` + +!!! warning + + Ensure that the specified folder exists before generating the HTML report. + +Examples of the command: + +```bash +java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --github.pull.request=https://github.com/Samsung/LPVS/pull/2 +java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --github.pull.request=https://github.com/Samsung/LPVS/pull/2 --build.html.report=report.html +``` diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index 579fa191..15b7b15c 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -33,9 +33,9 @@ nav: - GitHub webhook configuration: user-guide/service/webhook.md - Run service using pre-built Docker image: user-guide/service/docker.md #- Build and run service from the source code: user-guide/service/source_code.md - #- Command line mode: - # - Single scan of the pull request: user-guide/cli/cli_pr.md - # - Single scan of local files: user-guide/cli/cli_file.md + - Command line mode: + - Single scan of the pull request: user-guide/cli/cli_pr.md + - Single scan of local files: user-guide/cli/cli_file.md #- Configuration options: # - Possible configuration options: user-guide/config/options.md # - Database with licenses and license conflicts: user-guide/config/database.md From e883afcde832b5e491812bd9a0106c8713b3db7b Mon Sep 17 00:00:00 2001 From: Oleg Kopysov Date: Thu, 5 Sep 2024 19:42:57 +0300 Subject: [PATCH 2/5] docs: Add Database description page Signed-off-by: Oleg Kopysov --- doc/docs/user-guide/config/database.md | 61 ++++++++++++++++++++++++++ doc/mkdocs.yml | 4 +- 2 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 doc/docs/user-guide/config/database.md diff --git a/doc/docs/user-guide/config/database.md b/doc/docs/user-guide/config/database.md new file mode 100644 index 00000000..a355c857 --- /dev/null +++ b/doc/docs/user-guide/config/database.md @@ -0,0 +1,61 @@ +# Database description + +Database with licenses and license conflicts. + +--- + +The LPVS database consists of several tables. You have to pay attention to two tables: `lpvs_license_list` +and `lpvs_license_conflicts`. These tables are used to store information about licenses and their conflicts +respectively. Here's a brief description of each table. + +### Table `lpvs_license_list` + +This table stores license information. + +```sql +CREATE TABLE IF NOT EXISTS lpvs_license_list ( + id bigint NOT NULL AUTO_INCREMENT, + license_usage varchar(255) DEFAULT NULL, + license_name varchar(255) NOT NULL, + license_spdx varchar(255) NOT NULL, + license_alternative_names longtext DEFAULT NULL, + PRIMARY KEY (id), + UNIQUE (license_spdx) +); +``` + +The meanings of each field are as follows: +- `id`: Unique identifier which auto increments. +- `license_usage`: Indicates the possibility to use the license in your code and could be `PERMITTED`, `RESTRICTED`, `PROHIBITED`, or `UNREVIEWED`. +- `license_name`: Name of the license. +- `license_spdx`: SPDX identifier of the license. +- `license_alternative_names`: Alternative names of the license. + +### Table `lpvs_license_conflicts` + +This table stores license conflicts information. + +```sql +CREATE TABLE IF NOT EXISTS lpvs_license_conflicts ( + id bigint NOT NULL AUTO_INCREMENT, + conflict_license_id bigint NOT NULL, + repository_license_id bigint NOT NULL, + PRIMARY KEY (id), + KEY (conflict_license_id), + KEY (repository_license_id), + FOREIGN KEY (conflict_license_id) REFERENCES lpvs_license_list (id), + FOREIGN KEY (repository_license_id) REFERENCES lpvs_license_list (id) +); +``` + +The meanings of each field are as follows: +- `id`: Unique identifier which auto increments. +- `conflict_license_id`: ID of the conflicting license. +- `repository_license_id`: License ID of the repository. + +!!! warning + + Both tables `lpvs_license_list` and `lpvs_license_conflicts` should be filled by the user manually. + +When a new license is detected by LPVS, it will be added to the table `lpvs_license_list` with +`license_usage` status set to `UNREVIEWED`, indicating that it has not been reviewed yet. \ No newline at end of file diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index 15b7b15c..3e32cfbc 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -36,9 +36,9 @@ nav: - Command line mode: - Single scan of the pull request: user-guide/cli/cli_pr.md - Single scan of local files: user-guide/cli/cli_file.md - #- Configuration options: + - Configuration options: # - Possible configuration options: user-guide/config/options.md - # - Database with licenses and license conflicts: user-guide/config/database.md + - Database description: user-guide/config/database.md - Developer Guide: - Developer guide: dev-guide/README.md - Contributing guide: dev-guide/contributing.md From aa7d56004b995c82de6b1e0c8e21ab05ccc34fa1 Mon Sep 17 00:00:00 2001 From: Oleg Kopysov Date: Fri, 6 Sep 2024 06:51:17 +0300 Subject: [PATCH 3/5] docs: Add Database customization and Config options pages Signed-off-by: Oleg Kopysov --- doc/docs/css/mkdocsoad.css | 191 +++++++++++++++++++++++++ doc/docs/user-guide/README.md | 31 ++-- doc/docs/user-guide/config/database.md | 11 +- doc/docs/user-guide/config/options.md | 126 ++++++++++++++++ doc/mkdocs.yml | 12 +- 5 files changed, 347 insertions(+), 24 deletions(-) create mode 100644 doc/docs/css/mkdocsoad.css create mode 100644 doc/docs/user-guide/config/options.md diff --git a/doc/docs/css/mkdocsoad.css b/doc/docs/css/mkdocsoad.css new file mode 100644 index 00000000..9ec52fca --- /dev/null +++ b/doc/docs/css/mkdocsoad.css @@ -0,0 +1,191 @@ +/** + * CSS for OpenAPI HTML generated with PyMdown Extensions option. + * + * This CSS file works when using the OAD plugin with pymdownx. + * See here how to use it: + * https://www.neoteroi.dev/mkdocs-plugins/web/oad/ + * + * https://github.com/Neoteroi/mkdocs-plugins +**/ + +:root { + --http-get-color: green; + --http-delete-color: #dc0101; + --http-head-color: slateblue; + --http-options-color: steelblue; + --http-patch-color: darkorange; + --http-post-color: darkblue; + --http-put-color: darkmagenta; + --http-trace-color: darkcyan; + --http-route-param-color: rgb(51, 128, 210); + --oad-operation-separator-border-color: gray; + --oad-block-border-color: #00bfa5; + --oad-small-note-color: #666; + --oad-indent-border-color: #c5c5c5; +} + +@media screen { + /* Slate theme, i.e. dark mode */ + [data-md-color-scheme="slate"] { + --http-get-color: #2ea82e; + --http-post-color: #0093c0; + --http-put-color: #c333c3; + --oad-small-note-color: #afafaf; + } +} + +.api-tag { + font-weight: bold; +} + +span[class^="http-"] { + font-weight: bold; + color: #fff; + padding: 4px 1rem; + border-radius: 2px; + margin-right: .5rem; +} + +.http-get { + background-color: var(--http-get-color); +} + +.http-delete { + background-color: var(--http-delete-color); +} + +.http-post { + background-color: var(--http-post-color); +} + +.http-patch { + background-color: var(--http-patch-color); +} + +.http-trace { + background-color: var(--http-trace-color); +} + +.http-put { + background-color: var(--http-put-color); +} + +.http-head { + background-color: var(--http-head-color); +} + +.http-options { + background-color: var(--http-options-color); +} + +.route-param { + color: var(--http-route-param-color); +} + +.operation-separator + h3[id^="get"] .route-param { + color: var(--http-get-color); +} + +.operation-separator + h3[id^="delete"] .route-param { + color: var(--http-delete-color); +} + + +.operation-separator + h3[id^="post"] .route-param { + color: var(--http-post-color); +} + +.operation-separator + h3[id^="patch"] .route-param { + color: var(--http-patch-color); +} + +.operation-separator + h3[id^="trace"] .route-param { + color: var(--http-trace-color); +} + +.operation-separator + h3[id^="put"] .route-param { + color: var(--http-put-color); +} + +.operation-separator + h3[id^="head"] .route-param { + color: var(--http-head-color); +} + +.operation-separator + h3[id^="options"] .route-param { + color: var(--http-options-color); +} + +.api-version { + font-size: 1.2rem; +} + +.operation-separator { + margin: 0 !important; + border-bottom: 2px dotted var(--oad-operation-separator-border-color) !important; + padding-top: .5rem; +} + +.operation-separator + h3 { + margin-top: 1rem; +} + +.string-type { + color: var(--md-code-hl-string-color); +} + +.integer-type, .number-type { + color: var(--md-code-hl-number-color); +} + +.boolean-type { + color: var(--md-code-hl-keyword-color); +} + +.format { + color: var(--md-code-hl-name-color); +} + +.null-type { + color: var(--md-code-hl-keyword-color); +} + +a.ref-link { + color: var(--md-code-hl-special-color); +} + +.request-block + div { + padding-left: 1rem; + border-left: 2px dashed var(--oad-block-border-color); +} + +.small-note { + font-size: 14px; + color: var(--oad-small-note-color); +} + +.request-body-title { + margin-bottom: 4px; +} + +.request-body-title + .tabbed-set, +.response-title + .tabbed-set, +.message-separator + .tabbed-set, +.common-response, +.response-section { + margin-top: 2px; + padding-left: 1rem; + border-left: 2px dotted var(--oad-indent-border-color); +} + +.info-data { + font-size: .6rem; +} + +.message-separator { + visibility: hidden; +} + +.sub-section-title { + font-style: italic; + font-size: 14px; +} diff --git a/doc/docs/user-guide/README.md b/doc/docs/user-guide/README.md index da55f722..fa5df056 100644 --- a/doc/docs/user-guide/README.md +++ b/doc/docs/user-guide/README.md @@ -12,28 +12,23 @@ There are 2 modes of LPVS operation: - Service mode - LPVS works as a service that is started by the system and runs continuously. - Command line mode - LPVS is executed from the command line and runs until it completes its task. -## Service mode - -- [GitHub webhook configuration] -- [Run service using pre-built Docker image] -- [Build and run service from the source code] +## Prerequisites -## Command line mode +- [Scanner installation] +- [Database configuration] -- [Single scan of the pull request] -- [Single scan of local files] +## Service mode -## Configuration options +- [GitHub webhook configuration](service/webhook.md) +- [Run service using pre-built Docker image](service/docker.md) +- [Build and run service from the source code](service/source_code.md) -- [Possible configuration options] -- [Database with licenses and license conflicts] +## Command line mode -[GitHub webhook configuration]: service/webhook.md -[Run service using pre-built Docker image]: service/docker.md -[Build and run service from the source code]: service/source_code.md +- [Single scan of the pull request](cli/cli_pr.md) +- [Single scan of local files](cli/cli_file.md) -[Single scan of the pull request]: cli/cli_pr.md -[Single scan of local files]: cli/cli_file.md +## Configuration -[Possible configuration options]: config/options.md -[Database with licenses and license conflicts]: config/database.md +- [Configuration options](config/options.md) +- [Database customization](config/database.md) diff --git a/doc/docs/user-guide/config/database.md b/doc/docs/user-guide/config/database.md index a355c857..e2196792 100644 --- a/doc/docs/user-guide/config/database.md +++ b/doc/docs/user-guide/config/database.md @@ -1,6 +1,6 @@ -# Database description +# Database customization -Database with licenses and license conflicts. +Customization of the database with licenses and license conflicts. --- @@ -58,4 +58,9 @@ The meanings of each field are as follows: Both tables `lpvs_license_list` and `lpvs_license_conflicts` should be filled by the user manually. When a new license is detected by LPVS, it will be added to the table `lpvs_license_list` with -`license_usage` status set to `UNREVIEWED`, indicating that it has not been reviewed yet. \ No newline at end of file +`license_usage` status set to `UNREVIEWED`, indicating that it has not been reviewed yet. + +!!! info + + Sample MySQL dump file is located in the [repository](https://github.com/Samsung/LPVS/blob/main/src/main/resources/database_dump.sql) + and can be used as a reference. diff --git a/doc/docs/user-guide/config/options.md b/doc/docs/user-guide/config/options.md new file mode 100644 index 00000000..e5a5039d --- /dev/null +++ b/doc/docs/user-guide/config/options.md @@ -0,0 +1,126 @@ +# Configuration options + +Description of the meaningful properties and its possible values. + +--- + +## Properties in the application profile files + +There are three profiles available in the LPVS application: + +- `application.properties`: default profile which contains all main properties of the application +- `application-singlescan.properties`: profile for single scan mode +- `application-web.properties`: profile for connecting the frontend application to the backend + +--- + +### `application.properties` profile + +- `debug`: This setting determines whether the application runs in debug mode. If set to false, +the application will run in normal execution mode. Default: `false`. + +- `server.port`: This setting specifies the port number used by the application. Default: `7896`. + +- `scanner`: This setting specifies the name of the scanning tool being used. Currently supporting +[ScanOSS](https://github.com/scanoss). Default: `scanoss`. + +- `license_conflict`: This setting specifies where the license conflict information comes from. +It can be either `db` (from the manually filled table in database, refer to the +[Database guide](database.md#table-lpvs_license_conflicts)) or `scanner` (if supported by scanner). +Default: `db`. + +- `license_source`: This setting specifies the URL of the external [OSORI](https://osori-db.github.io/en/) +license database. Default: `https://olis.or.kr:8082`. + +- `spring.profiles.active`: This setting specifies which Spring profile is active. + +- `github.*`: These settings include various configurations related to GitHub such as login, token, API URL. +Fill in your own values for these settings. Please refer to the [Personal token creation guide](../service/webhook.md#create-a-personal-github-access-token) +for more details. + +- `github.secret`: This setting specifies the secret string used for configuring webhooks. Please refer to the +[Webhook configuration guide](../service/webhook.md#configure-the-webhook-in-your-github-repository-settings) +for more details. Default: `LPVS`. + +- `lpvs.*`: These settings include various configurations specific to the LPVS application like core pool size, +number of scan attempts, and version. + +- `spring.jpa.properties.hibernate.default_schema`: This setting specifies the default schema name that Hibernate +should use. Default: `lpvs`. + +- `spring.datasource.*`: These settings specify the data source configurations including URL, username, password, etc. +Fill in your own values for these settings. + +--- + +### `application-singlescan.properties` profile + +- `spring.sql.init.data-locations`: This setting specifies the location of the SQL script files that will +be executed when initializing the database. By default, it is set to `classpath*:database_dump.sql`, indicating +that the default dump file named `database_dump.sql` should be found in any package within the classpath. + +- `spring.datasource.username`: This setting specifies the username for accessing the datasource. By default, +it is left blank, indicating that no username is required for authentication. + +- `spring.datasource.password`: This setting specifies the password for accessing the datasource. Again, +it is left blank here, indicating that no password is needed for authentication. + +--- + +### `application-web.properties` profile + +These properties configure OAuth2 clients for different providers such as Google, Naver, and Kakao. +For each provider, client ID, client secret, redirect URI, authorization grant type, scope, and other +relevant details should be specified. Additionally, there are frontend configuration options for +specifying the main page URL and allowed origins for CORS. Overall, these properties enable +integration with multiple authentication providers and provide flexibility in handling user authentication. + +!!! warning + + All properties in `application-web.properties` profile must be filled in case of connecting some + frontend application. If you don't need to connect any frontend application, you can ignore this file. + +!!! info + + Sample frontend application is available at [LPVS repository](https://github.com/Samsung/LPVS/tree/main/frontend) + and can be used as a reference. + +--- + +## Command line options + +All missing properties from application profiles can be specified via command line options. +For example: + +```bash +java -jar -Dgithub.token= -Dgithub.secret= lpvs-*.jar +``` + +But there are several options that are not supported by this method and only must be set for the one-time scans +in command line mode. + +The following command line options are available: + +- `--build.html.report`: This setting specifies the path to the HTML report file which will be generated after the scan. +If it is not specified, no HTML report will be generated and result of the scan will be displayed in the console. + +- `--github.pull.request`: This setting specifies the pull request URL which should be scanned by the LPVS application. + +- `--local.path`: This setting specifies the path to the local file or folder which should be scanned by the LPVS application. + +!!! warning + + Options `--github.pull.request` and `--local.path` can't be use simultaneously. If both options are specified, + LPVS application will throw an exception and exit with error code. + +--- + +## Environment variables + +Alternatively, you can provide the necessary values for several properties using the following environment variables: + +- `LPVS_GITHUB_LOGIN`: Equivalent to the property `github.login`. +- `LPVS_GITHUB_TOKEN`: Equivalent to the property `github.token`. +- `LPVS_GITHUB_API_URL`: Equivalent to the property `github.api.url`. +- `LPVS_GITHUB_SECRET`: Equivalent to the property `github.secret`. +- `LPVS_LICENSE_CONFLICT`: Equivalent to the property `license_conflict`. diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index 3e32cfbc..617a0589 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -18,6 +18,8 @@ markdown_extensions: - pymdownx.superfences - pymdownx.tasklist: clickable_checkbox: true + - pymdownx.tabbed: + alternate_style: true plugins: - search @@ -29,6 +31,9 @@ nav: #- Getting Started: getting-started.md - User Guide: - User guide: user-guide/README.md + #- Prerequisites: + #- Scanner installation: + #- Database configuration: - Service mode: - GitHub webhook configuration: user-guide/service/webhook.md - Run service using pre-built Docker image: user-guide/service/docker.md @@ -36,9 +41,9 @@ nav: - Command line mode: - Single scan of the pull request: user-guide/cli/cli_pr.md - Single scan of local files: user-guide/cli/cli_file.md - - Configuration options: - # - Possible configuration options: user-guide/config/options.md - - Database description: user-guide/config/database.md + - Configuration: + - Configuration options: user-guide/config/options.md + - Database customization: user-guide/config/database.md - Developer Guide: - Developer guide: dev-guide/README.md - Contributing guide: dev-guide/contributing.md @@ -52,6 +57,7 @@ nav: extra_css: - css/extra.css + - css/mkdocsoad.css exclude_docs: | *.py From 0c002201c1f7ba315e9e15c0965cfb8048f76083 Mon Sep 17 00:00:00 2001 From: Oleg Kopysov Date: Fri, 6 Sep 2024 08:23:25 +0300 Subject: [PATCH 4/5] docs: Add Build and run, DB configuration, Scanner installation pages Signed-off-by: Oleg Kopysov --- doc/docs/dev-guide/integration.md | 2 +- doc/docs/user-guide/README.md | 9 +- doc/docs/user-guide/cli/cli_file.md | 15 +- doc/docs/user-guide/cli/cli_pr.md | 15 +- doc/docs/user-guide/service/build-and-run.md | 39 ++ doc/docs/user-guide/service/database.md | 49 +++ doc/docs/user-guide/service/scanner.md | 31 ++ doc/mkdocs.yml | 9 +- doc/quick-start-guide-and-build.md | 362 ------------------- 9 files changed, 150 insertions(+), 381 deletions(-) create mode 100644 doc/docs/user-guide/service/build-and-run.md create mode 100644 doc/docs/user-guide/service/database.md create mode 100644 doc/docs/user-guide/service/scanner.md delete mode 100644 doc/quick-start-guide-and-build.md diff --git a/doc/docs/dev-guide/integration.md b/doc/docs/dev-guide/integration.md index 7a8dd9e8..24542cd3 100644 --- a/doc/docs/dev-guide/integration.md +++ b/doc/docs/dev-guide/integration.md @@ -60,7 +60,7 @@ Example `settings.xml`: !!! note - if your `settings.xml` file is not located in `~/.m2/settings.xml`, then you need + If your `settings.xml` file is not located in `~/.m2/settings.xml`, then you need to add the `-s path/to/file/settings.xml` option to `mvn` command --- diff --git a/doc/docs/user-guide/README.md b/doc/docs/user-guide/README.md index fa5df056..35d4b492 100644 --- a/doc/docs/user-guide/README.md +++ b/doc/docs/user-guide/README.md @@ -12,16 +12,13 @@ There are 2 modes of LPVS operation: - Service mode - LPVS works as a service that is started by the system and runs continuously. - Command line mode - LPVS is executed from the command line and runs until it completes its task. -## Prerequisites - -- [Scanner installation] -- [Database configuration] - ## Service mode - [GitHub webhook configuration](service/webhook.md) - [Run service using pre-built Docker image](service/docker.md) -- [Build and run service from the source code](service/source_code.md) +- [Scanner installation](service/scanner.md) +- [Database configuration](service/database.md) +- [Build and run service from the source code](service/build-and-run.md) ## Command line mode diff --git a/doc/docs/user-guide/cli/cli_file.md b/doc/docs/user-guide/cli/cli_file.md index af642931..b0b878d5 100644 --- a/doc/docs/user-guide/cli/cli_file.md +++ b/doc/docs/user-guide/cli/cli_file.md @@ -5,11 +5,18 @@ One-time scan on a specific local files from the command line. --- You can perform a one-time scan on a specific local file or folder using the single scan mode. -LPVS should be compiled from the source code. Please refer to the [detailed guideline](../../user-guide/service/source_code.md). +LPVS should be compiled from the source code. Please refer to the [detailed guideline](../../user-guide/service/build-and-run.md). Follow the next steps: -* Install LPVS and navigate to the target directory. +* Make sure that the ScanOSS scanner is installed. If not, please follow + the [installation guide](../../user-guide/service/scanner.md). + +* Fill in all required properties in the profile file. Please refer to + the [profile configuration guide](../config/options.md). + +* Build and install LPVS, navigate to the target directory. For more details, + please refer to the [build instructions](../service/build-and-run.md). ```bash mvn clean install @@ -30,8 +37,8 @@ Example: !!! note - By default, the above commands require a pre-configured MySQL database. Use the "singlescan" profile to skip - setting up a pre-configured MySQL database: + By default, the above commands require a [pre-configured MySQL database](../../user-guide/service/database.md). + Use the "singlescan" profile to skip setting up a pre-configured MySQL database: ```bash java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path= diff --git a/doc/docs/user-guide/cli/cli_pr.md b/doc/docs/user-guide/cli/cli_pr.md index 1a6073e1..9fa63262 100644 --- a/doc/docs/user-guide/cli/cli_pr.md +++ b/doc/docs/user-guide/cli/cli_pr.md @@ -5,11 +5,18 @@ One-time scan on a specific pull request from the command line. --- You can perform a one-time scan on a specific pull request using the single scan mode. -LPVS should be compiled from the source code. Please refer to the [detailed guideline](../../user-guide/service/source_code.md). +LPVS should be compiled from the source code. Please refer to the [detailed guideline](../../user-guide/service/build-and-run.md). Follow the next steps: -* Install LPVS and navigate to the target directory. +* Make sure that the ScanOSS scanner is installed. If not, please follow + the [installation guide](../../user-guide/service/scanner.md). + +* Fill in all required properties in the profile file. Please refer to + the [profile configuration guide](../config/options.md). + +* Build and install LPVS, navigate to the target directory. For more details, + please refer to the [build instructions](../service/build-and-run.md). ```bash mvn clean install @@ -30,8 +37,8 @@ Example: !!! note - By default, the above commands require a pre-configured MySQL database. Use the "singlescan" profile to skip - setting up a pre-configured MySQL database: + By default, the above commands require a [pre-configured MySQL database](../../user-guide/service/database.md). + Use the "singlescan" profile to skip setting up a pre-configured MySQL database: ```bash java -jar -Dspring.profiles.active=singlescan -Dgithub.token= lpvs-*.jar --github.pull.request= diff --git a/doc/docs/user-guide/service/build-and-run.md b/doc/docs/user-guide/service/build-and-run.md new file mode 100644 index 00000000..0b760d71 --- /dev/null +++ b/doc/docs/user-guide/service/build-and-run.md @@ -0,0 +1,39 @@ +# Build and run service from the source code + +Basic steps to build and run LPVS. + +--- + +To build and run the LPVS application you have to follow the next steps: + +* Fill in the properties file according to the selected profile with your own data. Refer to the [guideline](../config/options.md#properties-in-the-application-profile-files). + +!!! note + + Alternatively, you can provide the necessary values using the [environment variables](../config/options.md#environment-variables), + or using [command line options](../config/options.md#command-line-options). + +* Build the LPVS application using [Maven](https://maven.apache.org/). + +```bash +mvn clean install +``` + +* Navigate to the target directory. + +```bash +cd target/ +``` + +* Run the LPVS service using the following command: + +```bash +java -jar lpvs-*.jar +``` + +LPVS is now built and running. You can create a new pull request or update an existing one with commits, +and LPVS will automatically start scanning and provide comments about the licenses found in the project. + +!!! warning + + Make sure that you configured GitHub webhooks according to the [guideline](webhook.md). diff --git a/doc/docs/user-guide/service/database.md b/doc/docs/user-guide/service/database.md new file mode 100644 index 00000000..404afef7 --- /dev/null +++ b/doc/docs/user-guide/service/database.md @@ -0,0 +1,49 @@ +# Database configuration + +How to install and configure the LPVS database. + +--- + +## Installation + +Install MySQL server locally using the command: + +```bash +sudo apt install mysql-server +``` + +--- + +## Configuration + +* Start the MySQL server: + +```bash +sudo service mysql start +``` + +* Open the MySQL command line interface: + +```bash +sudo mysql +``` + +* Run the following commands in the MySQL command line interface to create the necessary database and user: + +```sql +mysql> create database lpvs; +mysql> create user username; +mysql> grant all on lpvs.* to username; +mysql> alter user username identified by 'password'; +mysql> exit; +``` + +* (**Optional**) If you have an existing dump file, import it into the newly created database using the command: + +```bash +mysql -u[username] -p[password] < /path/to/dump/file/database_dump.sql +``` + +* (**Optional**) Fill in the `lpvs_license_list` and `lpvs_license_conflicts` tables according to the [Database customization guideline](../config/database.md). + +* Update the properties related to the database operation in the profile file according to the [guideline](../config/options.md). diff --git a/doc/docs/user-guide/service/scanner.md b/doc/docs/user-guide/service/scanner.md new file mode 100644 index 00000000..79895781 --- /dev/null +++ b/doc/docs/user-guide/service/scanner.md @@ -0,0 +1,31 @@ +# Scanner installation + +How to install scan engine ScanOSS. + +--- + +The ScanOSS Python package provides a simple easy to consume library for interacting with ScanOSS APIs/Engine. + +To install ScanOSS and dependencies, use the command: + +```bash +pip3 install --require-hashes -r requirements.txt +``` + +!!! info + + File `requirements.txt` could be found at the [root folder](https://github.com/Samsung/LPVS/blob/main/requirements.txt) + of the LPVS repository. + +Make sure that the `PATH` variable is added to the environment: + +```bash +export PATH="$HOME/.local/bin:$PATH" +``` + +For more details, please refer to the [official guideline](https://github.com/scanoss/scanoss.py#installation). + +!!! warning + + If installing on Ubuntu 2023.04, Fedora 38, Debian 11, etc. a few additional steps are required before + installing ScanOSS. More details can be found [here](https://itsfoss.com/externally-managed-environment/). diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index 617a0589..556eb6d1 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -31,13 +31,14 @@ nav: #- Getting Started: getting-started.md - User Guide: - User guide: user-guide/README.md - #- Prerequisites: - #- Scanner installation: - #- Database configuration: - Service mode: - GitHub webhook configuration: user-guide/service/webhook.md - Run service using pre-built Docker image: user-guide/service/docker.md - #- Build and run service from the source code: user-guide/service/source_code.md + - Build and run service from the source code: + - Prerequisites: + - Scanner installation: user-guide/service/scanner.md + - Database configuration: user-guide/service/database.md + - Build and run service: user-guide/service/build-and-run.md - Command line mode: - Single scan of the pull request: user-guide/cli/cli_pr.md - Single scan of local files: user-guide/cli/cli_file.md diff --git a/doc/quick-start-guide-and-build.md b/doc/quick-start-guide-and-build.md deleted file mode 100644 index 6b237413..00000000 --- a/doc/quick-start-guide-and-build.md +++ /dev/null @@ -1,362 +0,0 @@ -# Quick Start Guide & Build - -## Contents - ### [Quick Start Guide](#quick-start-guide-1) - 1. [Setting up your project to interact with _LPVS_](#1-setting-up-your-project-to-interact-with-lpvs) - 2. [Using pre-built _LPVS_ Docker images](#2-using-pre-built-lpvs-docker-images) - - ### [How to Build and Run _LPVS_ from Source Code](#how-to-build-and-run-lpvs-from-source-code-1) - 1. [Build Prerequisites](#1-build-prerequisites) - 2. [Create Necessary MySQL Database and User (optional if not using a database)](#2-create-necessary-mysql-database-and-user-optional-if-not-using-a-database) - 3. [Setting up _LPVS_ application.properties](#3-setting-up-lpvs-applicationproperties) - 4. [Build _LPVS_ Application with Maven and Run it](#4-build-lpvs-application-with-maven-and-run-it) - ---- - -## Quick Start Guide - -### 1. Setting up your project to interact with _LPVS_ - -To enable _LPVS_ license scanning for your project, you need to set up GitHub Webhooks: - -1.1 Create a personal github access token (`personal-token`): - - Follow the instructions [here](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token#creating-a-fine-grained-personal-access-token) to create a personal access token with the necessary permissions. - - > [!NOTE] - > Pay attention that the token must be copied immediately after creation, because you will not be able to see it later!! - -2.1 Configure the webhook in your GitHub repository settings: - - Go to `Settings` -> `Webhooks`. - - Click on `Add webhook`. - - Fill in the `Payload URL` with: `http://:7896/webhooks`. - > If you're using ngrok, the `Payload URL` should be like `https://50be-62-205-136-206.ngrok-free.app/webhooks`. - - Install ngrok and connect your account from [here](https://ngrok.com/docs/getting-started/#step-2-install-the-ngrok-agent) (follow steps 1 and 2). - - Run ngrok using the command: `ngrok http 7896`. - - Specify the content type as `application/json`. - - Fill in the `Secret` field with the passphrase: `LPVS`. - - Save the same passphrase in `github.secret` of the _LPVS_ backend `application.properties` or `docker-compose.yml` files. - - Select `Let me select individual events` -> `Pull requests` (make sure only `Pull requests` is selected). - - Set the webhook to `Active`. - - Click `Add Webhook`. - -Configuration from your project side is now complete! - -Alternatively, you can use the Pull Request Single Scan API to analyze the code of a specific pull request. -Please refer to the [API Documentation](lpvs-api.yaml) for more information. - ---- - -### 2. Using pre-built _LPVS_ Docker images - -This section explains how to download and run pre-built _LPVS_ Docker images without building the _LPVS_ project. - -For the Docker deployment scenario, you may need to fill in the environment variables in the `docker-compose.yml` file. - -#### 2.1 Setting up _LPVS_ Docker environment variables - -2.1.1 Open `docker-compose.yml` file. - -2.1.2 In the `environment` part of the `lpvs` service, find `## Github data for fetching code` and fill in the github `login` and personal `token` that was generated earlier - -```yaml -- github.login= -- github.token= -``` - -2.1.3 In case you plan to use a database user other than `root` reflect this in the appropriate lines in the `## Database Configuration` part of the `lpvs` service in `environment` section: - -```yaml -- spring.datasource.username=user -- spring.datasource.password=password -``` - -2.1.4 Make the following changes in the `environment` section of `mysqldb` service near `MYSQL_ROOT_PASSWORD` value: - -```yaml -- MYSQL_USER: user -- MYSQL_PASSWORD: password -``` - -If you are using only the `root` user, make the following change: - -```yaml -- spring.datasource.username=root -- spring.datasource.password=rootpassword -``` -```yaml -- MYSQL_ROOT_PASSWORD: rootpassword -``` - -In both cases, ensure that the `MYSQL_ROOT_PASSWORD` field is filled. - -2.1.5 You can also change the directory for storing MySQL data by modifying the following line: - -```yaml -- ./mysql-lpvs-data:/var/lib/mysql # db storage by default it is a directory in the root of the repository with the name mysql-lpvs-data -``` - -#### 2.2 Running _LPVS_ and MySQL Docker images with Docker Compose - -Start the _LPVS_ services using `docker-compose` (before Compose V2): - -```bash -docker-compose up -d -``` - -Start the _LPVS_ services using `docker compose` (after Compose V2): - -```bash -docker compose up -d -``` - -Stop the _LPVS_ services using `docker-compose` (before Compose V2): - -```bash -docker-compose down -v --rmi local -``` - -Stop the _LPVS_ services using `docker compose` (after Compose V2): - -```bash -docker compose down -``` - -You can now create a new pull request or update an existing one with commits. _LPVS_ will automatically start scanning and provide comments about the licenses found in the project. - ---- - -## How to Build and Run _LPVS_ from Source Code - -### 1. Build Prerequisites - -Before building _LPVS_ from source code, ensure that you have the following prerequisites installed: - -- SCANOSS Python package by following the [guidelines](https://github.com/scanoss/scanoss.py#installation). Install it using the command: - ```bash - pip3 install scanoss - ``` - Make sure that the path variable is added to the environment: - ```bash - export PATH="$HOME/.local/bin:$PATH" - ``` - -- MySQL server installed locally. Install it using the command: - ```bash - sudo apt install mysql-server - ``` - -### 2. Create Necessary MySQL Database and User (optional if not using a database) - -2.1 Start the MySQL server: - ```bash - sudo service mysql start - ``` - -2.2 Open the MySQL command line interface: - ```bash - sudo mysql - ``` - -2.3 Run the following commands in the MySQL command line interface to create the necessary database and user: - ```sql - mysql> create database lpvs; - mysql> create user username; - mysql> grant all on lpvs.* to username; - mysql> alter user username identified by 'password'; - mysql> exit; - ``` - -2.4 (Optional) If you have an existing dump file, import it into the newly created database using the command: - ```bash - mysql -u[username] -p[password] < src/main/resources/database_dump.sql - ``` - -2.5 Fill in the `lpvs_license_list` and `lpvs_license_conflicts` tables with the information about permitted, restricted, and prohibited licenses, as well as their compatibility specifics. You can find an example database dump file in the repository at [`src/main/resources/database_dump.sql`](../src/main/resources/database_dump.sql). - -2.6 Update the following lines in the [`src/main/resources/application.properties`](../src/main/resources/application.properties) file: - ```properties - spring.datasource.username=username - spring.datasource.password=password - ``` - -### 3. Setting up _LPVS_ `application.properties` - -Fill in the following lines in the [`src/main/resources/application.properties`](../src/main/resources/application.properties) file: - -```properties -# GitHub configuration (github.token and github.secret required) -github.token= -github.login= -github.api.url= -github.secret=LPVS -``` - > [!NOTE] - > For personal GitHub account use `https://api.github.com` in field `github.api.url=`. - -```text -# Used license scanner: scanoss (at the moment, only this scanner is supported) -scanner=scanoss - -# Used license conflicts source: -# > option "db": take conflicts from MySQL database - 'lpvs_license_conflicts' table (should be filled manually -# according to the example at 'src/main/resources/database_dump.sql') -# > option "scanner": take conflicts from the scanner response - license_conflict=db - -# DB configuration (username and password) -... -spring.datasource.username= -spring.datasource.password= -``` - -Alternatively, you can provide the necessary values using the following environment variables: `LPVS_GITHUB_LOGIN`, `LPVS_GITHUB_TOKEN`, `LPVS_GITHUB_API_URL`, `LPVS_GITHUB_SECRET`, and `LPVS_LICENSE_CONFLICT`. - -### 4. Build _LPVS_ Application with Maven and Run it - -#### 4.1 Service mode (default) - -To build _LPVS_ from source code and run it in the default service mode, follow these steps: - -4.1.1 Build the _LPVS_ application using Maven: - ```bash - mvn clean install - ``` - -4.1.2 Navigate to the target directory: - ```bash - cd target/ - ``` - -4.1.3 Run the _LPVS_ application. - - Service is run using the following command: - ```bash - java -jar lpvs-*.jar - ``` - - Alternatively, you can provide the necessary values associated with GitHub and license using the command line: - ```bash - java -jar -Dgithub.token= -Dgithub.secret= lpvs-*.jar - ``` - > [!NOTE] - > Use `LPVS` as the value for the `-Dgithub.secret=` parameter. - -_LPVS_ is now built and running. You can create a new pull request or update an existing one with commits, and _LPVS_ will automatically start scanning and provide comments about the licenses found in the project. - -#### 4.2 Single scan mode - -Alternatively, you can perform a one-time scan on a specific pull request or local files using the single scan mode. Follow these steps: - -4.2.1 Install and navigate to the target directory as described in step 4.1.1 and 4.1.2: - - ```bash - mvn clean install - cd target/ - ``` - -4.2.2 Choose either to scan a specific pull request from GitHub or local files. - -4.2.2.1 To scan a specific pull request from GitHub, execute the following command: - - ```bash - java -jar -Dgithub.token= lpvs-*.jar --github.pull.request= - ``` - -4.2.2.2 To scan local files or directories, execute the following command: - - ```bash - java -jar lpvs-*.jar --local.path= - ``` - -4.2.3 By default, the above commands require a pre-configured MySQL database. Use the "singlescan" profile to skip setting up a pre-configured MySQL database: - - ```bash - java -jar -Dspring.profiles.active=singlescan -Dgithub.token= lpvs-*.jar --github.pull.request= - java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path= - ``` - -4.2.4 Optionally, generate an HTML report and save it in a specified folder. Replace `path/to/your/folder` with the full path to the folder where you want to save the HTML report, and `your_report_filename.html` with the desired filename for the report. - - ```bash - java -jar -Dspring.profiles.active=singlescan -Dgithub.token= lpvs-*.jar --github.pull.request= --build.html.report= - java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path= --build.html.report= - ``` - -Note: Ensure that the specified folder exists before generating the HTML report. - -4.2.5 Examples of commands: - - ```bash - java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --github.pull.request=https://github.com/Samsung/LPVS/pull/2 - java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --github.pull.request=https://github.com/Samsung/LPVS/pull/2 --build.html.report=report.html - java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path=test.c - java -jar -Dspring.profiles.active=singlescan lpvs-*.jar --local.path=test --build.html.report=test/report.html - ``` - -#### 4.3 Use of _LPVS_ JAR `lpvs-x.y.z.jar` in your project - -4.3.1 Authenticating with a personal access token - -You can authenticate to GitHub Packages with Apache Maven by editing your `~/.m2/settings.xml` file to include your personal access token -> [!NOTE] -> Create a token with minimally sufficient rights: -> - Fine-grained tokens **(recommended)** - _Only select repositories -> Permissions -> Repository permissions -> Metadata -> Read-only_ -> - Tokens (classic) - _Select scopes -> read:packages_ - -Create a new `~/.m2/settings.xml` file if one doesn't exist. - -Example `settings.xml` -``` - - - - github - - - - - github - - - github - https://maven.pkg.github.com/samsung/lpvs - - true - - - - - - - - - github - USERNAME - TOKEN - - - -``` -> [!NOTE] -> if your `settings.xml` file is not located in `~/.m2/settings.xml`, then you need to add the `-s path/to/file/settings.xml` option to `mvn` command - -4.3.2 Installing a package - -Edit the `` element of the `pom.xml` file located in your project directory. - -``` -... - - - com.lpvs - lpvs - x.y.z - - -... -``` From b2f16257fe44192d338b2c5fd61a0a0a629376ce Mon Sep 17 00:00:00 2001 From: Oleg Kopysov Date: Fri, 6 Sep 2024 10:46:42 +0300 Subject: [PATCH 5/5] fix: Fix Integration page Signed-off-by: Oleg Kopysov --- doc/docs/dev-guide/integration.md | 4 ++-- doc/mkdocs.yml | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/doc/docs/dev-guide/integration.md b/doc/docs/dev-guide/integration.md index 24542cd3..4a4448f4 100644 --- a/doc/docs/dev-guide/integration.md +++ b/doc/docs/dev-guide/integration.md @@ -4,7 +4,7 @@ Use of LPVS JAR in your project --- -## 1. Authenticating with a personal access token +## Authenticating with a personal access token You can authenticate to GitHub Packages with Apache Maven by editing your `~/.m2/settings.xml` file to include your personal access token. @@ -65,7 +65,7 @@ Example `settings.xml`: --- -## 2 Installing a package +## Installing a package Edit the `` element of the `pom.xml` file located in your project directory. diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index 556eb6d1..6b1b6b32 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -65,14 +65,4 @@ exclude_docs: | use_directory_urls: true -#extra: -# # files to be copied from repository root to docs_path -# files_to_copy: -# - contributing.md -# # path to documentation root -# docs_path: docs/about - -#hooks: -# - docs/hooks.py - copyright: Copyright © 2024, Samsung Electronics Co., Ltd. All rights reserved.