diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml index 24ffe1695b53..f944c8170539 100644 --- a/.github/workflows/checkstyle.yml +++ b/.github/workflows/checkstyle.yml @@ -1,4 +1,4 @@ -name: Checkstyle, Findbugs, Doc Check, etc. +name: Checkstyle, Findbugs, Doc Check, SortPom, etc. on: [pull_request] @@ -8,7 +8,7 @@ jobs: steps: - name: checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 @@ -16,7 +16,7 @@ jobs: node-version: '10.11.0' - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -24,24 +24,23 @@ jobs: ${{ runner.os }}-maven- - name: Cache local Go modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }} - - name: Execute license check, checkstyle, findbugs + - name: Execute license check, checkstyle, findbugs, sortpom run: | mkdir -p ~/.m2 ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ ALLUXIO_CHECKSTYLE=true \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl tests + dev/github/run_docker.sh timeout-minutes: 60 - name: Archive artifacts continue-on-error: true - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: always() with: name: artifact diff --git a/.github/workflows/fuse_integration_tests.yml b/.github/workflows/fuse_integration_tests.yml deleted file mode 100644 index ea9422bf8ded..000000000000 --- a/.github/workflows/fuse_integration_tests.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Fuse Integration Tests - -on: [pull_request] - -jobs: - build: - name: "modules: fuse" - - strategy: - fail-fast: false - matrix: - java: ["8", "11"] - version: ["2", "3"] - - runs-on: ubuntu-latest - if: "!contains(github.event.pull_request.title, 'DOCFIX') && - !contains(github.event.pull_request.title, 'SKIPCI')" - - steps: - - name: checkout repo - uses: actions/checkout@v2 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: '10.11.0' - - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-java${{ matrix.java }}-${{ hashFiles('**/pom.xml') }} - - - name: Cache local Go modules - uses: actions/cache@v2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }} - - - name: Run tests - id: test0 - run: | - mkdir -p ~/.m2 - ALLUXIO_DOCKER_NO_TTY=true \ - ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!shaded/client,!integration/tools/hms,!integration/yarn,!assembly/client,!assembly/server,!table/server/underdb/glue,!underfs/hdfs,!underfs/ozone,!underfs/adl,!underfs/abfs,!underfs/cosn,!underfs/wasb,!underfs/cos,!underfs/kodo,!underfs/oss,!underfs/swift \ - dev/github/run_docker.sh "\"-Dtest=alluxio.client.fuse.**\"" "\"-Dalluxio.fuse.jnifuse.libfuse.version=${{ matrix.version }}\"" -pl tests - timeout-minutes: 60 - - - name: Archive artifacts - continue-on-error: true - uses: actions/upload-artifact@v2 - if: always() - with: - name: artifact - path: | - **/target/surefire-reports/* - **/target/artifacts/* - **/target/logs/* - retention-days: 7 diff --git a/.github/workflows/java8_container_tests.yml b/.github/workflows/java8_container_tests.yml new file mode 100644 index 000000000000..e6615929fc24 --- /dev/null +++ b/.github/workflows/java8_container_tests.yml @@ -0,0 +1,66 @@ +name: Java 8 Container Tests + +# This build runs integration tests using testcontainers +# The user executing the test command must have access to a running docker daemon + +on: [pull_request] + +jobs: + build: + name: "modules: " + + strategy: + fail-fast: false + matrix: + modules: + - >- + alluxio.membership.** + + runs-on: ubuntu-latest + if: "!contains(github.event.pull_request.title, 'DOCFIX') && + !contains(github.event.pull_request.title, 'SKIPCI')" + + steps: + - name: checkout repo + uses: actions/checkout@v3 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: '10.11.0' + + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }} + + - name: Cache local Go modules + uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }} + + - name: Run tests + id: test0 + run: | + mkdir -p ~/.m2 + ALLUXIO_DOCKER_NO_TTY=true \ + ALLUXIO_DOCKER_GIT_CLEAN=true \ + ALLUXIO_DOCKER_ID=0 \ + ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests/testcontainers \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh + timeout-minutes: 60 + + - name: Archive artifacts + continue-on-error: true + uses: actions/upload-artifact@v3 + if: always() + with: + name: artifact + path: | + **/target/surefire-reports/* + **/target/artifacts/* + **/target/logs/* + retention-days: 7 diff --git a/.github/workflows/java8_integration_tests.yml b/.github/workflows/java8_integration_tests.yml index c10b5f648adc..6ae55c3af9a8 100644 --- a/.github/workflows/java8_integration_tests.yml +++ b/.github/workflows/java8_integration_tests.yml @@ -1,5 +1,11 @@ name: Java 8 Integration Tests +# This build runs the default integration tests in the dora/tests module +# It runs all the tests besides those categorized as: +# - Fault tolerance = alluxio.server.ft +# - Web UI = alluxio.web +# - FUSE = alluxio.client.fuse + on: [pull_request] jobs: @@ -11,15 +17,11 @@ jobs: matrix: modules: - >- - alluxio.client.cli.**,!alluxio.client.cli.fs.** - - >- - alluxio.client.cli.fs.** - - >- - alluxio.client.fs.**,!alluxio.client.fs.concurrent.**,!alluxio.client.fs.io.** + alluxio.client.cli.** - >- - alluxio.client.fs.concurrent.**,alluxio.client.fs.io.** + alluxio.client.fs.** - >- - alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**,!alluxio.client.fuse.** + alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.** - >- alluxio.job.**,alluxio.master.**,alluxio.stress.** - >- @@ -31,7 +33,7 @@ jobs: steps: - name: checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 @@ -39,13 +41,13 @@ jobs: node-version: '10.11.0' - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-java8-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }} - name: Cache local Go modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }} @@ -56,14 +58,14 @@ jobs: mkdir -p ~/.m2 ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!shaded/client,!integration/tools/hms,!integration/yarn,!assembly/client,!assembly/server,!table/server/underdb/glue,!underfs/hdfs,!underfs/ozone,!underfs/adl,!underfs/abfs,!underfs/cosn,!underfs/wasb,!underfs/cos,!underfs/kodo,!underfs/oss,!underfs/swift \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl tests + ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests/integration \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh timeout-minutes: 60 - name: Archive artifacts continue-on-error: true - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: always() with: name: artifact diff --git a/.github/workflows/java8_integration_tests_ft.yml b/.github/workflows/java8_integration_tests_ft.yml index 1a22af092bbb..afec49a09314 100644 --- a/.github/workflows/java8_integration_tests_ft.yml +++ b/.github/workflows/java8_integration_tests_ft.yml @@ -1,5 +1,10 @@ name: Java 8 Fault Tolerant Integration Tests +# This build runs the fault tolerance specific integration tests in the dora/tests module +# The key differences between this build and the default integration test build are +# - Only runs integration tests with prefix alluxio.server.ft +# - Sets fork count to 1 to prevent flakey errors due to running concurrent tests; default count is 2 + on: [pull_request] jobs: @@ -21,7 +26,7 @@ jobs: steps: - name: checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 @@ -29,13 +34,13 @@ jobs: node-version: '10.11.0' - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-java8-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }} - name: Cache local Go modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }} @@ -48,14 +53,14 @@ jobs: ALLUXIO_DOCKER_FORK_COUNT=1 \ ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!shaded/client,!integration/tools/hms,!integration/yarn,!assembly/client,!assembly/server,!table/server/underdb/glue,!underfs/hdfs,!underfs/ozone,!underfs/adl,!underfs/abfs,!underfs/cosn,!underfs/wasb,!underfs/cos,!underfs/kodo,!underfs/oss,!underfs/swift \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl tests + ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests/integration \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh timeout-minutes: 60 - name: Archive artifacts continue-on-error: true - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: always() with: name: artifact diff --git a/.github/workflows/java8_integration_tests_webui.yml b/.github/workflows/java8_integration_tests_webui.yml index 85c4ba987062..4276ecd6e8fc 100644 --- a/.github/workflows/java8_integration_tests_webui.yml +++ b/.github/workflows/java8_integration_tests_webui.yml @@ -1,5 +1,10 @@ name: Java 8 Integration Tests (w/ webui) +# This build runs the integration tests in the dora/tests module that require the web UI +# The key differences between this build and the default integration test build are +# - Only runs integration tests with prefix alluxio.web +# - Adds webui to the project list + on: [pull_request] jobs: @@ -19,7 +24,7 @@ jobs: steps: - name: checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 @@ -27,13 +32,13 @@ jobs: node-version: '10.11.0' - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-java8-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }} - name: Cache local Go modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }} @@ -44,14 +49,14 @@ jobs: mkdir -p ~/.m2 ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!shaded/client,!integration/tools/hms,!integration/yarn,!assembly/client,!assembly/server \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl tests + ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests/integration,webui \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh timeout-minutes: 60 - + - name: Archive artifacts continue-on-error: true - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: always() with: name: artifact diff --git a/.github/workflows/java8_unit_tests.yml b/.github/workflows/java8_unit_tests.yml index 833688c22806..04adccdb0fd0 100644 --- a/.github/workflows/java8_unit_tests.yml +++ b/.github/workflows/java8_unit_tests.yml @@ -21,7 +21,7 @@ jobs: steps: - name: checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 @@ -29,13 +29,13 @@ jobs: node-version: '10.11.0' - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-java8-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }} - name: Cache local Go modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }} @@ -46,14 +46,14 @@ jobs: mkdir -p ~/.m2 ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!shaded/client,!assembly/client,!assembly/server \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl '!tests,!webui' + ALLUXIO_DOCKER_MVN_PROJECT_LIST=\!assembly/client,\!assembly/server,\!dora/tests/integration,\!dora/tests/testcontainers,\!dora/microbench,\!webui \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh timeout-minutes: 60 - name: Archive artifacts continue-on-error: true - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: always() with: name: artifact diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml new file mode 100644 index 000000000000..72ba9f046749 --- /dev/null +++ b/.github/workflows/stale.yaml @@ -0,0 +1,36 @@ +name: "Mark stale issues and PRs" +on: + schedule: + # Run the stalebot every day at 3pm UTC + - cron: "00 15 * * *" + +permissions: + contents: read + +jobs: + stale: + permissions: + issues: write # for writing stale message + pull-requests: write # for writing stale message + runs-on: ubuntu-22.04 + if: github.repository == 'alluxio/alluxio' + steps: + - uses: actions/stale@v6 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + ascending: true # old issues/PRs first + operations-per-run: 1000 # default is 30, enlarge for dealing with more issues/PRs + days-before-stale: 30 + days-before-close: -1 + stale-issue-message: > + This issue has been automatically marked as stale because it has not had recent activity. + It will be closed in two weeks if no further activity occurs. + Thank you for your contributions. + stale-pr-message: > + This pull request has been automatically marked as stale because it has not had + recent activity. It will be closed in two weeks if no further activity occurs. + Thank you for your contributions. + stale-pr-label: "stale" + stale-issue-label: "stale" + exempt-issue-labels: "keepalive,priority-high" + exempt-pr-labels: "keepalive,priority-high" diff --git a/.gitignore b/.gitignore index aa46e7430854..0557b4ec6a98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ **/.port_coordination -**/alluxio-site.properties +**/alluxio-site.properties* +!**/alluxio-site.properties.template **/nb-configuration*.xml **/nbproject/ *.iml @@ -17,37 +18,24 @@ .port_coordination/ .project .settings/ +.java-version /client/ /conf/alluxio-env.sh /conf/alluxio-env.sh.bootstrap.bk /conf/metrics.properties /conf/rocks-*.ini /data/ -/dev/scripts/*.tar.gz -/dev/scripts/tarballs -/dev/scripts/workdir /docs/_site/ /docs/api/ /docs/serve/ /extensions/ /generated/ /integration/docker/*.tar.gz -/integration/vagrant/.request_ids -/integration/vagrant/.vagrant/ -/integration/vagrant/conf/ec2.yml -/integration/vagrant/conf/gce.yml -/integration/vagrant/conf/init.yml -/integration/vagrant/files/ -/integration/vagrant/shared/ -/integration/vagrant/spot/roles/create_ec2/vars/ -/integration/vagrant/spot/roles/tag_ec2/vars/ -/integration/vagrant/vbox/.vagrant/ -/integration/vagrant/vbox/alluxio-dev.box +/job_results/ /journal/ /lib/*.jar -/logs/*.log* -/logs/*.out* -/logs/user/* +/libexec/version.sh +/logs/ /metastore /underFSStorage/ /underfs/hdfs/src/main/java/alluxio/UfsConstants.java @@ -71,3 +59,4 @@ dependency-reduced-pom.xml target/ tests.log* +worker_identity diff --git a/README.md b/README.md index aeb94026ac50..ad5d1fd0e2e6 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Release](https://img.shields.io/github/release/alluxio/alluxio/all.svg)](https://www.alluxio.io/download) [![Docker Pulls](https://img.shields.io/docker/pulls/alluxio/alluxio.svg)](https://hub.docker.com/r/alluxio/alluxio) [![Documentation](https://img.shields.io/badge/docs-reference-blue.svg)](https://www.alluxio.io/docs) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/Alluxio/alluxio/badge)](https://api.securityscorecards.dev/projects/github.com/Alluxio/alluxio) [![Twitter Follow](https://img.shields.io/twitter/follow/alluxio.svg?label=Follow&style=social)](https://twitter.com/intent/follow?screen_name=alluxio) [![License](https://img.shields.io/github/license/alluxio/alluxio.svg)](https://github.com/Alluxio/alluxio/blob/master/LICENSE) @@ -105,7 +106,7 @@ Alluxio project provides several different client artifacts for external project [HDFS-Compatible file system API](https://docs.alluxio.io/os/user/stable/en/api/Java-API.html#hadoop-compatible-java-client). This artifact is included in `alluxio-shaded-client`. -Here are examples to declare the dependecies on `alluxio-shaded-client` using Maven: +Here are examples to declare the dependencies on `alluxio-shaded-client` using Maven: ```xml diff --git a/assembly/README.md b/assembly/README.md index b877e3ed848e..271a537d00a6 100644 --- a/assembly/README.md +++ b/assembly/README.md @@ -24,4 +24,4 @@ should be included. ## Usage For more details on how to use the generated tar, go -[to the docs](https://docs.alluxio.io/os/user/stable/en/deploy/Running-Alluxio-Locally.html). +[to the docs](https://docs.alluxio.io/os/user/stable/en/overview/Getting-Started.html). diff --git a/assembly/client/pom.xml b/assembly/client/pom.xml index 6a0963086242..5480e853f700 100644 --- a/assembly/client/pom.xml +++ b/assembly/client/pom.xml @@ -1,3 +1,4 @@ + org.alluxio alluxio-core-client-fs @@ -66,16 +66,16 @@ uber-jar - package shade + package ${project.artifactId}-${project.version}-jar-with-dependencies - - - + + + @@ -85,7 +85,7 @@ - *:* + *:* LICENSE META-INF/*.SF diff --git a/assembly/pom.xml b/assembly/pom.xml index 1a22a707deaf..6ca3c7dbc540 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -1,3 +1,4 @@ + ${project.parent.basedir}/build false + + + client + server + diff --git a/assembly/server/pom.xml b/assembly/server/pom.xml index b531ec810194..63be667f0354 100644 --- a/assembly/server/pom.xml +++ b/assembly/server/pom.xml @@ -1,3 +1,4 @@ + org.alluxio alluxio-core-server-master @@ -56,11 +56,6 @@ alluxio-job-server ${project.version} - - org.alluxio - alluxio-logserver - ${project.version} - @@ -71,16 +66,16 @@ uber-jar - package shade + package ${project.artifactId}-${project.version}-jar-with-dependencies - - - + + + @@ -90,7 +85,7 @@ - *:* + *:* LICENSE META-INF/*.SF diff --git a/bin/alluxio b/bin/alluxio index 0e2488219b85..0806b5d221b4 100755 --- a/bin/alluxio +++ b/bin/alluxio @@ -10,396 +10,20 @@ # See the NOTICE file distributed with this work for information regarding copyright ownership. # -function printUsage { - echo "Usage: alluxio COMMAND [GENERIC_COMMAND_OPTIONS] [COMMAND_ARGS]" - echo - echo "COMMAND is one of:" - echo -e " format [-s] \t Format Alluxio master and all workers (if -s specified, only format if underfs is local and doesn't already exist)" - echo -e " formatJournal \t Format Alluxio master journal locally" - echo -e " formatMasters \t Format Alluxio master nodes" - echo -e " formatWorker \t Format Alluxio worker nodes" - echo -e " bootstrapConf \t Generate a config file if one doesn't exist" - echo -e " fs \t Command line tool for interacting with the Alluxio filesystem." - echo -e " fsadmin \t Command line tool for use by Alluxio filesystem admins." - echo -e " getConf [key] \t Look up a configuration key, or print all configuration." - echo -e " job \t Command line tool for interacting with the job service." - echo -e " logLevel \t Set or get log level of Alluxio servers." - echo -e " runClass \t Run the main method of an Alluxio class." - echo -e " runTest \t Run an end-to-end test on an Alluxio cluster." - echo -e " runTests \t Run all end-to-end tests on an Alluxio cluster." - echo -e " runHmsTests \t Test the integration between Alluxio and the target hive metastore. Try 'alluxio runHmsTests -h' for more help." - echo -e " \t NOTE: This command requires valid hive metastore uris to run tests against." - echo -e " runHdfsMountTests\t Test the integration between Alluxio and the target HDFS path. " - echo -e " runJournalCrashTest\t Test the Master Journal System in a crash scenario. Try 'alluxio runJournalCrashTest -help' for more help." - echo -e " \t NOTE: This command will stop the existing server and creates a new one!" - echo -e " runUfsIOTest \t Test the Alluxio throughput to the UFS path. Try 'alluxio runUfsIOTest -help' for more help." - echo -e " \t NOTE: This command requires a valid ufs path to write temporary files into." - echo -e " runUfsTests \t Test the integration between Alluxio and the target under filesystem. Try 'alluxio runUfsTests -help' for more help." - echo -e " \t NOTE: This command requires a valid ufs path to run tests against." - echo -e " readJournal \t Read an Alluxio journal file from stdin and write a human-readable version of it to stdout." - echo -e " upgradeJournal\t Upgrade an Alluxio journal version 0 (Alluxio version < 1.5.0) to an Alluxio journal version 1 (Alluxio version >= 1.5.0)." - echo -e " killAll [-s] \t Kill processes containing the WORD (if '-s' specified, processes won't be forcibly killed even if the operation exceeds the timeout)." - echo -e " copyDir \t Copy the PATH to all master/worker nodes." - echo -e " clearCache \t Clear OS buffer cache of the machine." - echo -e " docGen \t Generate docs automatically." - echo -e " version \t Print Alluxio version and exit." - echo -e " validateConf \t Validate Alluxio conf and exit." - echo -e " validateEnv \t Validate Alluxio environment." - echo -e " collectInfo \t Collects information to troubleshoot an Alluxio cluster." - echo - echo "GENERIC_COMMAND_OPTIONS supports:" - echo -e " -D\t Use a value for a given Alluxio property" - echo - # TODO(binfan): Fix help function for alluxio script - echo "Commands print help when invoked without parameters." -} - -function bootstrapConf { - local usage="Usage: $0 bootstrapConf " - if [[ $# -lt 1 ]]; then - echo ${usage} >&2 - exit 2 - fi - local master=${1} - - local dst="${ALLUXIO_CONF_DIR}/alluxio-site.properties" - - if [[ -e ${dst} ]]; then - echo "${dst} already exists" - return - fi - - cat > ${dst} <\t Kill processes containing the WORD (if '-s' specified, processes won't be forcibly killed even if the operation exceeds the timeout)." >&2 - exit 2 - fi - if [[ "$1" == "-s" ]]; then - keyword=$2 - soft_kill_option=true - else - keyword=$1 - soft_kill_option=false - fi - local success_count=0 - local failed_count=0 - for pid in $(ps -Aww -o pid,command | grep -i "[j]ava" | grep ${keyword} | awk '{print $1}'); do - kill -15 ${pid} > /dev/null 2>&1 - local maxWait=120 - local cnt=${maxWait} - fail_to_kill=false - while kill -0 ${pid} > /dev/null 2>&1; do - if [[ ${cnt} -gt 1 ]]; then - # still not dead, wait - cnt=$(expr ${cnt} - 1) - sleep 1 - elif [[ "${soft_kill_option}" = true ]] ; then - # waited long enough, give up killing the process - echo "Process [${pid}] did not complete after "${maxWait}" seconds, failing to kill it." >&2 - fail_to_kill=true - break; - else - # waited long enough, forcibly kill the process - echo "Process [${pid}] did not complete after "${maxWait}" seconds, forcibly killing it." >&2 - kill -9 ${pid} 2> /dev/null - fi - done - if [[ "$fail_to_kill" = true ]]; then - failed_count=$(expr ${failed_count} + 1) - else - success_count=$(expr ${success_count} + 1) - fi - done - if [[ $success_count -gt 0 ]]; then - echo "Successfully Killed ${success_count} process(es) successfully on $(hostname)" - fi - if [[ $failed_count -gt 0 ]]; then - echo "Soft kill option -s specified, failed to kill ${failed_count} process(es) on $(hostname)" - fi -} - -function copyDir { - if [[ $# -ne 1 ]]; then - echo "Usage: alluxio copyDir " >&2 - exit 2 - fi - - ABS_PATH_COMMAND="readlink -f" - if [[ $(uname -s) = "Darwin" ]] - then - ABS_PATH_COMMAND="realpath" - fi - - MASTERS=$(cat ${ALLUXIO_CONF_DIR}/masters | grep -v '^#') - WORKERS=$(cat ${ALLUXIO_CONF_DIR}/workers | grep -v '^#') - - DIR=$(${ABS_PATH_COMMAND} $1) - DIR=$(echo ${DIR}|sed 's@/$@@') - DEST=$(dirname ${DIR}) - - SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5" - - echo "RSYNC'ing ${DIR} to masters..." - for master in ${MASTERS}; do - echo ${master} - rsync -e "ssh ${SSH_OPTS}" -az ${DIR} ${master}:${DEST} & sleep 0.05 - done - - echo "RSYNC'ing ${DIR} to workers..." - for worker in ${WORKERS}; do - echo ${worker} - rsync -e "ssh ${SSH_OPTS}" -az ${DIR} ${worker}:${DEST} & sleep 0.05 - done - wait -} - -function runJavaClass { - CLASS_ARGS=() - local debug_opts="" - for arg in "$@"; do - case "${arg}" in - -debug) - debug_opts+="${ALLUXIO_USER_ATTACH_OPTS}" ;; - -D* | -X* | -agentlib* | -javaagent*) - ALLUXIO_SHELL_JAVA_OPTS+=" ${arg}" ;; - *) - CLASS_ARGS+=("${arg}") - esac - done - "${JAVA}" ${debug_opts} -cp ${CLASSPATH} ${ALLUXIO_USER_JAVA_OPTS} ${ALLUXIO_SHELL_JAVA_OPTS} ${CLASS} ${PARAMETER} "${CLASS_ARGS[@]}" -} - -function formatJournal { - echo "Formatting Alluxio Master @ $(hostname -f)" - CLASS="alluxio.cli.Format" - CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} - PARAMETER="master" - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" -} - -function formatMasters { - JOURNAL_TYPE=$(${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} \ - alluxio.master.journal.type | awk '{print toupper($0)}') - if [[ ${JOURNAL_TYPE} == "EMBEDDED" ]]; then - # Embedded journal is stored on-disk, so format needs to be run on each master - ${LAUNCHER} ${BIN}/alluxio-masters.sh ${BIN}/alluxio formatJournal - else - formatJournal - fi -} - -function main { - LAUNCHER= - # If debugging is enabled propagate that through to sub-shells - if [[ $- == *x* ]]; then - LAUNCHER="bash -x" - fi - BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) - - if [[ $# == 0 ]]; then - printUsage - exit 1 - fi - - COMMAND=$1 - shift - - DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec - ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} - . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh - - PARAMETER="" - - case ${COMMAND} in - "bootstrapConf") - bootstrapConf "$@" - ;; - "format") - if [[ $# -eq 1 ]]; then - if [[ $1 == "-s" ]]; then - if [[ -e ${ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS} ]]; then - # if ufs is local filesystem and already exists - exit 0 - else - # if ufs is not set it will default to local filesystem - if [[ ! -z ${ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS} ]] && [[ ${ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS} != /* ]] && [[ ${ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS} != file://* ]]; then - # if ufs is not local filesystem, don't format - exit 0 - fi - - shift # remove -s param - fi - else - echo "Usage: alluxio format [-s]" >&2 - exit 2 - fi - elif [[ $# -gt 1 ]]; then - echo "Usage: alluxio format [-s]" >&2 - exit 2 - fi - - ${LAUNCHER} ${BIN}/alluxio-workers.sh ${BIN}/alluxio formatWorker - - formatMasters - ;; - "formatWorker") - echo "Formatting Alluxio Worker @ $(hostname -f)" - CLASS="alluxio.cli.Format" - CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} - PARAMETER="worker" - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "formatJournal") - formatJournal - ;; - "formatMaster") - echo "formatMaster is deprecated - use formatJournal or formatMasters instead" - formatJournal - ;; - "formatMasters") - formatMasters - ;; - "fs") - CLASS="alluxio.cli.fs.FileSystemShell" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "fsadmin") - CLASS="alluxio.cli.fsadmin.FileSystemAdminShell" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "getConf") - CLASS="alluxio.cli.GetConf" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.conf.validation.enabled=false" - runJavaClass "$@" - ;; - "collectInfo") - CLASS="alluxio.cli.bundler.CollectInfo" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "job") - CLASS="alluxio.cli.job.JobShell" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "logLevel") - CLASS="alluxio.cli.LogLevel" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "runClass") - CLASS=$1 - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} # this should be the common case - shift - runJavaClass "$@" - ;; - "runTest") - CLASS="alluxio.cli.TestRunner" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "runTests") - CLASS="alluxio.cli.TestRunner" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "runMiniBenchmark") - CLASS="alluxio.cli.MiniBenchmark" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "runHmsTests") - CLASS="alluxio.cli.HmsTests" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "runHdfsMountTests") - CLASS="alluxio.cli.ValidateHdfsMount" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "runJournalCrashTest") - CLASS="alluxio.cli.JournalCrashTest" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "runUfsIOTest") - CLASS="alluxio.stress.cli.UfsIOBench" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "runUfsTests") - CLASS="alluxio.cli.UnderFileSystemContractTest" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "readJournal") - CLASS="alluxio.master.journal.tool.JournalTool" - CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "upgradeJournal") - CLASS="alluxio.master.journal.JournalUpgrader" - CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "killAll") - killAll "$@" - ;; - "copyDir") - copyDir "$@" - ;; - "docGen") - CLASS="alluxio.cli.DocGenerator" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "clearCache") - sync; echo 3 > /proc/sys/vm/drop_caches ; - ;; - "version") - CLASS="alluxio.cli.Version" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; - "validateConf") - CLASS="alluxio.cli.ValidateConf" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - "validateEnv") - CLASS="alluxio.cli.ValidateEnv" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" - runJavaClass "$@" - ;; - *) - echo "Unsupported command ${COMMAND}" >&2 - printUsage - exit 1 - ;; - esac -} - -main "$@" +set -eu + +BIN_DIR=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) +ROOT_PATH="${BIN_DIR}/.." +# run the compiled go binary according to the system OS and arch +CLI_PATH="${ROOT_PATH}/cli/src/alluxio.org/cli/bin/alluxioCli-$(uname)-$(uname -m)" + +# if pom.xml exists, assume development environment and compile before running +if [[ -f "${ROOT_PATH}/pom.xml" ]]; then + "${ROOT_PATH}/build/cli/build-cli.sh" + "${CLI_PATH}" --rootPath="${ROOT_PATH}" "$@" + exit +fi + +# otherwise, assume deployed environment which will have binaries precompiled +# run with the flag to point to assembly jars in their deployed locations +"${CLI_PATH}" --rootPath="${ROOT_PATH}" --deployed-env "$@" diff --git a/bin/alluxio-bash b/bin/alluxio-bash new file mode 100755 index 000000000000..ec15731d8835 --- /dev/null +++ b/bin/alluxio-bash @@ -0,0 +1,344 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +function printUsage { + echo "Usage: alluxio COMMAND [GENERIC_COMMAND_OPTIONS] [COMMAND_ARGS]" + echo + echo "COMMAND is one of:" + echo -e " format [-s] \t Format Alluxio master and all workers (if -s specified, only format if underfs is local and doesn't already exist)" + echo -e " formatJournal \t Format Alluxio master journal locally" + echo -e " formatMasters \t Format Alluxio master nodes" + echo -e " formatWorker \t Format Alluxio worker nodes" + echo -e " fs \t Command line tool for interacting with the Alluxio filesystem." + echo -e " fsadmin \t Command line tool for use by Alluxio filesystem admins." + echo -e " getConf [key] \t Look up a configuration key, or print all configuration." + echo -e " logLevel \t Set or get log level of Alluxio servers." + echo -e " runClass \t Run the main method of an Alluxio class." + echo -e " runTests \t Run all end-to-end tests on an Alluxio cluster." + echo -e " runHdfsMountTests\t Test the integration between Alluxio and the target HDFS path. " + echo -e " runUfsIOTest \t Test the Alluxio throughput to the UFS path. Try 'alluxio runUfsIOTest -help' for more help." + echo -e " \t NOTE: This command requires a valid ufs path to write temporary files into." + echo -e " runUfsTests \t Test the integration between Alluxio and the target under filesystem. Try 'alluxio runUfsTests -help' for more help." + echo -e " \t NOTE: This command requires a valid ufs path to run tests against." + echo -e " readJournal \t Read an Alluxio journal file from stdin and write a human-readable version of it to stdout." + echo -e " killAll [-s] \t Kill processes containing the WORD (if '-s' specified, processes won't be forcibly killed even if the operation exceeds the timeout)." + echo -e " copyDir \t Copy the PATH to all master/worker nodes." + echo -e " clearCache \t Clear OS buffer cache of the machine." + echo -e " docGen \t Generate docs automatically." + echo -e " version \t Print Alluxio version and exit." + echo -e " validateConf \t Validate Alluxio conf and exit." + echo -e " validateEnv \t Validate Alluxio environment." + echo -e " collectInfo \t Collects information to troubleshoot an Alluxio cluster." + echo + echo "GENERIC_COMMAND_OPTIONS supports:" + echo -e " -D\t Use a value for a given Alluxio property" + echo + # TODO(binfan): Fix help function for alluxio script + echo "Commands print help when invoked without parameters." +} + +function killAll { + if [[ $# -gt 2 ]]; then + echo "Usage: alluxio killAll [-s] \t Kill processes containing the WORD (if '-s' specified, processes won't be forcibly killed even if the operation exceeds the timeout)." >&2 + exit 2 + fi + if [[ "$1" == "-s" ]]; then + keyword=$2 + soft_kill_option=true + else + keyword=$1 + soft_kill_option=false + fi + local success_count=0 + local failed_count=0 + for pid in $(ps -Aww -o pid,command | grep -i "[j]ava" | grep ${keyword} | awk '{print $1}'); do + kill -15 ${pid} > /dev/null 2>&1 + local maxWait=120 + local cnt=${maxWait} + fail_to_kill=false + while kill -0 ${pid} > /dev/null 2>&1; do + if [[ ${cnt} -gt 1 ]]; then + # still not dead, wait + cnt=$(expr ${cnt} - 1) + sleep 1 + elif [[ "${soft_kill_option}" = true ]] ; then + # waited long enough, give up killing the process + echo "Process [${pid}] did not complete after "${maxWait}" seconds, failing to kill it." >&2 + fail_to_kill=true + break; + else + # waited long enough, forcibly kill the process + echo "Process [${pid}] did not complete after "${maxWait}" seconds, forcibly killing it." >&2 + kill -9 ${pid} 2> /dev/null + fi + done + if [[ "$fail_to_kill" = true ]]; then + failed_count=$(expr ${failed_count} + 1) + else + success_count=$(expr ${success_count} + 1) + fi + done + if [[ $success_count -gt 0 ]]; then + echo "Successfully Killed ${success_count} process(es) successfully on $(hostname)" + fi + if [[ $failed_count -gt 0 ]]; then + echo "Soft kill option -s specified, failed to kill ${failed_count} process(es) on $(hostname)" + fi +} + +function copyDir { + if [[ $# -ne 1 ]]; then + echo "Usage: alluxio copyDir " >&2 + exit 2 + fi + + ABS_PATH_COMMAND="readlink -f" + if [[ $(uname -s) = "Darwin" ]] + then + ABS_PATH_COMMAND="realpath" + fi + + MASTERS=$(cat ${ALLUXIO_CONF_DIR}/masters | grep -v '^#') + WORKERS=$(cat ${ALLUXIO_CONF_DIR}/workers | grep -v '^#') + + DIR=$(${ABS_PATH_COMMAND} $1) + DIR=$(echo ${DIR}|sed 's@/$@@') + DEST=$(dirname ${DIR}) + + SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5" + + echo "RSYNC'ing ${DIR} to masters..." + for master in ${MASTERS}; do + echo ${master} + rsync -e "ssh ${SSH_OPTS}" -az ${DIR} ${master}:${DEST} & sleep 0.05 + done + + echo "RSYNC'ing ${DIR} to workers..." + for worker in ${WORKERS}; do + echo ${worker} + rsync -e "ssh ${SSH_OPTS}" -az ${DIR} ${worker}:${DEST} & sleep 0.05 + done + wait +} + +function runJavaClass { + CLASS_ARGS=() + local debug_opts="" + for arg in "$@"; do + case "${arg}" in + -debug) + debug_opts+="${ALLUXIO_USER_ATTACH_OPTS}" ;; + -D* | -X* | -agentlib* | -javaagent*) + ALLUXIO_SHELL_JAVA_OPTS+=" ${arg}" ;; + *) + CLASS_ARGS+=("${arg}") + esac + done + "${JAVA}" ${debug_opts} -cp ${CLASSPATH} ${ALLUXIO_USER_JAVA_OPTS} ${ALLUXIO_SHELL_JAVA_OPTS} ${CLASS} ${PARAMETER} "${CLASS_ARGS[@]}" +} + +function formatJournal { + echo "Formatting Alluxio Master @ $(hostname -f)" + CLASS="alluxio.cli.Format" + CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} + PARAMETER="master" + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" +} + +function formatMasters { + JOURNAL_TYPE=$(${BIN}/alluxio-bash getConf ${ALLUXIO_MASTER_JAVA_OPTS} \ + alluxio.master.journal.type | awk '{print toupper($0)}') + if [[ ${JOURNAL_TYPE} == "EMBEDDED" ]]; then + # Embedded journal is stored on-disk, so format needs to be run on each master + ${LAUNCHER} ${BIN}/alluxio-masters-bash.sh ${BIN}/alluxio-bash formatJournal + else + formatJournal + fi +} + +function main { + LAUNCHER= + # If debugging is enabled propagate that through to sub-shells + if [[ $- == *x* ]]; then + LAUNCHER="bash -x" + fi + BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) + + if [[ $# == 0 ]]; then + printUsage + exit 1 + fi + + COMMAND=$1 + shift + + DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec + ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} + . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh + + PARAMETER="" + + case ${COMMAND} in + "format") + if [[ $# -eq 1 ]]; then + if [[ $1 == "-s" ]]; then + if [[ -e ${ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS} ]]; then + # if ufs is local filesystem and already exists + exit 0 + else + # if ufs is not set it will default to local filesystem + if [[ ! -z ${ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS} ]] && [[ ${ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS} != /* ]] && [[ ${ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS} != file://* ]]; then + # if ufs is not local filesystem, don't format + exit 0 + fi + + shift # remove -s param + fi + else + echo "Usage: alluxio format [-s]" >&2 + exit 2 + fi + elif [[ $# -gt 1 ]]; then + echo "Usage: alluxio format [-s]" >&2 + exit 2 + fi + + ${LAUNCHER} ${BIN}/alluxio-workers-bash.sh ${BIN}/alluxio-bash formatWorker + + formatMasters + ;; + "formatWorker") + echo "Formatting Alluxio Worker @ $(hostname -f)" + CLASS="alluxio.cli.Format" + CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} + PARAMETER="worker" + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" + ;; + "formatJournal") + formatJournal + ;; + "formatMaster") + echo "formatMaster is deprecated - use formatJournal or formatMasters instead" + formatJournal + ;; + "formatMasters") + formatMasters + ;; + "fs") + CLASS="alluxio.cli.fs.FileSystemShell" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + runJavaClass "$@" + ;; + "fsadmin") + CLASS="alluxio.cli.fsadmin.FileSystemAdminShell" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + runJavaClass "$@" + ;; + "getConf") + CLASS="alluxio.cli.GetConf" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.conf.validation.enabled=false" + runJavaClass "$@" + ;; + "collectInfo") + CLASS="alluxio.cli.bundler.CollectInfo" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + runJavaClass "$@" + ;; + "logLevel") + CLASS="alluxio.cli.LogLevel" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + runJavaClass "$@" + ;; + "runClass") + CLASS=$1 + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} # this should be the common case + shift + runJavaClass "$@" + ;; + "runTests") + CLASS="alluxio.cli.TestRunner" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" + ;; + "runMiniBenchmark") + CLASS="alluxio.cli.MiniBenchmark" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" + ;; + "runHdfsMountTests") + CLASS="alluxio.cli.ValidateHdfsMount" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" + ;; + "runUfsIOTest") + CLASS="alluxio.stress.cli.UfsIOBench" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + runJavaClass "$@" + ;; + "runUfsTests") + CLASS="alluxio.cli.UnderFileSystemContractTest" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + runJavaClass "$@" + ;; + "readJournal") + CLASS="alluxio.master.journal.tool.JournalTool" + CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" + ;; + "killAll") + killAll "$@" + ;; + "copyDir") + copyDir "$@" + ;; + "docGen") + CLASS="alluxio.cli.DocGenerator" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" + ;; + "clearCache") + sync; echo 3 > /proc/sys/vm/drop_caches ; + ;; + "version") + CLASS="alluxio.cli.Version" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + runJavaClass "$@" + ;; + "validateConf") + CLASS="alluxio.cli.ValidateConf" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" + ;; + "validateEnv") + CLASS="alluxio.cli.ValidateEnv" + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console" + runJavaClass "$@" + ;; + *) + echo "Unsupported command ${COMMAND}" >&2 + printUsage + exit 1 + ;; + esac +} + +main "$@" diff --git a/bin/alluxio-common-bash.sh b/bin/alluxio-common-bash.sh new file mode 100755 index 000000000000..ad86766aa8e6 --- /dev/null +++ b/bin/alluxio-common-bash.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +LAUNCHER= +# If debugging is enabled propagate that through to sub-shells +if [[ "$-" == *x* ]]; then + LAUNCHER="bash -x" +fi +BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) + +### Utility functions for alluxio scripts ### + +# Fetches the specified property from Alluxio configuration +function get_alluxio_property() { + local property_key="${1:-}" + if [[ -z ${property_key} ]]; then + echo "No property provided to get_alluxio_property()" + exit 1 + fi + + local property=$(${BIN}/alluxio-bash getConf ${property_key}) + if [[ ${?} -ne 0 ]]; then + echo "Failed to fetch value for Alluxio property key: ${property_key}" + exit 1 + fi + + echo "${property}" +} + +# Generates an array of ramdisk paths in global variable RAMDISKARRAY +function get_ramdisk_array() { + local tier_path=$(get_alluxio_property "alluxio.worker.page.store.dirs") + local patharray + + # Use "Internal Field Separator (IFS)" variable to split strings on a + # delimeter and parse into an array + # - https://stackoverflow.com/a/918931 + local oldifs=$IFS + IFS=',' + read -ra patharray <<< "$tier_path" + read -ra mediumtypearray <<< "$medium_type" + + # iterate over the array elements using indices + # - https://stackoverflow.com/a/6723516 + RAMDISKARRAY=() + for i in "${!patharray[@]}"; do + local dir=${patharray[$i]} + if [[ -z "${dir}" ]]; then + echo "Alluxio has a configured cache directory with an empty path: ${tier_path}" + exit 1 + fi + + RAMDISKARRAY+=(${patharray[$i]}) + done + IFS=$oldifs +} + +# Compose the ssh command according to the hostname +function ssh_command() { + local host=$1 + local command="" + if [[ $host != "localhost" && $host != "127.0.0.1" ]]; then + command="ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${host}" + fi + echo "${command}" +} diff --git a/bin/alluxio-common.sh b/bin/alluxio-common.sh deleted file mode 100755 index 69023a43482f..000000000000 --- a/bin/alluxio-common.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# -# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 -# (the "License"). You may not use this work except in compliance with the License, which is -# available at www.apache.org/licenses/LICENSE-2.0 -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied, as more fully set forth in the License. -# -# See the NOTICE file distributed with this work for information regarding copyright ownership. -# - -LAUNCHER= -# If debugging is enabled propagate that through to sub-shells -if [[ "$-" == *x* ]]; then - LAUNCHER="bash -x" -fi -BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) - -### Utility functions for alluxio scripts ### - -# Fetches the specified property from Alluxio configuration -function get_alluxio_property() { - local property_key="${1:-}" - if [[ -z ${property_key} ]]; then - echo "No property provided to get_alluxio_property()" - exit 1 - fi - - local property=$(${BIN}/alluxio getConf ${property_key}) - if [[ ${?} -ne 0 ]]; then - echo "Failed to fetch value for Alluxio property key: ${property_key}" - exit 1 - fi - - echo "${property}" -} - -# Generates an array of ramdisk paths in global variable RAMDISKARRAY -# - Only examines level0 of the tiered store for "MEM"-type paths -function get_ramdisk_array() { - local tier_path=$(get_alluxio_property "alluxio.worker.tieredstore.level0.dirs.path") - local medium_type=$(get_alluxio_property "alluxio.worker.tieredstore.level0.dirs.mediumtype") - local patharray - local mediumtypearray - - # Use "Internal Field Separator (IFS)" variable to split strings on a - # delimeter and parse into an array - # - https://stackoverflow.com/a/918931 - local oldifs=$IFS - IFS=',' - read -ra patharray <<< "$tier_path" - read -ra mediumtypearray <<< "$medium_type" - - # iterate over the array elements using indices - # - https://stackoverflow.com/a/6723516 - RAMDISKARRAY=() - for i in "${!patharray[@]}"; do - if [ "${mediumtypearray[$i]}" = "MEM" ]; then - local dir=${patharray[$i]} - if [[ -z "${dir}" ]]; then - echo "Alluxio has a configured ramcache with an empty path" - exit 1 - fi - - RAMDISKARRAY+=(${patharray[$i]}) - fi - done - IFS=$oldifs -} diff --git a/bin/alluxio-masters-bash.sh b/bin/alluxio-masters-bash.sh new file mode 100755 index 000000000000..ccd35552096e --- /dev/null +++ b/bin/alluxio-masters-bash.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -o pipefail + +. $(dirname "$0")/alluxio-common-bash.sh + +USAGE="Usage: alluxio-masters-bash.sh command..." + +# if no args specified, show usage +if [[ $# -le 0 ]]; then + echo ${USAGE} >&2 + exit 1 +fi + +DEFAULT_LIBEXEC_DIR="${BIN}/../libexec" +ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} +. ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh + +HOSTLIST=($(echo $(cat "${ALLUXIO_CONF_DIR}/masters" | sed "s/#.*$//;/^$/d"))) +mkdir -p "${ALLUXIO_LOGS_DIR}" +ALLUXIO_TASK_LOG="${ALLUXIO_LOGS_DIR}/task.log" + +echo "Executing the following command on all master nodes and logging to ${ALLUXIO_TASK_LOG}: $@" | tee -a ${ALLUXIO_TASK_LOG} + +HA_ENABLED=$(${BIN}/alluxio-bash getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.zookeeper.enabled) +JOURNAL_TYPE=$(${BIN}/alluxio-bash getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.type | awk '{print toupper($0)}') +if [[ ${JOURNAL_TYPE} == "EMBEDDED" ]]; then + HA_ENABLED="true" +fi +for master in ${HOSTLIST[@]}; do + echo "[${master}] Connecting as ${USER}..." >> ${ALLUXIO_TASK_LOG} + nohup $(ssh_command ${master}) ${LAUNCHER} \ + $"${@// /\\ }" 2>&1 | while read line; do echo "[$(date '+%F %T')][${master}] ${line}"; done >> ${ALLUXIO_TASK_LOG} & + pids[${#pids[@]}]=$! +done + +# wait for all pids +echo "Waiting for tasks to finish..." +has_error=0 +for ((i=0; i< ${#pids[@]}; i++)); do + wait ${pids[$i]} + ret_code=$? + if [[ ${ret_code} -ne 0 ]]; then + has_error=1 + echo "Task on '${HOSTLIST[$i]}' fails, exit code: ${ret_code}" >&2 + fi +done + +# only show the log when all tasks run OK! +if [[ ${has_error} -eq 0 ]]; then + echo "All tasks finished" +else + echo "There are task failures, look at ${ALLUXIO_TASK_LOG} for details." >&2 +fi diff --git a/bin/alluxio-masters.sh b/bin/alluxio-masters.sh deleted file mode 100755 index 359c3daea348..000000000000 --- a/bin/alluxio-masters.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -# -# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 -# (the "License"). You may not use this work except in compliance with the License, which is -# available at www.apache.org/licenses/LICENSE-2.0 -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied, as more fully set forth in the License. -# -# See the NOTICE file distributed with this work for information regarding copyright ownership. -# - -set -o pipefail - -LAUNCHER= -# If debugging is enabled propagate that through to sub-shells -if [[ "$-" == *x* ]]; then - LAUNCHER="bash -x" -fi -BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) - -USAGE="Usage: alluxio-masters.sh command..." - -# if no args specified, show usage -if [[ $# -le 0 ]]; then - echo ${USAGE} >&2 - exit 1 -fi - -DEFAULT_LIBEXEC_DIR="${BIN}/../libexec" -ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} -. ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh - -HOSTLIST=($(echo $(cat "${ALLUXIO_CONF_DIR}/masters" | sed "s/#.*$//;/^$/d"))) -mkdir -p "${ALLUXIO_LOGS_DIR}" -ALLUXIO_TASK_LOG="${ALLUXIO_LOGS_DIR}/task.log" - -echo "Executing the following command on all master nodes and logging to ${ALLUXIO_TASK_LOG}: $@" | tee -a ${ALLUXIO_TASK_LOG} - -N=0 -HA_ENABLED=$(${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.zookeeper.enabled) -JOURNAL_TYPE=$(${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.type | awk '{print toupper($0)}') -if [[ ${JOURNAL_TYPE} == "EMBEDDED" ]]; then - HA_ENABLED="true" -fi -for master in ${HOSTLIST[@]}; do - echo "[${master}] Connecting as ${USER}..." >> ${ALLUXIO_TASK_LOG} - if [[ ${HA_ENABLED} == "true" || ${N} -eq 0 ]]; then - nohup ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${master} ${LAUNCHER} \ - $"${@// /\\ }" 2>&1 | while read line; do echo "[$(date '+%F %T')][${master}] ${line}"; done >> ${ALLUXIO_TASK_LOG} & - else - nohup ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${master} ${LAUNCHER} \ - $"export ALLUXIO_MASTER_SECONDARY=true; ${@// /\\ }" 2>&1 | while read line; do echo "[$(date '+%F %T')][${master}] ${line}"; done >> ${ALLUXIO_TASK_LOG} & - fi - pids[${#pids[@]}]=$! - N=$((N+1)) -done - -# wait for all pids -echo "Waiting for tasks to finish..." -has_error=0 -for ((i=0; i< ${#pids[@]}; i++)); do - wait ${pids[$i]} - ret_code=$? - if [[ ${ret_code} -ne 0 ]]; then - has_error=1 - echo "Task on '${HOSTLIST[$i]}' fails, exit code: ${ret_code}" >&2 - fi -done - -# only show the log when all tasks run OK! -if [[ ${has_error} -eq 0 ]]; then - echo "All tasks finished" -else - echo "There are task failures, look at ${ALLUXIO_TASK_LOG} for details." >&2 -fi diff --git a/bin/alluxio-monitor-bash.sh b/bin/alluxio-monitor-bash.sh new file mode 100755 index 000000000000..287bea261c07 --- /dev/null +++ b/bin/alluxio-monitor-bash.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +LAUNCHER= +# If debugging is enabled propagate that through to sub-shells +if [[ "$-" == *x* ]]; then + LAUNCHER="bash -x" +fi +BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) + +USAGE="Usage: alluxio-monitor-bash.sh [-hL] ACTION [host1,host2,...] +Where ACTION is one of: + all \tStart monitors for all master and worker nodes. + local \tStart monitors for local master and worker processes. + master \tStart a master monitor on this node. + masters \tStart monitors for all masters nodes. + worker \tStart a worker monitor on this node. + workers \tStart monitors for all workers nodes. + job_master \tStart a job_master monitor on this node. + job_masters \tStart monitors for all job_master nodes. + job_worker \tStart a job_worker monitor on this node. + job_workers \tStart monitors for all job_worker nodes. + proxy \tStart the proxy monitor on this node. + proxies \tStart monitors for all proxies nodes. +" +USAGE+=" +[host1,host2,...] is a comma separated list of host to monitor, if not given the default config for the target is used. + +-L enables the log mode, this option disable the monitor checks and causes alluxio-monitor to only print the node log tail. +-h display this help. +" + +RED='\033[1;31m' +GREEN='\033[1;32m' +PURPLE='\033[0;35m' +CYAN='\033[1;36m' +WHITE='\033[1;37m' +NC='\033[0m' + +get_env() { + DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec + ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} + . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh + CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} + ALLUXIO_TASK_LOG="${ALLUXIO_LOGS_DIR}/task.log" + + # Remove the remote debug configuration to avoid the error: "transport error 20: bind failed: Address already in use." + # See https://github.com/Alluxio/alluxio/issues/10958 + # See https://github.com/Alluxio/alluxio/issues/15168 + ALLUXIO_MASTER_MONITOR_JAVA_OPTS=$(echo ${ALLUXIO_MASTER_JAVA_OPTS} | sed 's/^-agentlib:jdwp=transport=dt_socket.*address=[0-9]*//' | sed 's/-Dcom.sun.management.jmxremote.port=[0-9]*//') + ALLUXIO_WORKER_MONITOR_JAVA_OPTS=$(echo ${ALLUXIO_WORKER_JAVA_OPTS} | sed 's/^-agentlib:jdwp=transport=dt_socket.*address=[0-9]*//' | sed 's/-Dcom.sun.management.jmxremote.port=[0-9]*//') + ALLUXIO_JOB_MASTER_MONITOR_JAVA_OPTS=$(echo ${ALLUXIO_JOB_MASTER_JAVA_OPTS} | sed 's/^-agentlib:jdwp=transport=dt_socket.*address=[0-9]*//' | sed 's/-Dcom.sun.management.jmxremote.port=[0-9]*//') + ALLUXIO_JOB_WORKER_MONITOR_JAVA_OPTS=$(echo ${ALLUXIO_JOB_WORKER_JAVA_OPTS} | sed 's/^-agentlib:jdwp=transport=dt_socket.*address=[0-9]*//'| sed 's/-Dcom.sun.management.jmxremote.port=[0-9]*//') +} + +prepare_monitor() { + local msg=$1 + echo -e "${WHITE}-----------------------------------------${NC}" + echo -e "${msg}" + echo -e "${WHITE}-----------------------------------------${NC}" + sleep 2 +} + +print_node_logs() { + local node_type=$1 + local node_logs=( "${ALLUXIO_LOGS_DIR}/${node_type}.log" "${ALLUXIO_LOGS_DIR}/${node_type}.out" ) + for node_log in "${node_logs[@]}"; do + echo -e "${CYAN}--- Printing the log tail for ${node_log}${NC}" + if [[ -f "${node_log}" ]]; then + local lines_count=$(cat "${node_log}" | wc -l) + if [[ lines_count -gt 0 ]]; then + echo -e "${CYAN}>>> BEGIN${NC}" + tail -30 "${node_log}" + echo -e "${CYAN}<<< EOF${NC}" + else + echo -e " --- EMPTY ---" + fi + fi + done +} + +run_monitor() { + local node_type=$1 + local mode=$2 + local alluxio_config="${ALLUXIO_JAVA_OPTS}" + + case "${node_type}" in + master) + monitor_exec=alluxio.master.AlluxioMasterMonitor + alluxio_config="${alluxio_config} ${ALLUXIO_MASTER_MONITOR_JAVA_OPTS}" + ;; + worker) + monitor_exec=alluxio.worker.AlluxioWorkerMonitor + alluxio_config="${alluxio_config} ${ALLUXIO_WORKER_MONITOR_JAVA_OPTS}" + ;; + job_master) + monitor_exec=alluxio.master.job.AlluxioJobMasterMonitor + alluxio_config="${alluxio_config} ${ALLUXIO_JOB_MASTER_MONITOR_JAVA_OPTS}" + ;; + job_worker) + monitor_exec=alluxio.worker.job.AlluxioJobWorkerMonitor + alluxio_config="${alluxio_config} ${ALLUXIO_JOB_WORKER_MONITOR_JAVA_OPTS}" + ;; + proxy) + monitor_exec=alluxio.proxy.AlluxioProxyMonitor + ;; + *) + echo "Error: Invalid NODE_TYPE: ${node_type}" >&2 + return 1 + ;; + esac + + if [[ "${mode}" == "-L" ]]; then + print_node_logs "${node_type}" + return 0 + else + "${JAVA}" -cp ${CLASSPATH} ${alluxio_config} ${monitor_exec} + if [[ $? -ne 0 ]]; then + echo -e "${WHITE}---${NC} ${RED}[ FAILED ]${NC} The ${CYAN}${node_type}${NC} @ ${PURPLE}$(hostname -f)${NC} is not serving requests after 120s. Please check if the process is running and the logs/ if necessary.${NC}" + print_node_logs "${node_type}" + return 1 + fi + fi + echo -e "${WHITE}---${NC} ${GREEN}[ OK ]${NC} The ${CYAN}${node_type}${NC} service @ ${PURPLE}$(hostname -f)${NC} is in a healthy state.${NC}" + return 0 +} + +# $1 --> The path to the masters/workers file +get_nodes() { + local node_file=$1 + echo "$(cat "${node_file}" | sed "s/#.*$//;/^$/d")" +} + +run_on_node() { + local node=$1 + # Appends every argument after $1 to the launcher terminal + + ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${node} ${LAUNCHER} ${@:2} \ + 2> >(while read line; do echo "[$(date '+%F %T')][${node}] ${line}" >> ${ALLUXIO_TASK_LOG}; done) +} + +run_monitors() { + local node_type=$1 + local nodes=$2 + local mode=$3 + if [[ -z "${nodes}" ]]; then + case "${node_type}" in + master) + nodes=$(get_nodes "${ALLUXIO_CONF_DIR}/masters") + ;; + worker) + nodes=$(get_nodes "${ALLUXIO_CONF_DIR}/workers") + ;; + job_master) + # Fall back to {conf}/masters if job_masters doesn't exist + local job_masters="${ALLUXIO_CONF_DIR}/job_masters" + if [[ ! -f ${job_masters} ]]; then job_masters=${ALLUXIO_CONF_DIR}/masters; fi + nodes=$(get_nodes "${job_masters}") + ;; + job_worker) + # Fall back to {conf}/workers if job_workers doesn't exist + local job_workers="${ALLUXIO_CONF_DIR}/job_workers" + if [[ ! -f ${job_workers} ]]; then job_workers=${ALLUXIO_CONF_DIR}/workers; fi + nodes=$(get_nodes "${job_workers}") + ;; + proxy) + nodes=$(awk '{print}' "${ALLUXIO_CONF_DIR}/masters" "${ALLUXIO_CONF_DIR}/workers" | sed "s/#.*$//;/^$/d" | sort | uniq) + ;; + *) + echo "Error: Invalid NODE_TYPE: ${node_type}" >&2 + exit 1 + ;; + esac + fi + + if [[ "${node_type}" == "master" ]]; then + # master check should only run once... + local master=$(echo -e "${nodes}" | head -n1) + run_on_node ${master} "${BIN}/alluxio-monitor-bash.sh" ${mode} "${node_type}" + + nodes=$(echo -e "${nodes}" | tail -n+2) + if [[ $? -ne 0 ]]; then + # if there is an error, print the log tail for the remaining master nodes. + batch_run_on_nodes "$(echo ${nodes})" "${BIN}/alluxio-monitor-bash.sh" -L "${node_type}" + else + HA_ENABLED=$(${BIN}/alluxio-bash getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.zookeeper.enabled) + JOURNAL_TYPE=$(${BIN}/alluxio-bash getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.type | awk '{print toupper($0)}') + if [[ ${JOURNAL_TYPE} == "EMBEDDED" ]]; then + HA_ENABLED="true" + fi + if [[ ${HA_ENABLED} == "true" ]]; then + batch_run_on_nodes "$(echo ${nodes})" "${BIN}/alluxio-monitor-bash.sh" "${mode}" "${node_type}" + fi + fi + else + batch_run_on_nodes "$(echo ${nodes})" "${BIN}/alluxio-monitor-bash.sh" "${mode}" "${node_type}" + fi +} + +# Used to run a command on multiple hosts concurrently. +# By default it limits concurrent tasks to 100. +batch_run_on_nodes() { + # String of nodes, seperated by a new line + local nodes=$1 + # Command to run on each node + local command=$2 + # Parameter for command + local params=${@:3} + + # How many nodes to run on concurrently + local batchCount=100 + + local taskCount=0 + for node in $nodes; do + run_on_node ${node} ${command} ${params} & + + # Wait for existing tasks, if batch is full + ((taskCount++)) + if [ $(( $taskCount % $batchCount )) == 0 ]; then wait; fi; + done + wait +} + +main() { + + # exceed the max number of args, show usage + if [[ $# -ge 3 ]]; then + echo "Error: Invalid number of arguments" >&2 + echo -e "${USAGE}" >&2 + exit 1 + fi + + get_env + + while getopts "hL" o; do + case "${o}" in + h) + echo -e "${USAGE}" + exit 0 + ;; + L) + MODE="-L" + ;; + *) + echo -e "${USAGE}" >&2 + exit 1 + ;; + esac + done + + shift $((${OPTIND} - 1)) + + ACTION=$1 + shift + + HOSTS=$1 + if [[ ! -z "${HOSTS}" ]]; then + HOSTS=$(echo "${HOSTS}" | tr ',' '\n') + fi + + case "${ACTION}" in + all) + prepare_monitor "Starting to monitor ${CYAN}all remote${NC} services." + run_monitors "master" "" "${MODE}" + run_monitors "worker" "" "${MODE}" + ;; + local) + prepare_monitor "Starting to monitor ${CYAN}all local${NC} services." + run_monitor "master" "${MODE}" + run_monitor "worker" "${MODE}" + ;; + master) + run_monitor "master" "${MODE}" + ;; + masters) + prepare_monitor "Starting to monitor all Alluxio ${CYAN}masters${NC}." + run_monitors "master" "${HOSTS}" + ;; + job_master) + run_monitor "job_master" "${MODE}" + ;; + job_masters) + prepare_monitor "Starting to monitor all Alluxio ${CYAN}job masters${NC}." + run_monitors "job_master" "${HOSTS}" + ;; + job_worker) + run_monitor "job_worker" "${MODE}" + ;; + job_workers) + prepare_monitor "Starting to monitor all Alluxio ${CYAN}job workers${NC}." + run_monitors "job_worker" "${HOSTS}" + ;; + proxy) + run_monitor "proxy" "${MODE}" + ;; + proxies) + prepare_monitor "Starting to monitor all Alluxio ${CYAN}proxies${NC}." + run_monitors "proxy" "${HOSTS}" "${MODE}" + ;; + worker) + run_monitor "worker" "${MODE}" + ;; + workers) + prepare_monitor "Starting to monitor all Alluxio ${CYAN}workers${NC}." + run_monitors "worker" "${HOSTS}" "${MODE}" + ;; + *) + echo "Error: Invalid ACTION: ${ACTION}" >&2 + echo -e "${USAGE}" >&2 + exit 1 + esac +} + +main "$@" diff --git a/bin/alluxio-monitor.sh b/bin/alluxio-monitor.sh deleted file mode 100755 index ed57a5578e18..000000000000 --- a/bin/alluxio-monitor.sh +++ /dev/null @@ -1,329 +0,0 @@ -#!/usr/bin/env bash -# -# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 -# (the "License"). You may not use this work except in compliance with the License, which is -# available at www.apache.org/licenses/LICENSE-2.0 -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied, as more fully set forth in the License. -# -# See the NOTICE file distributed with this work for information regarding copyright ownership. -# - -LAUNCHER= -# If debugging is enabled propagate that through to sub-shells -if [[ "$-" == *x* ]]; then - LAUNCHER="bash -x" -fi -BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) - -USAGE="Usage: alluxio-monitor.sh [-hL] ACTION [host1,host2,...] -Where ACTION is one of: - all \tStart monitors for all masters, proxies, and workers nodes. - local \tStart monitors for all process locally. - master \tStart a master monitor on this node. - masters \tStart monitors for all masters nodes. - worker \tStart a worker monitor on this node. - workers \tStart monitors for all workers nodes. - job_master \tStart a job_master monitor on this node. - job_masters \tStart monitors for all job_master nodes. - job_worker \tStart a job_worker monitor on this node. - job_workers \tStart monitors for all job_worker nodes. - proxy \tStart the proxy monitor on this node. - proxies \tStart monitors for all proxies nodes. -" -USAGE+=" -[host1,host2,...] is a comma separated list of host to monitor, if not given the default config for the target is used. - --L enables the log mode, this option disable the monitor checks and causes alluxio-monitor to only print the node log tail. --h display this help. -" - -RED='\033[1;31m' -GREEN='\033[1;32m' -PURPLE='\033[0;35m' -CYAN='\033[1;36m' -WHITE='\033[1;37m' -NC='\033[0m' - -get_env() { - DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec - ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} - . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - ALLUXIO_TASK_LOG="${ALLUXIO_LOGS_DIR}/task.log" - - # Remove the remote debug configuration to avoid the error: "transport error 20: bind failed: Address already in use." - # See https://github.com/Alluxio/alluxio/issues/10958 - # See https://github.com/Alluxio/alluxio/issues/15168 - ALLUXIO_MASTER_MONITOR_JAVA_OPTS=$(echo ${ALLUXIO_MASTER_JAVA_OPTS} | sed 's/^-agentlib:jdwp=transport=dt_socket.*address=[0-9]*//' | sed 's/-Dcom.sun.management.jmxremote.port=[0-9]*//') - ALLUXIO_WORKER_MONITOR_JAVA_OPTS=$(echo ${ALLUXIO_WORKER_JAVA_OPTS} | sed 's/^-agentlib:jdwp=transport=dt_socket.*address=[0-9]*//' | sed 's/-Dcom.sun.management.jmxremote.port=[0-9]*//') - ALLUXIO_JOB_MASTER_MONITOR_JAVA_OPTS=$(echo ${ALLUXIO_JOB_MASTER_JAVA_OPTS} | sed 's/^-agentlib:jdwp=transport=dt_socket.*address=[0-9]*//' | sed 's/-Dcom.sun.management.jmxremote.port=[0-9]*//') - ALLUXIO_JOB_WORKER_MONITOR_JAVA_OPTS=$(echo ${ALLUXIO_JOB_WORKER_JAVA_OPTS} | sed 's/^-agentlib:jdwp=transport=dt_socket.*address=[0-9]*//'| sed 's/-Dcom.sun.management.jmxremote.port=[0-9]*//') -} - -prepare_monitor() { - local msg=$1 - echo -e "${WHITE}-----------------------------------------${NC}" - echo -e "${msg}" - echo -e "${WHITE}-----------------------------------------${NC}" - sleep 2 -} - -print_node_logs() { - local node_type=$1 - local node_logs=( "${ALLUXIO_LOGS_DIR}/${node_type}.log" "${ALLUXIO_LOGS_DIR}/${node_type}.out" ) - for node_log in "${node_logs[@]}"; do - echo -e "${CYAN}--- Printing the log tail for ${node_log}${NC}" - if [[ -f "${node_log}" ]]; then - local lines_count=$(cat "${node_log}" | wc -l) - if [[ lines_count -gt 0 ]]; then - echo -e "${CYAN}>>> BEGIN${NC}" - tail -30 "${node_log}" - echo -e "${CYAN}<<< EOF${NC}" - else - echo -e " --- EMPTY ---" - fi - fi - done -} - -run_monitor() { - local node_type=$1 - local mode=$2 - local alluxio_config="${ALLUXIO_JAVA_OPTS}" - - case "${node_type}" in - master) - monitor_exec=alluxio.master.AlluxioMasterMonitor - alluxio_config="${alluxio_config} ${ALLUXIO_MASTER_MONITOR_JAVA_OPTS}" - ;; - worker) - monitor_exec=alluxio.worker.AlluxioWorkerMonitor - alluxio_config="${alluxio_config} ${ALLUXIO_WORKER_MONITOR_JAVA_OPTS}" - ;; - job_master) - monitor_exec=alluxio.master.job.AlluxioJobMasterMonitor - alluxio_config="${alluxio_config} ${ALLUXIO_JOB_MASTER_MONITOR_JAVA_OPTS}" - ;; - job_worker) - monitor_exec=alluxio.worker.job.AlluxioJobWorkerMonitor - alluxio_config="${alluxio_config} ${ALLUXIO_JOB_WORKER_MONITOR_JAVA_OPTS}" - ;; - proxy) - monitor_exec=alluxio.proxy.AlluxioProxyMonitor - ;; - *) - echo "Error: Invalid NODE_TYPE: ${node_type}" >&2 - return 1 - ;; - esac - - if [[ "${mode}" == "-L" ]]; then - print_node_logs "${node_type}" - return 0 - else - "${JAVA}" -cp ${CLASSPATH} ${alluxio_config} ${monitor_exec} - if [[ $? -ne 0 ]]; then - echo -e "${WHITE}---${NC} ${RED}[ FAILED ]${NC} The ${CYAN}${node_type}${NC} @ ${PURPLE}$(hostname -f)${NC} is not serving requests.${NC}" - print_node_logs "${node_type}" - return 1 - fi - fi - echo -e "${WHITE}---${NC} ${GREEN}[ OK ]${NC} The ${CYAN}${node_type}${NC} service @ ${PURPLE}$(hostname -f)${NC} is in a healthy state.${NC}" - return 0 -} - -# $1 --> The path to the masters/workers file -get_nodes() { - local node_file=$1 - echo "$(cat "${node_file}" | sed "s/#.*$//;/^$/d")" -} - -run_on_node() { - local node=$1 - # Appends every argument after $1 to the launcher terminal - - ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${node} ${LAUNCHER} ${@:2} \ - 2> >(while read line; do echo "[$(date '+%F %T')][${node}] ${line}" >> ${ALLUXIO_TASK_LOG}; done) -} - -run_monitors() { - local node_type=$1 - local nodes=$2 - local mode=$3 - if [[ -z "${nodes}" ]]; then - case "${node_type}" in - master) - nodes=$(get_nodes "${ALLUXIO_CONF_DIR}/masters") - ;; - worker) - nodes=$(get_nodes "${ALLUXIO_CONF_DIR}/workers") - ;; - job_master) - # Fall back to {conf}/masters if job_masters doesn't exist - local job_masters="${ALLUXIO_CONF_DIR}/job_masters" - if [[ ! -f ${job_masters} ]]; then job_masters=${ALLUXIO_CONF_DIR}/masters; fi - nodes=$(get_nodes "${job_masters}") - ;; - job_worker) - # Fall back to {conf}/workers if job_workers doesn't exist - local job_workers="${ALLUXIO_CONF_DIR}/job_workers" - if [[ ! -f ${job_workers} ]]; then job_workers=${ALLUXIO_CONF_DIR}/workers; fi - nodes=$(get_nodes "${job_workers}") - ;; - proxy) - nodes=$(awk '{print}' "${ALLUXIO_CONF_DIR}/masters" "${ALLUXIO_CONF_DIR}/workers" | sed "s/#.*$//;/^$/d" | sort | uniq) - ;; - *) - echo "Error: Invalid NODE_TYPE: ${node_type}" >&2 - exit 1 - ;; - esac - fi - - if [[ "${node_type}" == "master" ]]; then - # master check should only run once... - local master=$(echo -e "${nodes}" | head -n1) - run_on_node ${master} "${BIN}/alluxio-monitor.sh" ${mode} "${node_type}" - - nodes=$(echo -e "${nodes}" | tail -n+2) - if [[ $? -ne 0 ]]; then - # if there is an error, print the log tail for the remaining master nodes. - batch_run_on_nodes "$(echo ${nodes})" "${BIN}/alluxio-monitor.sh" -L "${node_type}" - else - HA_ENABLED=$(${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.zookeeper.enabled) - JOURNAL_TYPE=$(${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.type | awk '{print toupper($0)}') - if [[ ${JOURNAL_TYPE} == "EMBEDDED" ]]; then - HA_ENABLED="true" - fi - if [[ ${HA_ENABLED} == "true" ]]; then - batch_run_on_nodes "$(echo ${nodes})" "${BIN}/alluxio-monitor.sh" "${mode}" "${node_type}" - fi - fi - else - batch_run_on_nodes "$(echo ${nodes})" "${BIN}/alluxio-monitor.sh" "${mode}" "${node_type}" - fi -} - -# Used to run a command on multiple hosts concurrently. -# By default it limits concurrent tasks to 100. -batch_run_on_nodes() { - # String of nodes, seperated by a new line - local nodes=$1 - # Command to run on each node - local command=$2 - # Parameter for command - local params=${@:3} - - # How many nodes to run on concurrently - local batchCount=100 - - local taskCount=0 - for node in $nodes; do - run_on_node ${node} ${command} ${params} & - - # Wait for existing tasks, if batch is full - ((taskCount++)) - if [ $(( $taskCount % $batchCount )) == 0 ]; then wait; fi; - done - wait -} - -main() { - - # exceed the max number of args, show usage - if [[ $# -ge 3 ]]; then - echo "Error: Invalid number of arguments" >&2 - echo -e "${USAGE}" >&2 - exit 1 - fi - - get_env - - while getopts "hL" o; do - case "${o}" in - h) - echo -e "${USAGE}" - exit 0 - ;; - L) - MODE="-L" - ;; - *) - echo -e "${USAGE}" >&2 - exit 1 - ;; - esac - done - - shift $((${OPTIND} - 1)) - - ACTION=$1 - shift - - HOSTS=$1 - if [[ ! -z "${HOSTS}" ]]; then - HOSTS=$(echo "${HOSTS}" | tr ',' '\n') - fi - - case "${ACTION}" in - all) - prepare_monitor "Starting to monitor ${CYAN}all remote${NC} services." - run_monitors "master" "" "${MODE}" - run_monitors "job_master" "" "${MODE}" - run_monitors "worker" "" "${MODE}" - run_monitors "job_worker" "" "${MODE}" - run_monitors "proxy" "" "${MODE}" - ;; - local) - prepare_monitor "Starting to monitor ${CYAN}all local${NC} services." - run_monitor "master" "${MODE}" - run_monitor "job_master" "${MODE}" - run_monitor "worker" "${MODE}" - run_monitor "job_worker" "${MODE}" - run_monitor "proxy" "${MODE}" - ;; - master) - run_monitor "master" "${MODE}" - ;; - masters) - prepare_monitor "Starting to monitor all Alluxio ${CYAN}masters${NC}." - run_monitors "master" "${HOSTS}" - ;; - job_master) - run_monitor "job_master" "${MODE}" - ;; - job_masters) - prepare_monitor "Starting to monitor all Alluxio ${CYAN}job masters${NC}." - run_monitors "job_master" "${HOSTS}" - ;; - job_worker) - run_monitor "job_worker" "${MODE}" - ;; - job_workers) - prepare_monitor "Starting to monitor all Alluxio ${CYAN}job workers${NC}." - run_monitors "job_worker" "${HOSTS}" - ;; - proxy) - run_monitor "proxy" "${MODE}" - ;; - proxies) - prepare_monitor "Starting to monitor all Alluxio ${CYAN}proxies${NC}." - run_monitors "proxy" "${HOSTS}" "${MODE}" - ;; - worker) - run_monitor "worker" "${MODE}" - ;; - workers) - prepare_monitor "Starting to monitor all Alluxio ${CYAN}workers${NC}." - run_monitors "worker" "${HOSTS}" "${MODE}" - ;; - *) - echo "Error: Invalid ACTION: ${ACTION}" >&2 - echo -e "${USAGE}" >&2 - exit 1 - esac -} - -main "$@" diff --git a/bin/alluxio-mount-bash.sh b/bin/alluxio-mount-bash.sh new file mode 100755 index 000000000000..ecda7d072d0d --- /dev/null +++ b/bin/alluxio-mount-bash.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +# Starts the Alluxio master on this node. +# Starts an Alluxio worker on each node specified in conf/workers + +. $(dirname "$0")/alluxio-common-bash.sh + +USAGE="Usage: alluxio-mount-bash.sh [Mount|SudoMount|Umount|SudoUmount] [MACHINE] +\nIf omitted, MACHINE is default to be 'local'. MACHINE is one of:\n + local\t\t\tMount local machine\n + workers\t\tMount all the workers on worker nodes" + +function init_env() { + local libexec_dir=${ALLUXIO_LIBEXEC_DIR:-"${BIN}"/../libexec} + . ${libexec_dir}/alluxio-config.sh + RAMDISK_SIZE=$(${BIN}/alluxio-bash getConf --unit B alluxio.worker.ramdisk.size) + TIER_ALIAS=$(${BIN}/alluxio-bash getConf alluxio.worker.tieredstore.level0.alias) + get_ramdisk_array +} + +function check_space_linux() { + local total_mem=$(($(cat /proc/meminfo | awk 'NR==1{print $2}') * 1024)) + if [[ ${total_mem} -lt ${RAMDISK_SIZE} ]]; then + echo "ERROR: Memory(${total_mem}) is less than requested ramdisk size(${RAMDISK_SIZE}). Please + reduce alluxio.worker.ramdisk.size in alluxio-site.properties" >&2 + exit 1 + fi +} + +function mount_ramfs_linux() { + TIER_PATH=${1} + echo "Formatting RamFS: ${TIER_PATH} (${RAMDISK_SIZE})" + if [[ ${USE_SUDO} == true ]]; then + sudo mkdir -p ${TIER_PATH} + else + mkdir -p ${TIER_PATH} + fi + if [[ $? -ne 0 ]]; then + echo "ERROR: mkdir ${TIER_PATH} failed" >&2 + exit 1 + fi + + if [[ ${USE_SUDO} == true ]]; then + sudo mount -t ramfs -o size=${RAMDISK_SIZE} ramfs ${TIER_PATH} + else + mount -t ramfs -o size=${RAMDISK_SIZE} ramfs ${TIER_PATH} + fi + if [[ $? -ne 0 ]]; then + echo "ERROR: mount RamFS ${TIER_PATH} failed" >&2 + exit 1 + fi + + if [[ ${USE_SUDO} == true ]]; then + sudo chmod a+w ${TIER_PATH} + else + chmod a+w ${TIER_PATH} + fi + if [[ $? -ne 0 ]]; then + echo "ERROR: chmod RamFS ${TIER_PATH} failed" >&2 + exit 1 + fi +} + +function umount_ramfs_linux() { + TIER_PATH=${1} + if mount | grep -E "(^|[[:space:]])${TIER_PATH}($|[[:space:]])" > /dev/null; then + echo "Unmounting ${TIER_PATH}" + if [[ ${USE_SUDO} == true ]]; then + sudo umount -l -f ${TIER_PATH} + else + umount -l -f ${TIER_PATH} + fi + if [[ $? -ne 0 ]]; then + echo "ERROR: umount RamFS ${TIER_PATH} failed" >&2 + exit 1 + fi + fi +} + +function check_space_freebsd() { + local total_mem=$(sysctl -n hw.usermem) + if [[ ${total_mem} -lt ${RAMDISK_SIZE} ]]; then + echo "ERROR: Memory(${total_mem}) is less than requested ramdisk size(${RAMDISK_SIZE}). Please + reduce alluxio.worker.ramdisk.size in alluxio-site.properties" >&2 + exit 1 + fi +} + +function mount_ramfs_freebsd() { + TIER_PATH=${1} + echo "Formatting RamFS: ${TIER_PATH} (${RAMDISK_SIZE})" + if [[ ${USE_SUDO} == true ]]; then + sudo mkdir -p ${TIER_PATH} + else + mkdir -p ${TIER_PATH} + fi + if [[ $? -ne 0 ]]; then + echo "ERROR: mkdir ${TIER_PATH} failed" >&2 + exit 1 + fi + + if [[ ${USE_SUDO} == true ]]; then + sudo mount -t tmpfs -o size=${RAMDISK_SIZE} tmpfs ${TIER_PATH} + else + mount -t tmpfs -o size=${RAMDISK_SIZE} tmpfs ${TIER_PATH} + fi + if [[ $? -ne 0 ]]; then + echo "ERROR: mount RamFS ${TIER_PATH} failed" >&2 + exit 1 + fi + + if [[ ${USE_SUDO} == true ]]; then + sudo chmod a+w ${TIER_PATH} + else + chmod a+w ${TIER_PATH} + fi + if [[ $? -ne 0 ]]; then + echo "ERROR: chmod RamFS ${TIER_PATH} failed" >&2 + exit 1 + fi +} + +function umount_ramfs_freebsd() { + TIER_PATH=${1} + if mount | grep -E "(^|[[:space:]])${TIER_PATH}($|[[:space:]])" > /dev/null; then + echo "Unmounting ${TIER_PATH}" + if [[ ${USE_SUDO} == true ]]; then + sudo umount -f ${TIER_PATH} + else + umount -f ${TIER_PATH} + fi + if [[ $? -ne 0 ]]; then + echo "ERROR: umount RamFS ${TIER_PATH} failed" >&2 + exit 1 + fi + fi +} + +function mount_ramfs_mac() { + TIER_PATH=${1} + # Convert the memory size to number of sectors. Each sector is 512 Byte. + local num_sectors=$(${BIN}/alluxio-bash runClass alluxio.util.HFSUtils ${RAMDISK_SIZE} 512) + + # Format the RAM FS + echo "Formatting RamFS: ${TIER_PATH} ${num_sectors} sectors (${RAMDISK_SIZE})." + # Remove the "/Volumes/" part so we can get the name of the volume. + diskutil erasevolume HFS+ ${TIER_PATH/#\/Volumes\//} $(hdiutil attach -nomount ram://${num_sectors}) +} + +function umount_ramfs_mac() { + TIER_PATH=${1} + local device=$(df -l | grep -E "(^|[[:space:]])${TIER_PATH}($|[[:space:]])" | cut -d " " -f 1) + if [[ -n "${device}" ]]; then + echo "Unmounting ramfs at ${TIER_PATH}" + hdiutil detach -force ${device} + else + echo "Ramfs is not currently mounted at ${TIER_PATH}" + fi +} + +function mount_ramfs_local_all() { + for RAMDISKPATH in "${RAMDISKARRAY[@]}" + do + mount_ramfs_local $RAMDISKPATH + done +} + +function mount_ramfs_local() { + if [[ $(uname -a) == Darwin* ]]; then + # Assuming Mac OS X + umount_ramfs_mac $1 + mount_ramfs_mac $1 + elif [[ $(uname -a) == FreeBSD* ]]; then + # Assuming FreeBSD + check_space_freebsd + umount_ramfs_freebsd $1 + mount_ramfs_freebsd $1 + else + # Assuming Linux + check_space_linux + umount_ramfs_linux $1 + mount_ramfs_linux $1 + fi +} + +function umount_ramfs_local_all() { + for RAMDISKPATH in "${RAMDISKARRAY[@]}" + do + umount_ramfs_local $RAMDISKPATH + done +} + +function umount_ramfs_local() { + if [[ $(uname -a) == Darwin* ]]; then + umount_ramfs_mac $1 + else + umount_ramfs_linux $1 + fi +} + +function run_local() { + init_env + + if [[ ${TIER_ALIAS} != "MEM" ]]; then + # the top tier is not MEM, skip + exit 1 + fi + + mount_type=$1 + case "$mount_type" in + Mount) + USE_SUDO=false + mount_ramfs_local_all + ;; + SudoMount) + USE_SUDO=true + mount_ramfs_local_all + ;; + Umount) + USE_SUDO=false + umount_ramfs_local_all + ;; + SudoUmount) + USE_SUDO=true + umount_ramfs_local_all + ;; + *) + echo -e ${USAGE} >&2 + exit 1 + esac +} + +function main { + case "$1" in + Mount|SudoMount|Umount|SudoUmount) + case "$2" in + ""|local) + run_local $1 + ;; + workers) + ${LAUNCHER} ${BIN}/alluxio-workers-bash.sh ${BIN}/alluxio-mount-bash.sh $1 + ;; + *) + echo -e ${USAGE} >&2 + exit 1 + esac + ;; + *) + echo -e ${USAGE} >&2 + exit 1 + esac +} + +main "$@" diff --git a/bin/alluxio-mount.sh b/bin/alluxio-mount.sh index 8c83494cef3b..1a05df0d91c7 100755 --- a/bin/alluxio-mount.sh +++ b/bin/alluxio-mount.sh @@ -10,255 +10,7 @@ # See the NOTICE file distributed with this work for information regarding copyright ownership. # -# Starts the Alluxio master on this node. -# Starts an Alluxio worker on each node specified in conf/workers +BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) -. $(dirname "$0")/alluxio-common.sh - -USAGE="Usage: alluxio-mount.sh [Mount|SudoMount|Umount|SudoUmount] [MACHINE] -\nIf omitted, MACHINE is default to be 'local'. MACHINE is one of:\n - local\t\t\tMount local machine\n - workers\t\tMount all the workers on worker nodes" - -function init_env() { - local libexec_dir=${ALLUXIO_LIBEXEC_DIR:-"${BIN}"/../libexec} - . ${libexec_dir}/alluxio-config.sh - RAMDISK_SIZE=$(${BIN}/alluxio getConf --unit B alluxio.worker.ramdisk.size) - TIER_ALIAS=$(${BIN}/alluxio getConf alluxio.worker.tieredstore.level0.alias) - get_ramdisk_array -} - -function check_space_linux() { - local total_mem=$(($(cat /proc/meminfo | awk 'NR==1{print $2}') * 1024)) - if [[ ${total_mem} -lt ${RAMDISK_SIZE} ]]; then - echo "ERROR: Memory(${total_mem}) is less than requested ramdisk size(${RAMDISK_SIZE}). Please - reduce alluxio.worker.ramdisk.size in alluxio-site.properties" >&2 - exit 1 - fi -} - -function mount_ramfs_linux() { - TIER_PATH=${1} - echo "Formatting RamFS: ${TIER_PATH} (${RAMDISK_SIZE})" - if [[ ${USE_SUDO} == true ]]; then - sudo mkdir -p ${TIER_PATH} - else - mkdir -p ${TIER_PATH} - fi - if [[ $? -ne 0 ]]; then - echo "ERROR: mkdir ${TIER_PATH} failed" >&2 - exit 1 - fi - - if [[ ${USE_SUDO} == true ]]; then - sudo mount -t ramfs -o size=${RAMDISK_SIZE} ramfs ${TIER_PATH} - else - mount -t ramfs -o size=${RAMDISK_SIZE} ramfs ${TIER_PATH} - fi - if [[ $? -ne 0 ]]; then - echo "ERROR: mount RamFS ${TIER_PATH} failed" >&2 - exit 1 - fi - - if [[ ${USE_SUDO} == true ]]; then - sudo chmod a+w ${TIER_PATH} - else - chmod a+w ${TIER_PATH} - fi - if [[ $? -ne 0 ]]; then - echo "ERROR: chmod RamFS ${TIER_PATH} failed" >&2 - exit 1 - fi -} - -function umount_ramfs_linux() { - TIER_PATH=${1} - if mount | grep -E "(^|[[:space:]])${TIER_PATH}($|[[:space:]])" > /dev/null; then - echo "Unmounting ${TIER_PATH}" - if [[ ${USE_SUDO} == true ]]; then - sudo umount -l -f ${TIER_PATH} - else - umount -l -f ${TIER_PATH} - fi - if [[ $? -ne 0 ]]; then - echo "ERROR: umount RamFS ${TIER_PATH} failed" >&2 - exit 1 - fi - fi -} - -function check_space_freebsd() { - local total_mem=$(sysctl -n hw.usermem) - if [[ ${total_mem} -lt ${RAMDISK_SIZE} ]]; then - echo "ERROR: Memory(${total_mem}) is less than requested ramdisk size(${RAMDISK_SIZE}). Please - reduce alluxio.worker.ramdisk.size in alluxio-site.properties" >&2 - exit 1 - fi -} - -function mount_ramfs_freebsd() { - TIER_PATH=${1} - echo "Formatting RamFS: ${TIER_PATH} (${RAMDISK_SIZE})" - if [[ ${USE_SUDO} == true ]]; then - sudo mkdir -p ${TIER_PATH} - else - mkdir -p ${TIER_PATH} - fi - if [[ $? -ne 0 ]]; then - echo "ERROR: mkdir ${TIER_PATH} failed" >&2 - exit 1 - fi - - if [[ ${USE_SUDO} == true ]]; then - sudo mount -t tmpfs -o size=${RAMDISK_SIZE} tmpfs ${TIER_PATH} - else - mount -t tmpfs -o size=${RAMDISK_SIZE} tmpfs ${TIER_PATH} - fi - if [[ $? -ne 0 ]]; then - echo "ERROR: mount RamFS ${TIER_PATH} failed" >&2 - exit 1 - fi - - if [[ ${USE_SUDO} == true ]]; then - sudo chmod a+w ${TIER_PATH} - else - chmod a+w ${TIER_PATH} - fi - if [[ $? -ne 0 ]]; then - echo "ERROR: chmod RamFS ${TIER_PATH} failed" >&2 - exit 1 - fi -} - -function umount_ramfs_freebsd() { - TIER_PATH=${1} - if mount | grep -E "(^|[[:space:]])${TIER_PATH}($|[[:space:]])" > /dev/null; then - echo "Unmounting ${TIER_PATH}" - if [[ ${USE_SUDO} == true ]]; then - sudo umount -f ${TIER_PATH} - else - umount -f ${TIER_PATH} - fi - if [[ $? -ne 0 ]]; then - echo "ERROR: umount RamFS ${TIER_PATH} failed" >&2 - exit 1 - fi - fi -} - -function mount_ramfs_mac() { - TIER_PATH=${1} - # Convert the memory size to number of sectors. Each sector is 512 Byte. - local num_sectors=$(${BIN}/alluxio runClass alluxio.util.HFSUtils ${RAMDISK_SIZE} 512) - - # Format the RAM FS - echo "Formatting RamFS: ${TIER_PATH} ${num_sectors} sectors (${RAMDISK_SIZE})." - # Remove the "/Volumes/" part so we can get the name of the volume. - diskutil erasevolume HFS+ ${TIER_PATH/#\/Volumes\//} $(hdiutil attach -nomount ram://${num_sectors}) -} - -function umount_ramfs_mac() { - TIER_PATH=${1} - local device=$(df -l | grep -E "(^|[[:space:]])${TIER_PATH}($|[[:space:]])" | cut -d " " -f 1) - if [[ -n "${device}" ]]; then - echo "Unmounting ramfs at ${TIER_PATH}" - hdiutil detach -force ${device} - else - echo "Ramfs is not currently mounted at ${TIER_PATH}" - fi -} - -function mount_ramfs_local_all() { - for RAMDISKPATH in "${RAMDISKARRAY[@]}" - do - mount_ramfs_local $RAMDISKPATH - done -} - -function mount_ramfs_local() { - if [[ $(uname -a) == Darwin* ]]; then - # Assuming Mac OS X - umount_ramfs_mac $1 - mount_ramfs_mac $1 - elif [[ $(uname -a) == FreeBSD* ]]; then - # Assuming FreeBSD - check_space_freebsd - umount_ramfs_freebsd $1 - mount_ramfs_freebsd $1 - else - # Assuming Linux - check_space_linux - umount_ramfs_linux $1 - mount_ramfs_linux $1 - fi -} - -function umount_ramfs_local_all() { - for RAMDISKPATH in "${RAMDISKARRAY[@]}" - do - umount_ramfs_local $RAMDISKPATH - done -} - -function umount_ramfs_local() { - if [[ $(uname -a) == Darwin* ]]; then - umount_ramfs_mac $1 - else - umount_ramfs_linux $1 - fi -} - -function run_local() { - init_env - - if [[ ${TIER_ALIAS} != "MEM" ]]; then - # the top tier is not MEM, skip - exit 1 - fi - - mount_type=$1 - case "$mount_type" in - Mount) - USE_SUDO=false - mount_ramfs_local_all - ;; - SudoMount) - USE_SUDO=true - mount_ramfs_local_all - ;; - Umount) - USE_SUDO=false - umount_ramfs_local_all - ;; - SudoUmount) - USE_SUDO=true - umount_ramfs_local_all - ;; - *) - echo -e ${USAGE} >&2 - exit 1 - esac -} - -function main { - case "$1" in - Mount|SudoMount|Umount|SudoUmount) - case "$2" in - ""|local) - run_local $1 - ;; - workers) - ${LAUNCHER} ${BIN}/alluxio-workers.sh ${BIN}/alluxio-mount.sh $1 - ;; - *) - echo -e ${USAGE} >&2 - exit 1 - esac - ;; - *) - echo -e ${USAGE} >&2 - exit 1 - esac -} - -main "$@" +# temporary placeholder that redirects to alluxio-mount-bash.sh to prepare for golangCli branch merge +"${BIN}/alluxio-mount-bash.sh" "$@" diff --git a/bin/alluxio-start-bash.sh b/bin/alluxio-start-bash.sh new file mode 100755 index 000000000000..3c7415f084e0 --- /dev/null +++ b/bin/alluxio-start-bash.sh @@ -0,0 +1,470 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +. $(dirname "$0")/alluxio-common-bash.sh + +#start up alluxio + +USAGE="Usage: alluxio-start-bash.sh [-hNwm] ACTION [MOPT] [-f] [-c cache] +Where ACTION is one of: + all [MOPT] [-c cache] \tStart all master and worker processes. + job_master \tStart the job_master on this node. + job_masters \tStart job_masters on master nodes. + job_worker \tStart a job_worker on this node. + job_workers \tStart job_workers on worker nodes. + local [MOPT] [-c cache] \tStart local master and worker processes. + master \tStart the local master on this node. + masters \tStart masters on master nodes. + proxy \tStart the proxy on this node. + proxies \tStart proxies on master and worker nodes. + worker [MOPT] [-c cache] \tStart a worker on this node. + workers [MOPT] [-c cache] \tStart workers on worker nodes. +" +USAGE+=" +MOPT (Mount Option) is one of: + Mount \tMount the configured RamFS if it is not already mounted. + SudoMount\tMount the configured RamFS using sudo if it is not already mounted. + NoMount \tDo not mount the configured RamFS. + \tNotice: to avoid sudo requirement but using tmpFS in Linux, + set ALLUXIO_RAM_FOLDER=/dev/shm on each worker and use NoMount. + NoMount is assumed if MOPT is not specified. + +-a asynchronously start all processes. The script may exit before all + processes have been started. +-c cache populate the worker ramcache(s) for each worker node from the + specified directory (relative to each worker node's host filesystem). +-f format Journal, UnderFS Data and Workers Folder on master. +-h display this help. +-N do not try to kill previous running processes before starting new ones. +-w wait for processes to end before returning. + +Supported environment variables: + +ALLUXIO_JOB_WORKER_COUNT - identifies how many job workers to start per node (default = 1)" + +ensure_dirs() { + if [[ ! -d "${ALLUXIO_LOGS_DIR}" ]]; then + echo "ALLUXIO_LOGS_DIR: ${ALLUXIO_LOGS_DIR}" + mkdir -p ${ALLUXIO_LOGS_DIR} + fi +} + +get_env() { + DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec + ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} + . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh + CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} +} + +# Pass ram folder to check as $1 +# Return 0 if ram folder is mounted as tmpfs or ramfs, 1 otherwise +is_ram_folder_mounted() { + local mounted_fs="" + if [[ $(uname -s) == Darwin ]]; then + mounted_fs=$(mount -t "hfs" | grep '/Volumes/' | cut -d " " -f 3) + else + mounted_fs=$(mount -t "tmpfs,ramfs" | cut -d " " -f 3) + fi + + for fs in ${mounted_fs}; do + if [[ "${1}" == "${fs}" || "${1}" =~ ^"${fs}"\/.* ]]; then + return 0 + fi + done + + return 1 +} + +# pass mode as $1 +do_mount() { + MOUNT_FAILED=0 + case "$1" in + Mount|SudoMount) + local tier_alias=$(${BIN}/alluxio-bash getConf alluxio.worker.tieredstore.level0.alias) + local tier_path + get_ramdisk_array + if [[ ${tier_alias} != "MEM" ]]; then + echo "Can't Mount/SudoMount when alluxio.worker.tieredstore.level0.alias is not MEM" + exit 1 + fi + + for tier_path in "${RAMDISKARRAY[@]}" + do + is_ram_folder_mounted "${tier_path}" # Returns 0 if already mounted. + if [[ $? -eq 0 ]]; then + echo "Ramdisk ${tier_path} already mounted. Skipping mounting procedure." + else + echo "Ramdisk ${tier_path} not detected. Mounting..." + ${LAUNCHER} "${BIN}/alluxio-mount-bash.sh" "$1" + MOUNT_FAILED=$? + fi + done + ;; + NoMount) + ;; + *) + echo "This command requires a mount mode be specified" >&2 + echo -e "${USAGE}" >&2 + exit 1 + esac +} + +stop() { + ${BIN}/alluxio-stop-bash.sh $1 +} + +start_job_master() { + if [[ "$1" == "-f" ]]; then + ${LAUNCHER} "${BIN}/alluxio-bash" format + fi + + echo "Starting job master @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" + (nohup ${BIN}/launch-process-bash job_master > ${ALLUXIO_LOGS_DIR}/job_master.out 2>&1) & +} + +start_job_masters() { + ${LAUNCHER} "${BIN}/alluxio-masters-bash.sh" "${BIN}/alluxio-start-bash.sh" "-a" "job_master" +} + +start_job_worker() { + echo "Starting job worker @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" + (nohup ${BIN}/launch-process-bash job_worker > ${ALLUXIO_LOGS_DIR}/job_worker.out 2>&1) & + ALLUXIO_JOB_WORKER_JAVA_OPTS=" -Dalluxio.job.worker.rpc.port=0 -Dalluxio.job.worker.web.port=0" + local nworkers=${ALLUXIO_JOB_WORKER_COUNT:-1} + for (( c = 1; c < ${nworkers}; c++ )); do + echo "Starting job worker #$((c+1)) @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" + (ALLUXIO_JOB_WORKER_JAVA_OPTS=${ALLUXIO_JOB_WORKER_JAVA_OPTS} \ + nohup ${BIN}/launch-process-bash job_worker > ${ALLUXIO_LOGS_DIR}/job_worker.out 2>&1) & + done +} + +start_job_workers() { + ${LAUNCHER} "${BIN}/alluxio-workers-bash.sh" "${BIN}/alluxio-start-bash.sh" "-a" "job_worker" +} + +start_master() { + if [[ "$1" == "-f" ]]; then + ${LAUNCHER} ${BIN}/alluxio-bash format + elif [[ `${LAUNCHER} ${BIN}/alluxio-bash getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.type` == "EMBEDDED" ]]; then + JOURNAL_DIR=`${LAUNCHER} ${BIN}/alluxio-bash getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.folder` + if [ -f "${JOURNAL_DIR}" ]; then + echo "Journal location ${JOURNAL_DIR} is a file not a directory. Please remove the file before retrying." + elif [ ! -e "${JOURNAL_DIR}" ]; then + ${LAUNCHER} ${BIN}/alluxio-bash formatMaster + fi + fi + + echo "Starting master @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" + (nohup ${BIN}/launch-process-bash master > ${ALLUXIO_LOGS_DIR}/master.out 2>&1) & +} + +start_masters() { + ${LAUNCHER} "${BIN}/alluxio-masters-bash.sh" "${BIN}/alluxio-start-bash.sh" "-a" "master" $1 +} + +start_proxy() { + echo "Starting proxy @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" + (nohup ${BIN}/launch-process-bash proxy > ${ALLUXIO_LOGS_DIR}/proxy.out 2>&1) & +} + +start_proxies() { + ${LAUNCHER} "${BIN}/alluxio-masters-bash.sh" "${BIN}/alluxio-start-bash.sh" "-a" "proxy" + ${LAUNCHER} "${BIN}/alluxio-workers-bash.sh" "${BIN}/alluxio-start-bash.sh" "-a" "proxy" +} + +start_worker() { + do_mount $1 + if [ ${MOUNT_FAILED} -ne 0 ] ; then + echo "Mount failed, not starting worker" >&2 + exit 1 + fi + + if [[ ! -z "${cache}" ]] ; then + if [[ ! -e "${cache}" ]] ; then + echo "Cache path ${cache} does not exist; aborting" + exit 2 + fi + if [[ ! -d "${cache}" ]] ; then + echo "Cache path ${cache} is not a directory; aborting" + exit 2 + fi + if [[ -z $(ls -a "${cache}" ) ]]; then + echo "Cache path ${cache} is an empty directory; skipping cache population" + else + echo "Populating worker ramcache(s) with contents from ${cache}" + + get_ramdisk_array # see alluxio-common-bash.sh + for dir in "${RAMDISKARRAY[@]}"; do + if [[ ! -e "${dir}" ]]; then + echo "Alluxio has a configured ramcache path ${dir}, but that path does not exist" + exit 2 + fi + if [[ ! -d "${dir}" ]]; then + echo "Alluxio has a configured ramcache path ${dir}, but that path is not a directory" + exit 2 + fi + + echo "Populating worker ramcache at ${dir} with ${cache}/${dir}" + if [[ ! -e "${cache}/${dir}" ]]; then + echo "Path does not exist: ${cache}/${dir}" + exit 2 + fi + if [[ ! -d "${cache}/${dir}" ]]; then + echo "Path is not a directory: ${cache}/${dir}" + exit 2 + fi + + # recursively delete all (including hidden) files of the ramdisk + # - avoiding deleting the directory itself to avoid permission + # changes/requirements on the parent directory + shopt -s dotglob + rm -rf "${dir}/"* + shopt -u dotglob + + # recursively copy all contents of the src directory + # (including hidden files) to the destination directory + cp -R "${cache}/${dir}/." "${dir}/" + if [[ ${?} -ne 0 ]]; then + echo "Failed to populate ramcache at ${dir} with ${cache}/${dir}" + exit 2 + fi + done + fi + fi + + echo "Starting worker @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" + (nohup ${BIN}/launch-process-bash worker > ${ALLUXIO_LOGS_DIR}/worker.out 2>&1 ) & +} + +start_workers() { + start_opts="" + if [[ -n ${cache} ]]; then + start_opts="-c ${cache}" + fi + ${LAUNCHER} "${BIN}/alluxio-workers-bash.sh" "${BIN}/alluxio-start-bash.sh" "-a" "worker" $1 ${start_opts} +} + +get_offline_worker() { + local run= + local result="" + run=$(ps -ef | grep "alluxio.worker.AlluxioWorker" | grep "java" | wc | awk '{ print $1; }') + if [[ ${run} -eq 0 ]]; then + result=$(hostname -f) + fi + echo "${result}" +} + +get_offline_workers() { + local result="" + local run= + local i=0 + local workers=$(cat "${ALLUXIO_CONF_DIR}/workers" | sed "s/#.*$//;/^$/d") + for worker in $(echo ${workers}); do + if [[ ${i} -gt 0 ]]; then + result+="," + fi + run=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${worker} \ + ps -ef | grep "alluxio.worker.AlluxioWorker" | grep "java" | wc | awk '{ print $1; }') + if [[ ${run} -eq 0 ]]; then + result+="${worker}" + fi + i=$((i+1)) + done + echo "${result}" +} + +start_monitor() { + local action=$1 + local nodes=$2 + local run= + if [[ -z "${run}" ]]; then + ${LAUNCHER} "${BIN}/alluxio-monitor-bash.sh" "${action}" "${nodes}" + else + echo "Skipping the monitor checks..." + fi +} + +main() { + # get environment + get_env + + # ensure log/data dirs + ensure_dirs + + while getopts "ahNw:" o; do + case "${o}" in + a) + async="true" + ;; + h) + echo -e "${USAGE}" + exit 0 + ;; + N) + killonstart="no" + ;; + w) + wait="true" + ;; + *) + echo -e "${USAGE}" >&2 + exit 1 + ;; + esac + done + + shift $((${OPTIND} - 1)) + + ACTION=$1 + if [[ -z "${ACTION}" ]]; then + echo "Error: no ACTION specified" >&2 + echo -e "${USAGE}" >&2 + exit 1 + fi + shift + + MOPT=$1 + # Set MOPT. + case "${ACTION}" in + all|worker|workers|local) + if [[ -z "${MOPT}" ]]; then + echo "Assuming NoMount by default." + MOPT="NoMount" + elif [[ "${MOPT}" == "-f" ]]; then + echo "Assuming SudoMount given -f option." + MOPT="SudoMount" + else + shift + fi + ;; + *) + MOPT="" + ;; + esac + + OPTIND=1 # reset for `getopts` + while getopts "fc:" o; do + case "${o}" in + f) + FORMAT="-f" + ;; + c) + cache="${OPTARG}" + ;; + *) + echo -e "${USAGE}" >&2 + exit 1 + ;; + esac + done + + MONITOR_NODES= + if [[ ! "${async}" ]]; then + case "${ACTION}" in + *) + MONITOR_NODES="" + ;; + esac + fi + + if [[ "${killonstart}" != "no" ]]; then + case "${ACTION}" in + all | local | master | masters | job_master | job_masters | proxy | proxies | worker | workers | job_worker | job_workers ) + stop ${ACTION} + sleep 1 + ;; + esac + fi + + case "${ACTION}" in + all) + start_masters "${FORMAT}" + sleep 2 + start_workers "${MOPT}" + ;; + local) + local master_hostname=$(${BIN}/alluxio-bash getConf alluxio.master.hostname) + local is_master_set_and_local=false + if [[ -n ${master_hostname} ]]; then + local local_addresses=( "localhost" "127.0.0.1" $(hostname -s) $(hostname -f) ) + if [[ $(uname -a) != Darwin* ]]; then + # Assuming Linux + local_addresses+=( $(hostname --ip-address) ) + fi + for local_address in ${local_addresses[*]} + do + if [[ ${local_address} == ${master_hostname} ]]; then + is_master_set_and_local=true + break + fi + done + fi + if [[ ${is_master_set_and_local} != true ]]; then + echo -e "\n# The following line is auto-generated by command \"bin/alluxio-start-bash.sh local\"" \ + >> "${ALLUXIO_CONF_DIR}/alluxio-site.properties" + echo "alluxio.master.hostname=localhost" >> "${ALLUXIO_CONF_DIR}/alluxio-site.properties" + fi + if [[ "${FORMAT}" == "-f" ]]; then + ${LAUNCHER} ${BIN}/alluxio-bash formatJournal + ${LAUNCHER} ${BIN}/alluxio-bash formatWorker + fi + start_master + sleep 2 + start_worker "${MOPT}" + ;; + job_master) + start_job_master + ;; + job_masters) + start_job_masters + ;; + job_worker) + start_job_worker + ;; + job_workers) + start_job_workers + ;; + master) + start_master "${FORMAT}" + ;; + masters) + start_masters + ;; + proxy) + start_proxy + ;; + proxies) + start_proxies + ;; + worker) + start_worker "${MOPT}" + ;; + workers) + start_workers "${MOPT}" + ;; + *) + echo "Error: Invalid ACTION: ${ACTION}" >&2 + echo -e "${USAGE}" >&2 + exit 1 + esac + sleep 2 + + if [[ "${wait}" ]]; then + wait + fi + + if [[ ! "${async}" ]]; then + start_monitor "${ACTION}" "${MONITOR_NODES}" + fi +} + +main "$@" diff --git a/bin/alluxio-start.sh b/bin/alluxio-start.sh index 0b93e89335e9..7af95c94e071 100755 --- a/bin/alluxio-start.sh +++ b/bin/alluxio-start.sh @@ -10,618 +10,18 @@ # See the NOTICE file distributed with this work for information regarding copyright ownership. # -. $(dirname "$0")/alluxio-common.sh +BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) -#start up alluxio +echo "The alluxio-start.sh script is deprecated. Use the \"bin/alluxio process start\" command to start processes." -USAGE="Usage: alluxio-start.sh [-hNwm] [-i backup] ACTION [MOPT] [-f] [-c cache] -Where ACTION is one of: - all [MOPT] [-c cache] \tStart all masters, proxies, and workers. - job_master \tStart the job_master on this node. - job_masters \tStart job_masters on master nodes. - job_worker \tStart a job_worker on this node. - job_workers \tStart job_workers on worker nodes. - local [MOPT] [-c cache] \tStart all processes locally. - master \tStart the local master on this node. - secondary_master \tStart the local secondary master on this node. - masters \tStart masters on master nodes. - proxy \tStart the proxy on this node. - proxies \tStart proxies on master and worker nodes. - safe \tScript will run continuously and start the master if it's not running. - worker [MOPT] [-c cache] \tStart a worker on this node. - workers [MOPT] [-c cache] \tStart workers on worker nodes. - logserver \tStart the logserver - restart_worker \tRestart a failed worker on this node. - restart_workers \tRestart any failed workers on worker nodes. -" -USAGE+=" -MOPT (Mount Option) is one of: - Mount \tMount the configured RamFS if it is not already mounted. - SudoMount\tMount the configured RamFS using sudo if it is not already mounted. - NoMount \tDo not mount the configured RamFS. - \tNotice: to avoid sudo requirement but using tmpFS in Linux, - set ALLUXIO_RAM_FOLDER=/dev/shm on each worker and use NoMount. - NoMount is assumed if MOPT is not specified. - --a asynchronously start all processes. The script may exit before all - processes have been started. --c cache populate the worker ramcache(s) for each worker node from the - specified directory (relative to each worker node's host filesystem). --f format Journal, UnderFS Data and Workers Folder on master. --h display this help. --i backup a journal backup to restore the master from. The backup should be - a URI path within the root under filesystem, e.g. - hdfs://mycluster/alluxio_backups/alluxio-journal-YYYY-MM-DD-timestamp.gz. --N do not try to kill previous running processes before starting new ones. --w wait for processes to end before returning. - -Supported environment variables: - -ALLUXIO_JOB_WORKER_COUNT - identifies how many job workers to start per node (default = 1)" - -ensure_dirs() { - if [[ ! -d "${ALLUXIO_LOGS_DIR}" ]]; then - echo "ALLUXIO_LOGS_DIR: ${ALLUXIO_LOGS_DIR}" - mkdir -p ${ALLUXIO_LOGS_DIR} - fi -} - -get_env() { - DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec - ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} - . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh - CLASSPATH=${ALLUXIO_SERVER_CLASSPATH} -} - -# Pass ram folder to check as $1 -# Return 0 if ram folder is mounted as tmpfs or ramfs, 1 otherwise -is_ram_folder_mounted() { - local mounted_fs="" - if [[ $(uname -s) == Darwin ]]; then - mounted_fs=$(mount -t "hfs" | grep '/Volumes/' | cut -d " " -f 3) - else - mounted_fs=$(mount -t "tmpfs,ramfs" | cut -d " " -f 3) - fi - - for fs in ${mounted_fs}; do - if [[ "${1}" == "${fs}" || "${1}" =~ ^"${fs}"\/.* ]]; then - return 0 - fi - done - - return 1 -} - -check_mount_mode() { - case $1 in - Mount);; - SudoMount);; - NoMount) - local tier_alias=$(${BIN}/alluxio getConf alluxio.worker.tieredstore.level0.alias) - local tier_path - get_ramdisk_array - if [[ ${tier_alias} != "MEM" ]]; then - # if the top tier is not MEM, skip check - return - fi - for tier_path in "${RAMDISKARRAY[@]}" - do - is_ram_folder_mounted "${tier_path}" - if [[ $? -ne 0 ]]; then - echo "ERROR: Ramdisk ${tier_path} is not mounted with mount option NoMount. Use alluxio-mount.sh to mount ramdisk." >&2 - echo -e "${USAGE}" >&2 - exit 1 - fi - - if [[ "${tier_path}" =~ ^"/dev/shm"\/{0,1}$ ]]; then - echo "WARNING: Using tmpFS does not guarantee data to be stored in memory." - echo "WARNING: Check vmstat for memory statistics (e.g. swapping)." - fi - done - ;; - *) - if [[ -z $1 ]]; then - echo "This command requires a mount mode be specified" >&2 - else - echo "Invalid mount mode: $1" >&2 - fi - echo -e "${USAGE}" >&2 - exit 1 - esac -} - -# pass mode as $1 -do_mount() { - MOUNT_FAILED=0 - case "$1" in - Mount|SudoMount) - local tier_alias=$(${BIN}/alluxio getConf alluxio.worker.tieredstore.level0.alias) - local tier_path - get_ramdisk_array - if [[ ${tier_alias} != "MEM" ]]; then - echo "Can't Mount/SudoMount when alluxio.worker.tieredstore.level0.alias is not MEM" - exit 1 - fi - - for tier_path in "${RAMDISKARRAY[@]}" - do - is_ram_folder_mounted "${tier_path}" # Returns 0 if already mounted. - if [[ $? -eq 0 ]]; then - echo "Ramdisk ${tier_path} already mounted. Skipping mounting procedure." - else - echo "Ramdisk ${tier_path} not detected. Mounting..." - ${LAUNCHER} "${BIN}/alluxio-mount.sh" "$1" - MOUNT_FAILED=$? - fi - done - ;; - NoMount) - ;; - *) - echo "This command requires a mount mode be specified" >&2 - echo -e "${USAGE}" >&2 - exit 1 - esac -} - -stop() { - ${BIN}/alluxio-stop.sh $1 -} - -start_job_master() { - if [[ "$1" == "-f" ]]; then - ${LAUNCHER} "${BIN}/alluxio" format - fi - - if [[ ${ALLUXIO_MASTER_SECONDARY} != "true" ]]; then - echo "Starting job master @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" - (nohup ${BIN}/launch-process job_master > ${ALLUXIO_LOGS_DIR}/job_master.out 2>&1) & - fi -} - -start_job_masters() { - ${LAUNCHER} "${BIN}/alluxio-masters.sh" "${BIN}/alluxio-start.sh" "-a" "job_master" -} - -start_job_worker() { - echo "Starting job worker @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" - (nohup ${BIN}/launch-process job_worker > ${ALLUXIO_LOGS_DIR}/job_worker.out 2>&1) & - ALLUXIO_JOB_WORKER_JAVA_OPTS=" -Dalluxio.job.worker.rpc.port=0 -Dalluxio.job.worker.web.port=0" - local nworkers=${ALLUXIO_JOB_WORKER_COUNT:-1} - for (( c = 1; c < ${nworkers}; c++ )); do - echo "Starting job worker #$((c+1)) @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" - (ALLUXIO_JOB_WORKER_JAVA_OPTS=${ALLUXIO_JOB_WORKER_JAVA_OPTS} \ - nohup ${BIN}/launch-process job_worker > ${ALLUXIO_LOGS_DIR}/job_worker.out 2>&1) & - done -} - -start_job_workers() { - ${LAUNCHER} "${BIN}/alluxio-workers.sh" "${BIN}/alluxio-start.sh" "-a" "job_worker" -} - -start_logserver() { - if [[ ! -d "${ALLUXIO_LOGSERVER_LOGS_DIR}" ]]; then - echo "ALLUXIO_LOGSERVER_LOGS_DIR: ${ALLUXIO_LOGSERVER_LOGS_DIR}" - mkdir -p ${ALLUXIO_LOGSERVER_LOGS_DIR} - fi - - echo "Starting logserver @ $(hostname -f)." - (ALLUXIO_LOGSERVER_LOGS_DIR="${ALLUXIO_LOGSERVER_LOGS_DIR}" \ - nohup ${BIN}/launch-process logserver > ${ALLUXIO_LOGS_DIR}/logserver.out 2>&1) & - # Wait for 1s before starting other Alluxio servers, otherwise may cause race condition - # leading to connection errors. - sleep 1 -} - -start_master() { - if [[ "$1" == "-f" ]]; then - ${LAUNCHER} ${BIN}/alluxio format - elif [[ `${LAUNCHER} ${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.type` == "EMBEDDED" ]]; then - JOURNAL_DIR=`${LAUNCHER} ${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.folder` - if [ -f "${JOURNAL_DIR}" ]; then - echo "Journal location ${JOURNAL_DIR} is a file not a directory. Please remove the file before retrying." - elif [ ! -e "${JOURNAL_DIR}" ]; then - ${LAUNCHER} ${BIN}/alluxio formatMaster - fi - fi - - if [[ ${ALLUXIO_MASTER_SECONDARY} == "true" ]]; then - if [[ `${LAUNCHER} ${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.journal.type` == "EMBEDDED" ]]; then - echo "Secondary master is not supported for journal type: EMBEDDED" - exit 1 - fi - echo "Starting secondary master @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" - (nohup ${BIN}/launch-process secondary_master > ${ALLUXIO_LOGS_DIR}/secondary_master.out 2>&1) & - else - echo "Starting master @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" - (JOURNAL_BACKUP="${journal_backup}" nohup ${BIN}/launch-process master > ${ALLUXIO_LOGS_DIR}/master.out 2>&1) & - fi -} - -start_masters() { - start_opts="" - if [[ -n ${journal_backup} ]]; then - start_opts="-i ${journal_backup}" - fi - ${LAUNCHER} "${BIN}/alluxio-masters.sh" "${BIN}/alluxio-start.sh" ${start_opts} "-a" "master" $1 -} - -start_proxy() { - echo "Starting proxy @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" - (nohup ${BIN}/launch-process proxy > ${ALLUXIO_LOGS_DIR}/proxy.out 2>&1) & -} - -start_proxies() { - ${LAUNCHER} "${BIN}/alluxio-masters.sh" "${BIN}/alluxio-start.sh" "-a" "proxy" - ${LAUNCHER} "${BIN}/alluxio-workers.sh" "${BIN}/alluxio-start.sh" "-a" "proxy" -} - -start_worker() { - do_mount $1 - if [ ${MOUNT_FAILED} -ne 0 ] ; then - echo "Mount failed, not starting worker" >&2 - exit 1 - fi - - if [[ ! -z "${cache}" ]] ; then - if [[ ! -e "${cache}" ]] ; then - echo "Cache path ${cache} does not exist; aborting" - exit 2 - fi - if [[ ! -d "${cache}" ]] ; then - echo "Cache path ${cache} is not a directory; aborting" - exit 2 - fi - if [[ -z $(ls -a "${cache}" ) ]]; then - echo "Cache path ${cache} is an empty directory; skipping cache population" - else - echo "Populating worker ramcache(s) with contents from ${cache}" - - get_ramdisk_array # see alluxio-common.sh - for dir in "${RAMDISKARRAY[@]}"; do - if [[ ! -e "${dir}" ]]; then - echo "Alluxio has a configured ramcache path ${dir}, but that path does not exist" - exit 2 - fi - if [[ ! -d "${dir}" ]]; then - echo "Alluxio has a configured ramcache path ${dir}, but that path is not a directory" - exit 2 - fi - - echo "Populating worker ramcache at ${dir} with ${cache}/${dir}" - if [[ ! -e "${cache}/${dir}" ]]; then - echo "Path does not exist: ${cache}/${dir}" - exit 2 - fi - if [[ ! -d "${cache}/${dir}" ]]; then - echo "Path is not a directory: ${cache}/${dir}" - exit 2 - fi - - # recursively delete all (including hidden) files of the ramdisk - # - avoiding deleting the directory itself to avoid permission - # changes/requirements on the parent directory - shopt -s dotglob - rm -rf "${dir}/"* - shopt -u dotglob - - # recursively copy all contents of the src directory - # (including hidden files) to the destination directory - cp -R "${cache}/${dir}/." "${dir}/" - if [[ ${?} -ne 0 ]]; then - echo "Failed to populate ramcache at ${dir} with ${cache}/${dir}" - exit 2 - fi - done - fi - fi - - echo "Starting worker @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" - (nohup ${BIN}/launch-process worker > ${ALLUXIO_LOGS_DIR}/worker.out 2>&1 ) & -} - -start_workers() { - start_opts="" - if [[ -n ${cache} ]]; then - start_opts="-c ${cache}" - fi - ${LAUNCHER} "${BIN}/alluxio-workers.sh" "${BIN}/alluxio-start.sh" "-a" "worker" $1 ${start_opts} -} - -restart_worker() { - RUN=$(ps -ef | grep "alluxio.worker.AlluxioWorker" | grep "java" | wc | awk '{ print $1; }') - if [[ ${RUN} -eq 0 ]]; then - echo "Restarting worker @ $(hostname -f). Logging to ${ALLUXIO_LOGS_DIR}" - (nohup ${BIN}/launch-process worker > ${ALLUXIO_LOGS_DIR}/worker.out 2>&1) & - fi -} - -restart_workers() { - ${LAUNCHER} "${BIN}/alluxio-workers.sh" "${BIN}/alluxio-start.sh" "restart_worker" -} - -get_offline_worker() { - local run= - local result="" - run=$(ps -ef | grep "alluxio.worker.AlluxioWorker" | grep "java" | wc | awk '{ print $1; }') - if [[ ${run} -eq 0 ]]; then - result=$(hostname -f) - fi - echo "${result}" -} - -get_offline_workers() { - local result="" - local run= - local i=0 - local workers=$(cat "${ALLUXIO_CONF_DIR}/workers" | sed "s/#.*$//;/^$/d") - for worker in $(echo ${workers}); do - if [[ ${i} -gt 0 ]]; then - result+="," - fi - run=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${worker} \ - ps -ef | grep "alluxio.worker.AlluxioWorker" | grep "java" | wc | awk '{ print $1; }') - if [[ ${run} -eq 0 ]]; then - result+="${worker}" - fi - i=$((i+1)) - done - echo "${result}" -} - -start_monitor() { - local action=$1 - local nodes=$2 - local run= - if [[ "${action}" == "restart_worker" ]]; then - action="worker" - if [[ -z "${nodes}" ]]; then - run="false" - fi - elif [[ "${action}" == "restart_workers" ]]; then - action="workers" - if [[ -z "${nodes}" ]]; then - run="false" - fi - elif [[ "${action}" == "logserver" || "${action}" == "safe" ]]; then - run="false" - fi - if [[ -z "${run}" ]]; then - ${LAUNCHER} "${BIN}/alluxio-monitor.sh" "${action}" "${nodes}" - else - echo "Skipping the monitor checks..." - fi -} - -run_safe() { - while [ 1 ] - do - RUN=$(ps -ef | grep "alluxio.master.AlluxioMaster" | grep "java" | wc | awk '{ print $1; }') - if [[ ${RUN} -eq 0 ]]; then - echo "Restarting the system master..." - start_master - fi - echo "Alluxio is running... " - sleep 2 - done -} - -main() { - # get environment - get_env - - # ensure log/data dirs - ensure_dirs - - while getopts "ahNwi:" o; do - case "${o}" in - a) - async="true" - ;; - h) - echo -e "${USAGE}" - exit 0 - ;; - i) - journal_backup=${OPTARG} - ;; - N) - killonstart="no" - ;; - w) - wait="true" - ;; - *) - echo -e "${USAGE}" >&2 - exit 1 - ;; - esac - done - - shift $((${OPTIND} - 1)) - - ACTION=$1 - if [[ -z "${ACTION}" ]]; then - echo "Error: no ACTION specified" >&2 - echo -e "${USAGE}" >&2 - exit 1 - fi - shift - - MOPT=$1 - # Set MOPT. - case "${ACTION}" in - all|worker|workers|local) - if [[ -z "${MOPT}" ]]; then - echo "Assuming NoMount by default." - MOPT="NoMount" - elif [[ "${MOPT}" == "-f" ]]; then - echo "Assuming SudoMount given -f option." - MOPT="SudoMount" - else - shift - fi - if [[ "${ACTION}" = "worker" ]] || [[ "${ACTION}" = "local" ]]; then - check_mount_mode "${MOPT}" - fi - ;; - *) - MOPT="" - ;; - esac - - OPTIND=1 # reset for `getopts` - while getopts "fc:" o; do - case "${o}" in - f) - FORMAT="-f" - ;; - c) - cache="${OPTARG}" - ;; - *) - echo -e "${USAGE}" >&2 - exit 1 - ;; - esac - done - - MONITOR_NODES= - if [[ ! "${async}" ]]; then - case "${ACTION}" in - restart_worker) - MONITOR_NODES=$(get_offline_worker) - ;; - restart_workers) - MONITOR_NODES=$(get_offline_workers) - ;; - *) - MONITOR_NODES="" - ;; - esac - fi - - if [[ "${killonstart}" != "no" ]]; then - case "${ACTION}" in - all | local | master | masters | secondary_master | job_master | job_masters | proxy | proxies | worker | workers | job_worker | job_workers | logserver) - stop ${ACTION} - sleep 1 - ;; - esac - fi - - case "${ACTION}" in - all) - start_masters "${FORMAT}" - start_job_masters - sleep 2 - start_workers "${MOPT}" - start_job_workers - start_proxies - ;; - local) - local master_hostname=$(${BIN}/alluxio getConf alluxio.master.hostname) - local is_master_set_and_local=false - if [[ -n ${master_hostname} ]]; then - local local_addresses=( "localhost" "127.0.0.1" $(hostname -s) $(hostname -f) ) - if [[ $(uname -a) != Darwin* ]]; then - # Assuming Linux - local_addresses+=( $(hostname --ip-address) ) - fi - for local_address in ${local_addresses[*]} - do - if [[ ${local_address} == ${master_hostname} ]]; then - is_master_set_and_local=true - break - fi - done - fi - if [[ ${is_master_set_and_local} != true ]]; then - echo -e "\n# The following line is auto-generated by command \"bin/alluxio-start.sh local\"" \ - >> "${ALLUXIO_CONF_DIR}/alluxio-site.properties" - echo "alluxio.master.hostname=localhost" >> "${ALLUXIO_CONF_DIR}/alluxio-site.properties" - fi - if [[ "${FORMAT}" == "-f" ]]; then - ${LAUNCHER} ${BIN}/alluxio formatJournal - ${LAUNCHER} ${BIN}/alluxio formatWorker - fi - start_master - ALLUXIO_MASTER_SECONDARY=true - # We only start a secondary master when using a UFS journal. - local journal_type=$(${BIN}/alluxio getConf ${ALLUXIO_MASTER_JAVA_OPTS} \ - alluxio.master.journal.type | awk '{print toupper($0)}') - if [[ ${journal_type} == "UFS" ]]; then - start_master - fi - ALLUXIO_MASTER_SECONDARY=false - start_job_master - sleep 2 - start_worker "${MOPT}" - start_job_worker - start_proxy - ;; - job_master) - start_job_master - ;; - job_masters) - start_job_masters - ;; - job_worker) - start_job_worker - ;; - job_workers) - start_job_workers - ;; - master) - start_master "${FORMAT}" - ;; - secondary_master) - ALLUXIO_MASTER_SECONDARY=true - async=true # there does not exist a monitor process for secondary_master - start_master - ALLUXIO_MASTER_SECONDARY=false - ;; - masters) - start_masters - ;; - proxy) - start_proxy - ;; - proxies) - start_proxies - ;; - restart_worker) - restart_worker - ;; - restart_workers) - restart_workers - ;; - safe) - run_safe - ;; - worker) - start_worker "${MOPT}" - ;; - workers) - start_workers "${MOPT}" - ;; - logserver) - start_logserver - ;; - *) - echo "Error: Invalid ACTION: ${ACTION}" >&2 - echo -e "${USAGE}" >&2 - exit 1 - esac - sleep 2 - - if [[ "${wait}" ]]; then - wait - fi - - if [[ ! "${async}" ]]; then - start_monitor "${ACTION}" "${MONITOR_NODES}" +# while the leading argument starts with a -, continue to parse flags +# this will skip the mount related arguments that are no longer relevant and would cause an error to the new start command +startArgs=() +for arg in "$@"; do + startArgs+=("${arg}") + if [[ "${arg}" != -* ]]; then + break fi -} +done -main "$@" +"${BIN}"/alluxio process start "${startArgs[@]}" diff --git a/bin/alluxio-stop-bash.sh b/bin/alluxio-stop-bash.sh new file mode 100755 index 000000000000..89d8e9556b88 --- /dev/null +++ b/bin/alluxio-stop-bash.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +. $(dirname "$0")/alluxio-common-bash.sh + +USAGE="Usage: alluxio-stop-bash.sh [-h] [component] [-c cache] [-s] +Where component is one of: + all [-c cache] \tStop all masters, proxies, and workers. + job_master \tStop local job master. + job_masters \tStop job masters on master nodes. + job_worker \tStop local job worker. + job_workers \tStop job workers on worker nodes. + local [-c cache] \tStop all processes locally. + master \tStop local primary master. + masters \tStop masters on master nodes. + proxy \tStop local proxy. + proxies \tStop proxies on master and worker nodes. + worker [-c cache] \tStop local worker. + workers [-c cache] \tStop workers on worker nodes. +" +USAGE+=" +-c cache save the worker ramcache(s) from the worker node(s) to the + specified directory (relative to each worker node's host filesystem). +-s processes won't be forcibly killed even if operation is timeout. +-h display this help." + +DEFAULT_LIBEXEC_DIR="${BIN}/../libexec" +ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} +. ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh +declare -a KILL_COMMAND + +generate_kill_command() { + unset KILL_COMMAND + local process_class=$1 + KILL_COMMAND+=("${BIN}/alluxio-bash" "killAll") + if [[ "${soft_kill}" = true ]]; then + KILL_COMMAND+=("-s") + fi + KILL_COMMAND+=("${process_class}") +} + +generate_cluster_kill_command() { + unset KILL_COMMAND + local script_type=$1 + local process_type=$2 + if [[ "${script_type}" = "master" ]]; then + KILL_COMMAND+=("${BIN}/alluxio-masters-bash.sh" "${BIN}/alluxio-stop-bash.sh") + elif [[ "${script_type}" = "worker" ]]; then + KILL_COMMAND+=("${BIN}/alluxio-workers-bash.sh" "${BIN}/alluxio-stop-bash.sh") + else + echo "Error: Invalid script_type: ${script_type} " + exit 1 + fi + KILL_COMMAND+=("${process_type}") + if [[ "${soft_kill}" = true ]]; then + KILL_COMMAND+=("-s") + fi +} + +stop_job_master() { + generate_kill_command "alluxio.master.AlluxioJobMaster" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_job_masters() { + generate_cluster_kill_command "master" "job_master" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_job_worker() { + generate_kill_command "alluxio.worker.AlluxioJobWorker" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_job_workers() { + generate_cluster_kill_command "worker" "job_worker" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_master() { + generate_kill_command "alluxio.master.AlluxioMaster" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_masters() { + generate_cluster_kill_command "master" "master" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_proxy() { + generate_kill_command "alluxio.proxy.AlluxioProxy" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_proxies() { + generate_cluster_kill_command "master" "proxy" + ${LAUNCHER} "${KILL_COMMAND[@]}" + generate_cluster_kill_command "worker" "proxy" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_worker() { + if [[ ! -z "${cache}" ]]; then + echo "Cache directory: ${cache}" + if [[ -d "${cache}" ]]; then + echo "Directory already exists at ${cache}; overwritting its contents" + # recursively delete all (including hidden) files of the ramdisk + # - avoiding deleting the directory itself to avoid permission + # changes/requirements on the parent directory + shopt -s dotglob + rm -rf "${cache}/"* + shopt -u dotglob + elif [[ -e "${cache}" ]]; then + echo "Non-directory file already exists at the path ${cache}; aborting" + exit 1 + else + mkdir -p "${cache}" + if [[ ${?} -ne 0 ]]; then + echo "Failed to create directory: ${cache}" + exit 2 + fi + fi + + get_ramdisk_array # see alluxio-common-bash.sh + for dir in "${RAMDISKARRAY[@]}"; do + if [[ ! -e "${dir}" ]]; then + echo "Alluxio has a configured ramcache path ${dir}, but that path does not exist" + exit 2 + fi + if [[ ! -d "${dir}" ]]; then + echo "Alluxio has a configured ramcache path ${dir}, but that path is not a directory" + exit 2 + fi + + echo "Saving worker ramcache at ${dir} to ${cache}/${dir}" + mkdir -p "${cache}/${dir}" + if [[ ${?} -ne 0 ]]; then + echo "Failed to create directory: ${cache}/${dir}" + exit 2 + fi + + # recursively copy all contents of the src directory + # (including hidden files) to the destination directory + cp -R "${dir}/." "${cache}/${dir}/" + if [[ ${?} -ne 0 ]]; then + echo "Failed to copy worker ramcache from ${dir} to ${cache}/${dir}" + exit 1 + fi + done + fi + generate_kill_command "alluxio.worker.AlluxioWorker" + ${LAUNCHER} "${KILL_COMMAND[@]}" +} + +stop_workers() { + start_opts="" + if [[ -n ${cache} ]]; then + start_opts="-c ${cache}" + fi + generate_cluster_kill_command "worker" "worker" + ${LAUNCHER} "${KILL_COMMAND[@]}" ${start_opts} +} + +WHAT=${1:--h} + +# shift argument index for getopts +shift +while getopts "c:s" o; do + case "${o}" in + c) + cache="${OPTARG}" + ;; + s) + soft_kill=true + ;; + *) + echo -e "${USAGE}" >&2 + exit 1 + ;; + esac +done + +case "${WHAT}" in + all) + stop_workers + stop_masters + ;; + local) + stop_worker + stop_master + ;; + job_master) + stop_job_master + ;; + job_masters) + stop_job_masters + ;; + job_worker) + stop_job_worker + ;; + job_workers) + stop_job_workers + ;; + master) + stop_master + ;; + masters) + stop_masters + ;; + proxy) + stop_proxy + ;; + proxies) + stop_proxies + ;; + worker) + stop_worker + ;; + workers) + stop_workers + ;; + -h) + echo -e "${USAGE}" + exit 0 + ;; + *) + echo "Error: Invalid component: ${WHAT}" >&2 + echo -e "${USAGE}" >&2 + exit 1 + ;; +esac diff --git a/bin/alluxio-stop.sh b/bin/alluxio-stop.sh index d8bdd95b251b..53ff541e87ff 100755 --- a/bin/alluxio-stop.sh +++ b/bin/alluxio-stop.sh @@ -10,261 +10,8 @@ # See the NOTICE file distributed with this work for information regarding copyright ownership. # -. $(dirname "$0")/alluxio-common.sh +BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) -USAGE="Usage: alluxio-stop.sh [-h] [component] [-c cache] [-s] -Where component is one of: - all [-c cache] \tStop all masters, proxies, and workers. - job_master \tStop local job master. - job_masters \tStop job masters on master nodes. - job_worker \tStop local job worker. - job_workers \tStop job workers on worker nodes. - local [-c cache] \tStop all processes locally. - master \tStop local primary master. - secondary_master \tStop local secondary master. - masters \tStop masters on master nodes. - proxy \tStop local proxy. - proxies \tStop proxies on master and worker nodes. - worker [-c cache] \tStop local worker. - workers [-c cache] \tStop workers on worker nodes. - logserver \tStop the logserver -" -USAGE+=" --c cache save the worker ramcache(s) from the worker node(s) to the - specified directory (relative to each worker node's host filesystem). --s processes won't be forcibly killed even if operation is timeout. --h display this help." +echo "The alluxio-stop.sh script is deprecated. Use the \"bin/alluxio process stop\" command to stop processes." -DEFAULT_LIBEXEC_DIR="${BIN}/../libexec" -ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} -. ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh -declare -a KILL_COMMAND - -generate_kill_command() { - unset KILL_COMMAND - local process_class=$1 - KILL_COMMAND+=("${BIN}/alluxio" "killAll") - if [[ "${soft_kill}" = true ]]; then - KILL_COMMAND+=("-s") - fi - KILL_COMMAND+=("${process_class}") -} - -generate_cluster_kill_command() { - unset KILL_COMMAND - local script_type=$1 - local process_type=$2 - if [[ "${script_type}" = "master" ]]; then - KILL_COMMAND+=("${BIN}/alluxio-masters.sh" "${BIN}/alluxio-stop.sh") - elif [[ "${script_type}" = "worker" ]]; then - KILL_COMMAND+=("${BIN}/alluxio-workers.sh" "${BIN}/alluxio-stop.sh") - else - echo "Error: Invalid script_type: ${script_type} " - exit 1 - fi - KILL_COMMAND+=("${process_type}") - if [[ "${soft_kill}" = true ]]; then - KILL_COMMAND+=("-s") - fi -} - -stop_job_master() { - generate_kill_command "alluxio.master.AlluxioJobMaster" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - -stop_job_masters() { - generate_cluster_kill_command "master" "job_master" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - -stop_job_worker() { - generate_kill_command "alluxio.worker.AlluxioJobWorker" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - -stop_job_workers() { - generate_cluster_kill_command "worker" "job_worker" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - -stop_master() { - if [[ ${ALLUXIO_MASTER_SECONDARY} == "true" ]]; then - generate_kill_command "alluxio.master.AlluxioSecondaryMaster" - ${LAUNCHER} "${KILL_COMMAND[@]}" - else - generate_kill_command "alluxio.master.AlluxioMaster" - ${LAUNCHER} "${KILL_COMMAND[@]}" - fi -} - -stop_masters() { - generate_cluster_kill_command "master" "master" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - -stop_proxy() { - generate_kill_command "alluxio.proxy.AlluxioProxy" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - -stop_proxies() { - generate_cluster_kill_command "master" "proxy" - ${LAUNCHER} "${KILL_COMMAND[@]}" - generate_cluster_kill_command "worker" "proxy" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - -stop_worker() { - if [[ ! -z "${cache}" ]]; then - echo "Cache directory: ${cache}" - if [[ -d "${cache}" ]]; then - echo "Directory already exists at ${cache}; overwritting its contents" - # recursively delete all (including hidden) files of the ramdisk - # - avoiding deleting the directory itself to avoid permission - # changes/requirements on the parent directory - shopt -s dotglob - rm -rf "${cache}/"* - shopt -u dotglob - elif [[ -e "${cache}" ]]; then - echo "Non-directory file already exists at the path ${cache}; aborting" - exit 1 - else - mkdir -p "${cache}" - if [[ ${?} -ne 0 ]]; then - echo "Failed to create directory: ${cache}" - exit 2 - fi - fi - - get_ramdisk_array # see alluxio-common.sh - for dir in "${RAMDISKARRAY[@]}"; do - if [[ ! -e "${dir}" ]]; then - echo "Alluxio has a configured ramcache path ${dir}, but that path does not exist" - exit 2 - fi - if [[ ! -d "${dir}" ]]; then - echo "Alluxio has a configured ramcache path ${dir}, but that path is not a directory" - exit 2 - fi - - echo "Saving worker ramcache at ${dir} to ${cache}/${dir}" - mkdir -p "${cache}/${dir}" - if [[ ${?} -ne 0 ]]; then - echo "Failed to create directory: ${cache}/${dir}" - exit 2 - fi - - # recursively copy all contents of the src directory - # (including hidden files) to the destination directory - cp -R "${dir}/." "${cache}/${dir}/" - if [[ ${?} -ne 0 ]]; then - echo "Failed to copy worker ramcache from ${dir} to ${cache}/${dir}" - exit 1 - fi - done - fi - generate_kill_command "alluxio.worker.AlluxioWorker" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - -stop_workers() { - start_opts="" - if [[ -n ${cache} ]]; then - start_opts="-c ${cache}" - fi - generate_cluster_kill_command "worker" "worker" - ${LAUNCHER} "${KILL_COMMAND[@]}" ${start_opts} -} - -stop_logserver() { - generate_kill_command "alluxio.logserver.AlluxioLogServer" - ${LAUNCHER} "${KILL_COMMAND[@]}" -} - - -WHAT=${1:--h} - -# shift argument index for getopts -shift -while getopts "c:s" o; do - case "${o}" in - c) - cache="${OPTARG}" - ;; - s) - soft_kill=true - ;; - *) - echo -e "${USAGE}" >&2 - exit 1 - ;; - esac -done - -case "${WHAT}" in - all) - stop_proxies - stop_job_workers - stop_workers - stop_job_masters - stop_masters - ;; - local) - stop_proxy - stop_job_worker - stop_job_master - stop_worker - ALLUXIO_MASTER_SECONDARY=true - stop_master - ALLUXIO_MASTER_SECONDARY=false - stop_master - ;; - job_master) - stop_job_master - ;; - job_masters) - stop_job_masters - ;; - job_worker) - stop_job_worker - ;; - job_workers) - stop_job_workers - ;; - master) - stop_master - ;; - secondary_master) - ALLUXIO_MASTER_SECONDARY=true - stop_master - ALLUXIO_MASTER_SECONDARY=false - ;; - masters) - stop_masters - ;; - proxy) - stop_proxy - ;; - proxies) - stop_proxies - ;; - worker) - stop_worker - ;; - workers) - stop_workers - ;; - logserver) - stop_logserver - ;; - -h) - echo -e "${USAGE}" - exit 0 - ;; - *) - echo "Error: Invalid component: ${WHAT}" >&2 - echo -e "${USAGE}" >&2 - exit 1 - ;; -esac +"${BIN}"/alluxio process stop "$@" diff --git a/bin/alluxio-workers-bash.sh b/bin/alluxio-workers-bash.sh new file mode 100755 index 000000000000..392a6326322e --- /dev/null +++ b/bin/alluxio-workers-bash.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -o pipefail + +. $(dirname "$0")/alluxio-common-bash.sh + +USAGE="Usage: alluxio-workers-bash.sh command..." + +# if no args specified, show usage +if [[ $# -le 0 ]]; then + echo ${USAGE} >&2 + exit 1 +fi + +DEFAULT_LIBEXEC_DIR="${BIN}/../libexec" +ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} +. ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh + +HOSTLIST=($(echo $(cat "${ALLUXIO_CONF_DIR}/workers" | sed "s/#.*$//;/^$/d"))) +mkdir -p "${ALLUXIO_LOGS_DIR}" +ALLUXIO_TASK_LOG="${ALLUXIO_LOGS_DIR}/task.log" + +echo "Executing the following command on all worker nodes and logging to ${ALLUXIO_TASK_LOG}: $@" | tee -a ${ALLUXIO_TASK_LOG} + +for worker in ${HOSTLIST[@]}; do + echo "[${worker}] Connecting as ${USER}..." >> ${ALLUXIO_TASK_LOG} + nohup $(ssh_command ${worker}) ${LAUNCHER} \ + $"${@// /\\ }" 2>&1 | while read line; do echo "[$(date '+%F %T')][${worker}] ${line}"; done >> ${ALLUXIO_TASK_LOG} & + pids[${#pids[@]}]=$! +done + +# wait for all pids +echo "Waiting for tasks to finish..." +has_error=0 +for ((i=0; i< ${#pids[@]}; i++)); do + wait ${pids[$i]} + ret_code=$? + if [[ ${ret_code} -ne 0 ]]; then + has_error=1 + echo "Task on '${HOSTLIST[$i]}' fails, exit code: ${ret_code}" + fi +done + +# only show the log when all tasks run OK! +if [[ ${has_error} -eq 0 ]]; then + echo "All tasks finished" +else + echo "There are task failures, look at ${ALLUXIO_TASK_LOG} for details." +fi diff --git a/bin/alluxio-workers.sh b/bin/alluxio-workers.sh deleted file mode 100755 index 11dc9c9558ba..000000000000 --- a/bin/alluxio-workers.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# -# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 -# (the "License"). You may not use this work except in compliance with the License, which is -# available at www.apache.org/licenses/LICENSE-2.0 -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied, as more fully set forth in the License. -# -# See the NOTICE file distributed with this work for information regarding copyright ownership. -# - -set -o pipefail - -LAUNCHER= -# If debugging is enabled propagate that through to sub-shells -if [[ "$-" == *x* ]]; then - LAUNCHER="bash -x" -fi -BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) - -USAGE="Usage: alluxio-workers.sh command..." - -# if no args specified, show usage -if [[ $# -le 0 ]]; then - echo ${USAGE} >&2 - exit 1 -fi - -DEFAULT_LIBEXEC_DIR="${BIN}/../libexec" -ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} -. ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh - -HOSTLIST=($(echo $(cat "${ALLUXIO_CONF_DIR}/workers" | sed "s/#.*$//;/^$/d"))) -mkdir -p "${ALLUXIO_LOGS_DIR}" -ALLUXIO_TASK_LOG="${ALLUXIO_LOGS_DIR}/task.log" - -echo "Executing the following command on all worker nodes and logging to ${ALLUXIO_TASK_LOG}: $@" | tee -a ${ALLUXIO_TASK_LOG} - -for worker in ${HOSTLIST[@]}; do - echo "[${worker}] Connecting as ${USER}..." >> ${ALLUXIO_TASK_LOG} - nohup ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${worker} ${LAUNCHER} \ - $"${@// /\\ }" 2>&1 | while read line; do echo "[$(date '+%F %T')][${worker}] ${line}"; done >> ${ALLUXIO_TASK_LOG} & - pids[${#pids[@]}]=$! -done - -# wait for all pids -echo "Waiting for tasks to finish..." -has_error=0 -for ((i=0; i< ${#pids[@]}; i++)); do - wait ${pids[$i]} - ret_code=$? - if [[ ${ret_code} -ne 0 ]]; then - has_error=1 - echo "Task on '${HOSTLIST[$i]}' fails, exit code: ${ret_code}" - fi -done - -# only show the log when all tasks run OK! -if [[ ${has_error} -eq 0 ]]; then - echo "All tasks finished" -else - echo "There are task failures, look at ${ALLUXIO_TASK_LOG} for details." -fi diff --git a/bin/launch-process b/bin/launch-process deleted file mode 100755 index cc9c8f75dd24..000000000000 --- a/bin/launch-process +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env bash -# -# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 -# (the "License"). You may not use this work except in compliance with the License, which is -# available at www.apache.org/licenses/LICENSE-2.0 -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied, as more fully set forth in the License. -# -# See the NOTICE file distributed with this work for information regarding copyright ownership. -# - -set -e - -BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) - -USAGE="Usage: launch-process [-c] - -launch-process launches one of the Alluxio processes in the foreground. By -default it will only log to the configured log file. To configure the process, -add configuration properties in alluxio-site.properties or environment -variables in conf/alluxio-env.sh. - -Note that this script does not verify Alluxio configuration before starting. -It simply populates the necessary environment variables and launches the -desired process in the foreground. It is recommended to use alluxio-start.sh -unless you know what you are doing. -" -USAGE+=" - One of: master, secondary_master, job_master, worker, - job_worker, proxy, logserver" -USAGE+=" - - -c Provide this flag to also log directly to stdout - - -h Print this help text to stdout -" - -# prints "1" if "$1" contains "$2", "0" otherwise. -# -# Usage example: local result="$(contains "my string" "my")" -# + result="1" -# -# This function uses printf instead of echo because echo, by default, will add -# a newline to its output. This would force the callers of the function to -# strip newlines or force echo to not add newlines. Instead of using "-n" with -# echo, printf is used as a safer and more portable alternative because the "-n" -# argument is not in the UNIX standard. -contains() { - if [[ "$1" = *"$2"* ]]; then - printf "1" - fi - printf "0" -} - -# Sets environment variables by sourcing ${ALLUXIO_HOME}/libexec/alluxio-config.sh -# or the value of ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh -# -# The variables that should be set after running this function: -# - ALLUXIO_JAVA_OPTS -# - ALLUXIO_MASTER_JAVA_OPTS -# - ALLUXIO_SECONDARY_MASTER_JAVA_OPTS -# - ALLUXIO JOB_MASTER_JAVA_OPTS -# - ALLUXIO_WORKER_JAVA_OPTS -# - ALLUXIO_JOB_WORKER_JAVA_OPTS -# - ALLUXIO_PROXY_JAVA_OPTS -# - ALLUXIO_LOGSERVER_JAVA_OPTS - -# If any of these variables are not set after running this function, then the -# rest of this script may not function properly. -# -# Environment Variables: -# ALLUXIO_LIBEXEC_DIR: (Optional) Set this value if you want to use a -# libexec directory other than the one located in -# ${ALLUXIO_HOME}. -get_env() { - DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec - ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} - . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh -} - -# print the help text for this script -# -# Arguments: -# 1: The exit code -print_help() { - if [[ "${1}" -ne "0" ]]; then - echo -e "${USAGE}" >&2 - else - echo -e "${USAGE}" - fi - - exit ${1} -} - - -# Launches an Alluxio process -# -# Arguments: -# 1: any java opts to pass to the process -# 2: the main class to launch -# 3.... Additional arguments to the java program -launch_process() { - local attach_opts - local java_opts - local main_class - - attach_opts=${1} - java_opts=${2} - main_class=${3} - - exec ${JAVA} ${attach_opts} -cp ${ALLUXIO_SERVER_CLASSPATH} ${java_opts} "${main_class}" ${@:4} -} - -# Launch a master process -# -# Environment Variables: -# JOURNAL_BACKUP: (Optional) A location of a journal backup. If set, the -# master will read the value of this variable as a journal -# backup. -launch_master() { - if [[ -n ${JOURNAL_BACKUP} ]]; then - ALLUXIO_MASTER_JAVA_OPTS+=" -Dalluxio.master.journal.init.from.backup=${JOURNAL_BACKUP}" - fi - - # use a default Xmx value for the master - local res="$(contains "${ALLUXIO_MASTER_JAVA_OPTS}" "Xmx")" - if [[ "${res}" -eq "0" ]]; then - ALLUXIO_MASTER_JAVA_OPTS+=" -Xmx8g " - fi - # use a default MetaspaceSize value for the master - res="$(contains "${ALLUXIO_MASTER_JAVA_OPTS}" "XX:MetaspaceSize")" - if [[ "${res}" -eq "0" ]]; then - ALLUXIO_MASTER_JAVA_OPTS+=" -XX:MetaspaceSize=256M " - fi - - launch_process "${ALLUXIO_MASTER_ATTACH_OPTS}" \ - "${ALLUXIO_MASTER_JAVA_OPTS}" \ - "alluxio.master.AlluxioMaster" -} - -# Launch a secondary master process -launch_secondary_master() { - # use a default Xmx value for the master - local res="$(contains "${ALLUXIO_SECONDARY_MASTER_JAVA_OPTS}" "Xmx")" - if [[ "${res}" -eq "0" ]]; then - ALLUXIO_SECONDARY_MASTER_JAVA_OPTS+=" -Xmx8g " - fi - launch_process "${ALLUXIO_SECONDARY_MASTER_ATTACH_OPTS}" \ - "${ALLUXIO_SECONDARY_MASTER_JAVA_OPTS}" \ - "alluxio.master.AlluxioSecondaryMaster" -} - -# Launch a job master process -launch_job_master() { - launch_process "${ALLUXIO_JOB_MASTER_ATTACH_OPTS}" \ - "${ALLUXIO_JOB_MASTER_JAVA_OPTS}" \ - "alluxio.master.AlluxioJobMaster" -} - -# Launch a worker process -launch_worker() { - # use a default Xmx value for the worker - local res="$(contains "${ALLUXIO_WORKER_JAVA_OPTS}" "Xmx")" - if [[ "${res}" -eq "0" ]]; then - ALLUXIO_WORKER_JAVA_OPTS+=" -Xmx4g " - fi - - # use a default MaxDirectMemorySize value for the worker - res="$(contains "${ALLUXIO_WORKER_JAVA_OPTS}" "XX:MaxDirectMemorySize")" - if [[ "${res}" -eq "0" ]]; then - ALLUXIO_WORKER_JAVA_OPTS+=" -XX:MaxDirectMemorySize=4g " - fi - - launch_process "${ALLUXIO_WORKER_ATTACH_OPTS}" \ - "${ALLUXIO_WORKER_JAVA_OPTS}" \ - "alluxio.worker.AlluxioWorker" -} - -# Launch a job worker process -launch_job_worker() { - launch_process "${ALLUXIO_JOB_WORKER_ATTACH_OPTS}" \ - "${ALLUXIO_JOB_WORKER_JAVA_OPTS}" \ - "alluxio.worker.AlluxioJobWorker" -} - -# Launch a proxy process -launch_proxy() { - launch_process "${ALLUXIO_PROXY_ATTACH_OPTS}" \ - "${ALLUXIO_PROXY_JAVA_OPTS}" \ - "alluxio.proxy.AlluxioProxy" -} - -# Launch a logserver process -# -# Environment Variables: -# ALLUXIO_LOGSERVER_LOGS_DIR: This is a mandatory variable. The -# directory where the logserver will -# store logs. -launch_logserver() { - launch_process "${ALLUXIO_LOGSERVER_ATTACH_OPTS}" \ - "${ALLUXIO_LOGSERVER_JAVA_OPTS}" \ - "alluxio.logserver.AlluxioLogServer" \ - "${ALLUXIO_LOGSERVER_LOGS_DIR}" -} - -# Launch the desired process -main() { - if [[ "${#}" -gt "2" || "${#}" -eq "0" ]]; then - print_help 1 - fi - - local logger_var_name - local logger_var_value - local console_logging_enabled - local process - - process=${1} - shift - - while getopts "hc" o; do - case "${o}" in - h) - print_help 0 - ;; - c) - console_logging_enabled="true" - ;; - *) - print_help 1 - ;; - esac - done - - case "${process}" in - master) - logger_var_name="ALLUXIO_MASTER_LOGGER" - logger_var_value="MASTER_LOGGER" - ;; - secondary_master) - logger_var_name="ALLUXIO_SECONDARY_MASTER_LOGGER" - logger_var_value="SECONDARY_MASTER_LOGGER" - ;; - job_master) - logger_var_name="ALLUXIO_JOB_MASTER_LOGGER" - logger_var_value="JOB_MASTER_LOGGER" - ;; - worker) - logger_var_name="ALLUXIO_WORKER_LOGGER" - logger_var_value="WORKER_LOGGER" - ;; - job_worker) - logger_var_name="ALLUXIO_JOB_WORKER_LOGGER" - logger_var_value="JOB_WORKER_LOGGER" - ;; - logserver) - logger_var_name="ALLUXIO_LOGSERVER_LOGGER" - logger_var_value="LOGSERVER_LOGGER" - ;; - proxy) - logger_var_name="ALLUXIO_PROXY_LOGGER" - logger_var_value="PROXY_LOGGER" - ;; - *) - print_help 1 - ;; - esac - - if [[ "${console_logging_enabled}" == "true" ]]; then - local tmp_val=${logger_var_value} - logger_var_value="Console," - logger_var_value+="${tmp_val}" - fi - - # Set the logging variable equal to the appropriate value - eval ${logger_var_name}="${logger_var_value}" - - # This call should set all ALLUXIO_*_JAVA_OPTS variables - get_env - - case "${process}" in - master) - launch_master - ;; - secondary_master) - launch_secondary_master - ;; - job_master) - launch_job_master - ;; - worker) - launch_worker - ;; - job_worker) - launch_job_worker - ;; - logserver) - launch_logserver - ;; - proxy) - launch_proxy - ;; - *) - print_help 1 - ;; - esac -} - -main $@ diff --git a/bin/launch-process-bash b/bin/launch-process-bash new file mode 100755 index 000000000000..6ce641a44033 --- /dev/null +++ b/bin/launch-process-bash @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -e + +BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) + +USAGE="Usage: launch-process-bash [-c] + +launch-process-bash launches one of the Alluxio processes in the foreground. By +default it will only log to the configured log file. To configure the process, +add configuration properties in alluxio-site.properties or environment +variables in conf/alluxio-env.sh. + +Note that this script does not verify Alluxio configuration before starting. +It simply populates the necessary environment variables and launches the +desired process in the foreground. It is recommended to use alluxio-start-bash.sh +unless you know what you are doing. +" +USAGE+=" + One of: master, job_master, worker, + job_worker, proxy" +USAGE+=" + + -c Provide this flag to also log directly to stdout + + -h Print this help text to stdout +" + +# prints "1" if "$1" contains "$2", "0" otherwise. +# +# Usage example: local result="$(contains "my string" "my")" +# + result="1" +# +# This function uses printf instead of echo because echo, by default, will add +# a newline to its output. This would force the callers of the function to +# strip newlines or force echo to not add newlines. Instead of using "-n" with +# echo, printf is used as a safer and more portable alternative because the "-n" +# argument is not in the UNIX standard. +contains() { + if [[ "$1" = *"$2"* ]]; then + printf "1" + else + printf "0" + fi +} + +# Sets environment variables by sourcing ${ALLUXIO_HOME}/libexec/alluxio-config.sh +# or the value of ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh +# +# The variables that should be set after running this function: +# - ALLUXIO_JAVA_OPTS +# - ALLUXIO_MASTER_JAVA_OPTS +# - ALLUXIO JOB_MASTER_JAVA_OPTS +# - ALLUXIO_WORKER_JAVA_OPTS +# - ALLUXIO_JOB_WORKER_JAVA_OPTS +# - ALLUXIO_PROXY_JAVA_OPTS + +# If any of these variables are not set after running this function, then the +# rest of this script may not function properly. +# +# Environment Variables: +# ALLUXIO_LIBEXEC_DIR: (Optional) Set this value if you want to use a +# libexec directory other than the one located in +# ${ALLUXIO_HOME}. +get_env() { + DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec + ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-${DEFAULT_LIBEXEC_DIR}} + . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh +} + +# print the help text for this script +# +# Arguments: +# 1: The exit code +print_help() { + if [[ "${1}" -ne "0" ]]; then + echo -e "${USAGE}" >&2 + else + echo -e "${USAGE}" + fi + + exit ${1} +} + + +# Launches an Alluxio process +# +# Arguments: +# 1: any java opts to pass to the process +# 2: the main class to launch +# 3.... Additional arguments to the java program +launch_process() { + local attach_opts + local java_opts + local main_class + + attach_opts=${1} + java_opts=${2} + main_class=${3} + + exec ${JAVA} ${attach_opts} -cp ${ALLUXIO_SERVER_CLASSPATH} ${java_opts} "${main_class}" ${@:4} +} + +# Launch a master process +launch_master() { + # use a default Xmx value for the master + local contain_xmx="$(contains "${ALLUXIO_MASTER_JAVA_OPTS}" "Xmx")" + local contain_max_percentage="$(contains "${ALLUXIO_MASTER_JAVA_OPTS}" "MaxRAMPercentage")" + if [[ "${contain_xmx}" -eq "0" ]] && [[ "${contain_max_percentage}" -eq "0" ]]; then + ALLUXIO_MASTER_JAVA_OPTS+=" -Xmx8g " + fi + # use a default MetaspaceSize value for the master + res="$(contains "${ALLUXIO_MASTER_JAVA_OPTS}" "XX:MetaspaceSize")" + if [[ "${res}" -eq "0" ]]; then + ALLUXIO_MASTER_JAVA_OPTS+=" -XX:MetaspaceSize=256M " + fi + + launch_process "${ALLUXIO_MASTER_ATTACH_OPTS}" \ + "${ALLUXIO_MASTER_JAVA_OPTS}" \ + "alluxio.master.AlluxioMaster" +} + +# Launch a job master process +launch_job_master() { + launch_process "${ALLUXIO_JOB_MASTER_ATTACH_OPTS}" \ + "${ALLUXIO_JOB_MASTER_JAVA_OPTS}" \ + "alluxio.master.AlluxioJobMaster" +} + +# Launch a worker process +launch_worker() { + # use a default Xmx value for the worker + local contain_xmx="$(contains "${ALLUXIO_WORKER_JAVA_OPTS}" "Xmx")" + local contain_max_percentage="$(contains "${ALLUXIO_WORKER_JAVA_OPTS}" "MaxRAMPercentage")" + if [[ "${contain_xmx}" -eq "0" ]] && [[ "${contain_max_percentage}" -eq "0" ]]; then + ALLUXIO_WORKER_JAVA_OPTS+=" -Xmx4g " + fi + + # use a default MaxDirectMemorySize value for the worker + res="$(contains "${ALLUXIO_WORKER_JAVA_OPTS}" "XX:MaxDirectMemorySize")" + if [[ "${res}" -eq "0" ]]; then + ALLUXIO_WORKER_JAVA_OPTS+=" -XX:MaxDirectMemorySize=4g " + fi + + launch_process "${ALLUXIO_WORKER_ATTACH_OPTS}" \ + "${ALLUXIO_WORKER_JAVA_OPTS}" \ + "alluxio.worker.AlluxioWorker" +} + +# Launch a job worker process +launch_job_worker() { + launch_process "${ALLUXIO_JOB_WORKER_ATTACH_OPTS}" \ + "${ALLUXIO_JOB_WORKER_JAVA_OPTS}" \ + "alluxio.worker.AlluxioJobWorker" +} + +# Launch a proxy process +launch_proxy() { + launch_process "${ALLUXIO_PROXY_ATTACH_OPTS}" \ + "${ALLUXIO_PROXY_JAVA_OPTS}" \ + "alluxio.proxy.AlluxioProxy" +} + +# Launch the desired process +main() { + if [[ "${#}" -gt "2" || "${#}" -eq "0" ]]; then + print_help 1 + fi + + local logger_var_name + local logger_var_value + local console_logging_enabled + local process + + process=${1} + shift + + while getopts "hc" o; do + case "${o}" in + h) + print_help 0 + ;; + c) + console_logging_enabled="true" + ;; + *) + print_help 1 + ;; + esac + done + + case "${process}" in + master) + logger_var_name="ALLUXIO_MASTER_LOGGER" + logger_var_value="MASTER_LOGGER" + ;; + job_master) + logger_var_name="ALLUXIO_JOB_MASTER_LOGGER" + logger_var_value="JOB_MASTER_LOGGER" + ;; + worker) + logger_var_name="ALLUXIO_WORKER_LOGGER" + logger_var_value="WORKER_LOGGER" + ;; + job_worker) + logger_var_name="ALLUXIO_JOB_WORKER_LOGGER" + logger_var_value="JOB_WORKER_LOGGER" + ;; + proxy) + logger_var_name="ALLUXIO_PROXY_LOGGER" + logger_var_value="PROXY_LOGGER" + ;; + *) + print_help 1 + ;; + esac + + if [[ "${console_logging_enabled}" == "true" ]]; then + local tmp_val=${logger_var_value} + logger_var_value="Console," + logger_var_value+="${tmp_val}" + fi + + # Set the logging variable equal to the appropriate value + eval ${logger_var_name}="${logger_var_value}" + + # This call should set all ALLUXIO_*_JAVA_OPTS variables + get_env + + case "${process}" in + master) + launch_master + ;; + job_master) + launch_job_master + ;; + worker) + launch_worker + ;; + job_worker) + launch_job_worker + ;; + proxy) + launch_proxy + ;; + *) + print_help 1 + ;; + esac +} + +main $@ diff --git a/build/checkstyle/suppressions.xml b/build/checkstyle/suppressions.xml index 87b4da24cd8e..d07524f5e934 100644 --- a/build/checkstyle/suppressions.xml +++ b/build/checkstyle/suppressions.xml @@ -3,16 +3,15 @@ - + - + - - + + diff --git a/build/cli/build-cli.sh b/build/cli/build-cli.sh new file mode 100755 index 000000000000..446d0fe0f3be --- /dev/null +++ b/build/cli/build-cli.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -eu + +CWD=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) + +# tuple of GOOS, GOARCH, and combined uname value for binary name +OS_ARCH_TUPLES=( +"linux amd64 Linux-x86_64" +"darwin amd64 Darwin-x86_64" +"linux arm64 Linux-arm64" +"darwin arm64 Darwin-arm64" +) + +MAIN_PATH="cli/main.go" +USAGE="Usage: build-cli.sh [-a] +-a Build executables for all OS and architecture combinations +" + +main() { + build_all="false" + while getopts "a" opt; do + case "${opt}" in + a) + build_all="true" + ;; + *) + echo -e "${USAGE}" >&2 + exit 1 + ;; + esac + done + + # check go and its version + if ! command -v go > /dev/null; then + echo "Could not run the command 'go'. Please check that it is installed and accessible from \$PATH" + exit 1 + fi + go_version=$(go version) + if [[ ! ${go_version} =~ ^go\ version\ go1\.(1[5-9]|[2-9][0-9]) ]]; then + echo "Go version must be 1.15 or later, but got ${go_version}" + exit 1 + fi + + cliBinDir="${CWD}/../../cli/src/alluxio.org/cli/bin" + mkdir -p "${cliBinDir}" + + cd "${CWD}/../../cli/src/alluxio.org/" + go mod tidy + + if [[ ${build_all} == "false" ]]; then + GO111MODULE=on go build -o "${cliBinDir}/alluxioCli-$(uname)-$(uname -m)" "${MAIN_PATH}" + else + for val in "${OS_ARCH_TUPLES[@]}"; do + IFS=" " read -r -a tuple <<< "${val}" + echo "Building executable for ${tuple[0]} ${tuple[1]}" + GO111MODULE=on GOOS="${tuple[0]}" GOARCH="${tuple[1]}" go build -o "${cliBinDir}/alluxioCli-${tuple[2]}" "${MAIN_PATH}" + done + fi +} + +main "$@" diff --git a/build/findbugs/findbugs-exclude.xml b/build/findbugs/findbugs-exclude.xml index 2648d069b85b..0d0608bb99ff 100644 --- a/build/findbugs/findbugs-exclude.xml +++ b/build/findbugs/findbugs-exclude.xml @@ -6,11 +6,6 @@ - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build/version/write_version.sh b/build/version/write_version.sh new file mode 100755 index 000000000000..e2bc95b59eab --- /dev/null +++ b/build/version/write_version.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +echo "VERSION=${1}" > libexec/version.sh + diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 000000000000..2a8585ba6ebe --- /dev/null +++ b/cli/README.md @@ -0,0 +1,70 @@ +# Alluxio CLI + +The Alluxio command line interface is the single entrypoint for users to: +- Initialize and configure the Alluxio cluster +- Start and stop processes +- Expose information about the running cluster +- Interact with the filesystem, running commands such as `ls` and `cp` +- Perform administrator level actions such as format or backup + +The CLI is invoked through the shell script at `bin/alluxio`. +Commands follow the format of: +```console +bin/alluxio [--[=]] [] +``` + +Add the `-h` flag to view more details regarding a service or operation. + +## Layout and naming conventions + +The choice of names for services, operations, and flags should be succinct: short and unambiguous. +Use of a single word is strongly preferred, but otherwise the name parts should be delimited by a dash such as `foo-bar`. + +For example, let's assume there is a `mark` operation as part of a `item` service that can be `set` or `unset` on an item `name`. +The recommended format for a command is +```console +bin/alluxio item mark --set name +bin/alluxio item mark --unset name +``` +where it is expected that either `--set` or `--unset` are specified. +This is preferred over the alternative of two separate commands with `setMark` and `unsetMark` as the operations. + +## User input + +### Flags and arguments +After selecting the desired command, additional user input can be parsed, as a mix of arguments and/or flags: +- Arguments: `bin/alluxio command arg1 arg2 ...` +- Flags: `bin/alluxio command --flag1 --flag2 val2 ...` + +Flags are strictly preferred over arguments. +- The flag name conveys context; an argument does not +- Arguments must be ordered; flags can be declared arbitrarily +- Flags can be designated as required to ensure user input. +- Repeated flags can be defined to capture an ordered list of inputs, ex. `--target target1 --target target2` + +### Input validation + +User inputs should be validated by the CLI command as much as possible as opposed to the resulting invocation. + +## Output conventions and java invocation + +A majority of commands result in invoking a java class with arguments to execute the expected operation and possibly return some output. +The output returned from the java invocation should tend towards being plain or machine parseable, such as a JSON formatted string, +rather than terminal friendly or human readable format. +When appropriate, the CLI command will default to formatting this output to be terminal friendly, with an option to output in a machine parseable format. + +## Documentation + +The markdown file at `docs/en/operation/User-CLI.md` describes all the commands available through the `bin/alluxio` CLI. +This is a file generated by running `bin/alluxio generate user-cli`. + +The command definition determines the generated content: +- The string outlining the general usage of the command +- The list of flags with their name, default value, and description +- A lengthy description, defined by the `Long` field of the command +- A list of examples, defined by the `Examples` field of the command + - The list is generated in the doc by splitting `Example` string by two newline characters + +## References + +Github CLI guidelines: https://primer.style/design/native/cli/foundations diff --git a/cli/src/alluxio.org/cli/bin/.gitignore b/cli/src/alluxio.org/cli/bin/.gitignore new file mode 100644 index 000000000000..33a525764e24 --- /dev/null +++ b/cli/src/alluxio.org/cli/bin/.gitignore @@ -0,0 +1 @@ +alluxioCli* diff --git a/cli/src/alluxio.org/cli/cmd/cache/cache.go b/cli/src/alluxio.org/cli/cmd/cache/cache.go new file mode 100644 index 000000000000..de2e8b3916ce --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/cache/cache.go @@ -0,0 +1,25 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package cache + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "cache", + Description: "Worker-related file system and format operations.", + Commands: []env.Command{ + Format, + Free, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/cache/format.go b/cli/src/alluxio.org/cli/cmd/cache/format.go new file mode 100644 index 000000000000..153e71985d8a --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/cache/format.go @@ -0,0 +1,59 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package cache + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Format = &FormatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "format", + JavaClassName: "alluxio.cli.Format", + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console")}, + }, +} + +type FormatCommand struct { + *env.BaseJavaCommand +} + +func (c *FormatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *FormatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Format.CommandName, + Short: "Format Alluxio worker running locally", + Long: `The format command formats the Alluxio worker on this host. +This deletes all the cached data stored by the worker. Data in the under storage will not be changed. + +> Warning: Format should only be called when the worker is not running`, + Example: `# Format worker +$ ./bin/alluxio cache format`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *FormatCommand) Run(args []string) error { + javaArgs := []string{"worker"} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/cache/free.go b/cli/src/alluxio.org/cli/cmd/cache/free.go new file mode 100644 index 000000000000..f60d68796f6c --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/cache/free.go @@ -0,0 +1,77 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package cache + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Free = &FreeCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "free", + JavaClassName: names.FileSystemShellJavaClass, + }, +} + +type FreeCommand struct { + *env.BaseJavaCommand + worker string + path string +} + +func (c *FreeCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *FreeCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Free.CommandName, + Short: "Synchronously free cached files along a path or held by a specific worker", + Example: `# Free a file by its path +$ ./bin/alluxio cache free --path /path/to/file + +# Free files on a worker +$ ./bin/alluxio cache free --worker `, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.worker, "worker", "", "The worker to free") + cmd.Flags().StringVar(&c.path, "path", "", "The file or directory to free") + cmd.MarkFlagsMutuallyExclusive("worker", "path") + return cmd +} + +func (c *FreeCommand) Run(args []string) error { + var javaArgs []string + if c.worker == "" { + if c.path != "" { + // free directory + javaArgs = append(javaArgs, "free", c.path) + } else { + return stacktrace.NewError("neither worker nor path to free specified") + } + } else { + if c.path == "" { + // free workers + javaArgs = append(javaArgs, "freeWorker", c.worker) + } else { + return stacktrace.NewError("both worker and path to free specified") + } + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/conf/conf.go b/cli/src/alluxio.org/cli/cmd/conf/conf.go new file mode 100644 index 000000000000..1c42c088d4e4 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/conf.go @@ -0,0 +1,25 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "conf", + Description: "Get, set, and validate configuration settings, primarily those defined in conf/alluxio-site.properties", + Commands: []env.Command{ + Get, + Log, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/conf/get.go b/cli/src/alluxio.org/cli/cmd/conf/get.go new file mode 100644 index 000000000000..be002b7c4159 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/get.go @@ -0,0 +1,113 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "bytes" + "fmt" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Get = &GetCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "get", + JavaClassName: "alluxio.cli.GetConf", + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioConfValidationEnabled, false)}, + }, +} + +type GetCommand struct { + *env.BaseJavaCommand + + ShowMaster bool + ShowSource bool + Unit string +} + +func (c *GetCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *GetCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [key]", Get.CommandName), + Short: "Look up a configuration value by its property key or print all configuration if no key is provided", + Long: `The get command prints the configured value for the given key. +If the key is invalid, it returns a nonzero exit code. +If the key is valid but isn't set, an empty string is printed. +If no key is specified, the full configuration is printed. + +> Note: This command does not require the Alluxio cluster to be running.`, + Example: `# Display all the current node configuration +$ ./bin/alluxio conf get + +# Display the value of a property key +$ ./bin/alluxio conf get alluxio.master.hostname + +# Display the configuration of the current running Alluxio leading master +$ ./bin/alluxio conf get --master + +# Display the source of the configuration +$ ./bin/alluxio conf get --source + +# Display the values in a given unit +$ ./bin/alluxio conf get alluxio.user.block.size.bytes.default --unit KB +$ ./bin/alluxio conf get alluxio.master.journal.flush.timeout --unit S +`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVar(&c.ShowMaster, "master", false, "Show configuration properties used by the master") + cmd.Flags().BoolVar(&c.ShowSource, "source", false, "Show source of the configuration property instead of the value") + cmd.Flags().StringVar(&c.Unit, "unit", "", + `Unit of the value to return, converted to correspond to the given unit. +E.g., with "--unit KB", a configuration value of "4096B" will return 4 +Possible options include B, KB, MB, GB, TP, PB, MS, S, M, H, D`) + return cmd +} + +func (c *GetCommand) Run(args []string) error { + var javaArgs []string + if c.ShowMaster { + javaArgs = append(javaArgs, "--master") + } + if c.ShowSource { + javaArgs = append(javaArgs, "--source") + } + if c.Unit != "" { + javaArgs = append(javaArgs, "--unit", c.Unit) + } + javaArgs = append(javaArgs, args...) + + return c.Base().Run(javaArgs) +} + +func (c *GetCommand) FetchValue(key string) (string, error) { + cmd := c.RunJavaClassCmd([]string{key}) + + errBuf := &bytes.Buffer{} + cmd.Stderr = errBuf + + log.Logger.Debugln(cmd.String()) + out, err := cmd.Output() + if err != nil { + return "", stacktrace.Propagate(err, "error getting conf for %v\nstderr: %v", key, errBuf.String()) + } + return string(out), nil +} diff --git a/cli/src/alluxio.org/cli/cmd/conf/log.go b/cli/src/alluxio.org/cli/cmd/conf/log.go new file mode 100644 index 000000000000..a089d44127ed --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/log.go @@ -0,0 +1,84 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "strings" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Log = &LogCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "log", + JavaClassName: "alluxio.cli.LogLevel", + }, +} + +type LogCommand struct { + *env.BaseJavaCommand + + LogName string + Level string + Targets []string +} + +func (c *LogCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LogCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Log.CommandName, + Short: "Get or set the log level for the specified logger", + Long: `The log command returns the current value of or updates the log level of a particular class on specific instances. +Users are able to change Alluxio server-side log levels at runtime. + +The --target flag specifies which processes to apply the log level change to. +The target could be of the form and multiple targets can be listed as comma-separated entries. +The role can be one of master,worker,job_master,job_worker. +Using the role option is useful when an Alluxio process is configured to use a non-standard web port (e.g. if an Alluxio master does not use 19999 as its web port). +The default target value is the primary master, primary job master, all workers and job workers. + +> Note: This command requires the Alluxio cluster to be running.`, + Example: `# Set DEBUG level for DefaultFileSystemMaster class on master processes +$ ./bin/alluxio conf log --logName alluxio.master.file.DefaultFileSystemMaster --target=master --level=DEBUG + +# Set WARN level for PagedDoraWorker class on the worker process on host myHostName +$ ./bin/alluxio conf log --logName alluxio.worker.dora.PagedDoraWorker.java --target=myHostName:worker --level=WARN +`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const name = "name" + cmd.Flags().StringVar(&c.LogName, name, "", "Logger name (ex. alluxio.master.file.DefaultFileSystemMaster)") + cmd.MarkFlagRequired(name) + cmd.Flags().StringVar(&c.Level, "level", "", "If specified, sets the specified logger at the given level") + cmd.Flags().StringSliceVar(&c.Targets, "target", nil, "A target name among . Defaults to master,workers,job_master,job_workers") + return cmd +} + +func (c *LogCommand) Run(_ []string) error { + javaArgs := []string{"--logName", c.LogName} + if c.Level != "" { + javaArgs = append(javaArgs, "--level", c.Level) + } + if len(c.Targets) > 0 { + javaArgs = append(javaArgs, "--target", strings.Join(c.Targets, ",")) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/class.go b/cli/src/alluxio.org/cli/cmd/exec/class.go new file mode 100644 index 000000000000..c4d4edfeb892 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/class.go @@ -0,0 +1,70 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Class = &ClassCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "class", + }, +} + +type ClassCommand struct { + *env.BaseJavaCommand + mainClass string + jarFile string + module string +} + +func (c *ClassCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ClassCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "class", + Short: "Run the main method of an Alluxio class.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.jarFile, "jar", "", + "Determine a JAR file to run.") + cmd.Flags().StringVar(&c.module, "m", "", + "Determine a module to run.") + cmd.MarkFlagsMutuallyExclusive("jar", "m") + return cmd +} + +func (c *ClassCommand) Run(args []string) error { + var javaArgs []string + if c.jarFile != "" { + javaArgs = append(javaArgs, "-jar", c.jarFile) + } else if c.module != "" { + javaArgs = append(javaArgs, "-m", c.module) + } else if len(args) != 0 { + c.JavaClassName = args[0] + } else { + return stacktrace.Propagate(nil, "None of JAR, module, nor a java class is specified") + } + + if len(args) > 1 { + javaArgs = append(javaArgs, args[1:]...) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/exec.go b/cli/src/alluxio.org/cli/cmd/exec/exec.go new file mode 100644 index 000000000000..cd07c9b1b99d --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/exec.go @@ -0,0 +1,29 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "exec", + Description: "Run the main method of an Alluxio class, or end-to-end tests on an Alluxio cluster.", + Commands: []env.Command{ + Class, + TestHdfsMount, + TestRun, + TestUfs, + TestUfsIO, + TestRunCluster, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_basic_io.go b/cli/src/alluxio.org/cli/cmd/exec/test_basic_io.go new file mode 100644 index 000000000000..9f304e59530f --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_basic_io.go @@ -0,0 +1,93 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestRun = &TestRunCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "basicIOTest", + JavaClassName: "alluxio.cli.TestRunner", + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console")}, + }, +} + +type TestRunCommand struct { + *env.BaseJavaCommand + directory string + operation string + readType string + workers string + writeType string +} + +func (c *TestRunCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestRunCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "basicIOTest", + Args: cobra.NoArgs, + Short: "Run all end-to-end tests or a specific test, on an Alluxio cluster.", + Example: `# Run all permutations of IO tests +$ ./bin/alluxio exec basicIOTest + +# Run a specific permutation of the IO tests +$ ./bin/alluxio exec basicIOtest --operation BASIC --readType NO_CACHE --writeType THROUGH`, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.directory, "directory", "/", + "Alluxio path for the tests working directory. Default: /") + cmd.Flags().StringVar(&c.operation, "operation", "", + "The operation to test, either BASIC or BASIC_NON_BYTE_BUFFER. \n"+ + "By default both operations are tested.") + cmd.Flags().StringVar(&c.readType, "readType", "", + "The read type to use, one of NO_CACHE, CACHE, CACHE_PROMOTE. \n"+ + "By default all readTypes are tested.") + cmd.Flags().StringVar(&c.workers, "workers", "", + "Alluxio worker addresses to run tests on. \n"+ + "If not specified, random ones will be used.") + cmd.Flags().StringVar(&c.writeType, "writeType", "", + "The write type to use, one of MUST_CACHE, CACHE_THROUGH, THROUGH. \n"+ + "By default all writeTypes are tested.") + return cmd +} + +func (c *TestRunCommand) Run(args []string) error { + var javaArgs []string + if c.directory != "" { + javaArgs = append(javaArgs, "--directory", c.directory) + } + if c.operation != "" { + javaArgs = append(javaArgs, "--operation", c.operation) + } + if c.readType != "" { + javaArgs = append(javaArgs, "--readType", c.readType) + } + if c.workers != "" { + javaArgs = append(javaArgs, "--workers", c.workers) + } + if c.writeType != "" { + javaArgs = append(javaArgs, "--writeType", c.writeType) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go b/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go new file mode 100644 index 000000000000..c2ed26a60faf --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go @@ -0,0 +1,74 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestHdfsMount = &TestHdfsMountCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "hdfsMountTest", + JavaClassName: "alluxio.cli.ValidateHdfsMount", + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console")}, + }, +} + +type TestHdfsMountCommand struct { + *env.BaseJavaCommand + path string + readonly bool + shared bool + option string +} + +func (c *TestHdfsMountCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestHdfsMountCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "hdfsMountTest", + Args: cobra.NoArgs, + Short: "Tests runs a set of validations against the given hdfs path.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const path = "path" + cmd.Flags().StringVar(&c.path, path, "", "specifies the HDFS path you want to validate.") + cmd.MarkFlagRequired(path) + cmd.Flags().BoolVar(&c.readonly, "readonly", false, "mount point is readonly in Alluxio.") + cmd.Flags().BoolVar(&c.shared, "shared", false, "mount point is shared.") + cmd.Flags().StringVar(&c.option, "option", "", "options associated with this mount point.") + return cmd +} + +func (c *TestHdfsMountCommand) Run(args []string) error { + var javaArgs []string + javaArgs = append(javaArgs, c.path) + if c.readonly { + javaArgs = append(javaArgs, "--readonly") + } + if c.shared { + javaArgs = append(javaArgs, "--shared") + } + if c.option != "" { + javaArgs = append(javaArgs, "--option", c.option) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go b/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go new file mode 100644 index 000000000000..bf14b18013fb --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go @@ -0,0 +1,61 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestUfs = &TestUfsCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "ufsTest", + JavaClassName: "alluxio.cli.UnderFileSystemContractTest", + }, +} + +type TestUfsCommand struct { + *env.BaseJavaCommand + path string + test []string +} + +func (c *TestUfsCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestUfsCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "ufsTest", + Args: cobra.NoArgs, + Short: "Test the integration between Alluxio and the given UFS to validate UFS semantics", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const path = "path" + cmd.Flags().StringVar(&c.path, path, "", "the full UFS path to run tests against.") + cmd.MarkFlagRequired(path) + cmd.Flags().StringSliceVar(&c.test, "test", nil, "Test name, this option can be passed multiple times to indicate multipleZ tests") + + return cmd +} + +func (c *TestUfsCommand) Run(args []string) error { + javaArgs := []string{"--path", c.path} + for _, singleTest := range c.test { + javaArgs = append(javaArgs, "--test", singleTest) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_ufs_io.go b/cli/src/alluxio.org/cli/cmd/exec/test_ufs_io.go new file mode 100644 index 000000000000..5314106fbc9a --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_ufs_io.go @@ -0,0 +1,117 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var TestUfsIO = &TestUfsIOCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "ufsIOTest", + JavaClassName: "alluxio.stress.cli.UfsIOBench", + }, +} + +type TestUfsIOCommand struct { + *env.BaseJavaCommand + path string + ioSize string + threads int + cluster bool + clusterLimit int + javaOpt []string +} + +func (c *TestUfsIOCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestUfsIOCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: c.CommandName, + Short: "A benchmarking tool for the I/O between Alluxio and UFS.", + Long: `A benchmarking tool for the I/O between Alluxio and UFS. +This test will measure the I/O throughput between Alluxio workers and the specified UFS path. +Each worker will create concurrent clients to first generate test files of the specified size then read those files. +The write/read I/O throughput will be measured in the process.`, + Example: `# This runs the I/O benchmark to HDFS in your process locally +$ ./bin/alluxio runUfsIOTest --path hdfs:// + +# This invokes the I/O benchmark to HDFS in the Alluxio cluster +# 1 worker will be used. 4 threads will be created, each writing then reading 4G of data +$ ./bin/alluxio runUfsIOTest --path hdfs:// --cluster --cluster-limit 1 + +# This invokes the I/O benchmark to HDFS in the Alluxio cluster +# 2 workers will be used +# 2 threads will be created on each worker +# Each thread is writing then reading 512m of data +$ ./bin/alluxio runUfsIOTest --path hdfs:// --cluster --cluster-limit 2 --io-size 512m --threads 2`, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.path, "path", "", + "specifies the path to write/read temporary data in.") + cmd.Flags().StringVar(&c.ioSize, "io-size", "4G", + "specifies the amount of data each thread writes/reads.") + cmd.Flags().IntVar(&c.threads, "threads", 4, + "specifies the number of threads to concurrently use on each worker.") + cmd.Flags().BoolVar(&c.cluster, "cluster", false, + "specifies the benchmark is run in the Alluxio cluster.\n"+ + "If not specified, this benchmark will run locally.") + cmd.Flags().IntVar(&c.clusterLimit, "cluster-limit", 0, + "specifies how many Alluxio workers to run the benchmark concurrently.\n"+ + "If >0, it will only run on that number of workers.\n"+ + "If 0, it will run on all available cluster workers.\n"+ + "If <0, will run on the workers from the end of the worker list.\n"+ + "This flag is only used if --cluster is enabled.") + cmd.Flags().StringSliceVar(&c.javaOpt, "java-opt", nil, + "The java options to add to the command line to for the task.\n"+ + "This can be repeated. The options must be quoted and prefixed with a space.\n"+ + "For example: --java-opt \" -Xmx4g\" --java-opt \" -Xms2g\".") + err := cmd.MarkFlagRequired("path") + if err != nil { + log.Logger.Errorln("Required flag --path not specified.") + } + return cmd +} + +func (c *TestUfsIOCommand) Run(args []string) error { + if c.threads <= 0 { + return stacktrace.NewError("Flag --threads should be a positive number.") + } + + var javaArgs []string + if c.path != "" { + javaArgs = append(javaArgs, "--path", c.path) + } + if c.ioSize != "" { + javaArgs = append(javaArgs, "--io-size", c.ioSize) + } + javaArgs = append(javaArgs, "--threads", strconv.Itoa(c.threads)) + if c.cluster != false { + javaArgs = append(javaArgs, "--cluster", "--cluster-limit", strconv.Itoa(c.clusterLimit)) + } + for _, option := range c.javaOpt { + javaArgs = append(javaArgs, "--java-opt", "\""+option+"\"") + } + javaArgs = append(javaArgs, args...) + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_worker.go b/cli/src/alluxio.org/cli/cmd/exec/test_worker.go new file mode 100644 index 000000000000..45b108e68e29 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_worker.go @@ -0,0 +1,53 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestRunCluster = &TestClusterCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "checkCluster", + JavaClassName: "alluxio.cli.CheckCluster", + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console")}, + }, +} + +type TestClusterCommand struct { + *env.BaseJavaCommand +} + +func (c *TestClusterCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestClusterCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "checkCluster", + Args: cobra.NoArgs, + Short: "Test whether the workers have already run successfully.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *TestClusterCommand) Run(args []string) error { + var javaArgs []string + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/cat.go b/cli/src/alluxio.org/cli/cmd/fs/cat.go new file mode 100644 index 000000000000..874fa6014037 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/cat.go @@ -0,0 +1,55 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Cat(className string) env.Command { + return &CatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cat", + JavaClassName: className, + Parameters: []string{"cat"}, + }, + } +} + +type CatCommand struct { + *env.BaseJavaCommand +} + +func (c *CatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "cat [path]", + Short: "Print specified file's content", + Long: `The cat command prints the contents of a file in Alluxio to the shell.`, + Example: `# Print the contents of /output/part-00000 +$ ./bin/alluxio fs cat /output/part-00000`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *CatCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/check_cached.go b/cli/src/alluxio.org/cli/cmd/fs/check_cached.go new file mode 100644 index 000000000000..9a3fff0f1912 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/check_cached.go @@ -0,0 +1,67 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "strconv" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func CheckCached(className string) env.Command { + return &CheckCachedCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "check-cached", + JavaClassName: className, + Parameters: []string{"check-cached"}, + }, + } +} + +type CheckCachedCommand struct { + *env.BaseJavaCommand + + sample int + limit int +} + +func (c *CheckCachedCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CheckCachedCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "check-cached [path]", + Short: "Checks if files under a path have been cached in alluxio.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().IntVar(&c.sample, "sample", 1, "Sample ratio, 10 means sample 1 in every 10 files.") + cmd.Flags().IntVar(&c.limit, "limit", 1000, "Limit number of files to check") + return cmd +} + +func (c *CheckCachedCommand) Run(args []string) error { + var javaArgs []string + if c.sample != 0 { + javaArgs = append(javaArgs, "--sample", strconv.Itoa(c.sample)) + } + if c.limit != 0 { + javaArgs = append(javaArgs, "--limit", strconv.Itoa(c.limit)) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/checksum.go b/cli/src/alluxio.org/cli/cmd/fs/checksum.go new file mode 100644 index 000000000000..5a228e06089c --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/checksum.go @@ -0,0 +1,61 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Checksum(className string) env.Command { + return &ChecksumCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "checksum", + JavaClassName: className, + Parameters: []string{"checksum"}, + }, + } +} + +type ChecksumCommand struct { + *env.BaseJavaCommand +} + +func (c *ChecksumCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ChecksumCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "checksum [path]", + Short: "Calculates the md5 checksum of a specified file", + Long: `The checksum command outputs the md5 value of a file in Alluxio. +This can be used to verify the contents of a file stored in Alluxio.`, + Example: `# Compare the checksum values +# value from Alluxio filesystem +$ ./bin/alluxio fs checksum /LICENSE +md5sum: bf0513403ff54711966f39b058e059a3 +# value from local filesystem +md5 LICENSE +MD5 (LICENSE) = bf0513403ff54711966f39b058e059a3`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *ChecksumCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/chgrp.go b/cli/src/alluxio.org/cli/cmd/fs/chgrp.go new file mode 100644 index 000000000000..4c33bc328401 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/chgrp.go @@ -0,0 +1,66 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Chgrp(className string) env.Command { + return &ChgrpCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "chgrp", + JavaClassName: className, + }, + } +} + +type ChgrpCommand struct { + *env.BaseJavaCommand + recursive bool +} + +func (c *ChgrpCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ChgrpCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%s [group] [path]", c.CommandName), + Short: "Changes the group of a file or directory", + Long: `The chgrp command changes the group of the file or directory in Alluxio. +Alluxio supports file authorization with POSIX file permissions. +The file owner or superuser can execute this command.`, + Example: `# Change the group of a file +$ ./bin/alluxio fs chgrp alluxio-group-new /input/file1`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVarP(&c.recursive, "recursive", "R", false, + "change the group recursively for all files and directories under the given path") + return cmd +} + +func (c *ChgrpCommand) Run(args []string) error { + var javaArgs []string + if c.recursive { + javaArgs = append(javaArgs, "-R") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/chmod.go b/cli/src/alluxio.org/cli/cmd/fs/chmod.go new file mode 100644 index 000000000000..357fb0a7bab2 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/chmod.go @@ -0,0 +1,66 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Chmod(className string) env.Command { + return &ChmodCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "chmod", + JavaClassName: className, + }, + } +} + +type ChmodCommand struct { + *env.BaseJavaCommand + recursive bool +} + +func (c *ChmodCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ChmodCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%s [mode] [path]", c.CommandName), + Short: "Changes the permission of a file or directory", + Long: `The chmod command changes the permission of a file or directory in Alluxio. +The permission mode is represented as an octal 3 digit value. +Refer to https://en.wikipedia.org/wiki/Chmod#Numerical_permissions for a detailed description of the modes.`, + Example: `# Set mode 755 for /input/file +$ ./bin/alluxio fs chmod 755 /input/file1`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVarP(&c.recursive, "recursive", "R", false, + "change the permission recursively for all files and directories under the given path") + return cmd +} + +func (c *ChmodCommand) Run(args []string) error { + var javaArgs []string + if c.recursive { + javaArgs = append(javaArgs, "-R") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/chown.go b/cli/src/alluxio.org/cli/cmd/fs/chown.go new file mode 100644 index 000000000000..700df7cd05d2 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/chown.go @@ -0,0 +1,65 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Chown(className string) env.Command { + return &ChownCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "chown", + JavaClassName: className, + }, + } +} + +type ChownCommand struct { + *env.BaseJavaCommand + recursive bool +} + +func (c *ChownCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ChownCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%s [:] ", c.CommandName), + Short: "Changes the owner of a file or directory specified by args", + Long: `The chown command changes the owner of a file or directory in Alluxio. +The ownership of a file can only be altered by a superuser`, + Example: `# Change the owner of /input/file1 to alluxio-user +$ ./bin/alluxio fs chown alluxio-user /input/file1`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVarP(&c.recursive, "recursive", "R", false, + "change the owner recursively for all files and directories under the given path") + return cmd +} + +func (c *ChownCommand) Run(args []string) error { + var javaArgs []string + if c.recursive { + javaArgs = append(javaArgs, "-R") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/consistent_hash.go b/cli/src/alluxio.org/cli/cmd/fs/consistent_hash.go new file mode 100644 index 000000000000..d94463058dde --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/consistent_hash.go @@ -0,0 +1,76 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func ConsistentHash(className string) env.Command { + return &ConsistentHashCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "consistent-hash", + JavaClassName: className, + Parameters: []string{"consistent-hash"}, + }, + } +} + +type ConsistentHashCommand struct { + *env.BaseJavaCommand + + create bool + compare bool + clean bool +} + +func (c *ConsistentHashCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ConsistentHashCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "consistent-hash [--create]|[--compare <1stCheckFilePath> <2ndCheckFilePath>]|[--clean]", + Short: "This command is for checking whether the consistent hash ring is changed or not", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const create, compare, clean = "create", "compare", "clean" + cmd.Flags().BoolVar(&c.create, create, false, "Generate check file") + cmd.Flags().BoolVar(&c.compare, compare, false, "Compare check files to see if the hash ring has changed") + cmd.Flags().BoolVar(&c.clean, clean, false, "Delete generated check data") + cmd.MarkFlagsMutuallyExclusive(create, compare, clean) + return cmd +} + +func (c *ConsistentHashCommand) Run(args []string) error { + javaArgs := []string{} + if c.create { + javaArgs = append(javaArgs, "--create") + } + if c.compare { + if len(args) != 2 { + return stacktrace.NewError("expect 2 arguments with --compare but got %v", len(args)) + } + javaArgs = append(javaArgs, "--compare") + } + if c.clean { + javaArgs = append(javaArgs, "--clean") + } + + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/cp.go b/cli/src/alluxio.org/cli/cmd/fs/cp.go new file mode 100644 index 000000000000..e4183854b4f4 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/cp.go @@ -0,0 +1,95 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Cp(className string) env.Command { + return &CopyCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cp", + JavaClassName: className, + }, + } +} + +type CopyCommand struct { + *env.BaseJavaCommand + + bufferSize string + isRecursive bool + preserve bool + threads int +} + +func (c *CopyCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CopyCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "cp [srcPath] [dstPath]", + Short: "Copy a file or directory", + Long: `Copies a file or directory in the Alluxio filesystem or between local and Alluxio filesystems. +The file:// scheme indicates a local filesystem path and the alluxio:// scheme or no scheme indicates an Alluxio filesystem path.`, + Example: `# Copy within the Alluxio filesystem +$ ./bin/alluxio fs cp /file1 /file2 + +# Copy a local file to the Alluxio filesystem +$ ./bin/alluxio fs cp file:///file1 /file2 + +# Copy a file in Alluxio to local +$ ./bin/alluxio fs cp alluxio:///file1 file:///file2 + +# Recursively copy a directory within the Alluxio filesystem +$ ./bin/alluxio fs cp -R /dir1 /dir2 +`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.bufferSize, "buffer-size", "", "Read buffer size when coping to or from local, with defaults of 64MB and 8MB respectively") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "True to copy the directory subtree to the destination directory") + cmd.Flags().BoolVarP(&c.preserve, "preserve", "p", false, "Preserve file permission attributes when copying files; all ownership, permissions, and ACLs will be preserved") + cmd.Flags().IntVar(&c.threads, "thread", 0, "Number of threads used to copy files in parallel, defaults to 2 * CPU cores") + return cmd +} + +func (c *CopyCommand) Run(args []string) error { + if c.threads < 0 { + return stacktrace.NewError("thread value must be positive but was %v", c.threads) + } + + javaArgs := []string{"cp"} + if c.bufferSize != "" { + javaArgs = append(javaArgs, "--buffersize", c.bufferSize) + } + if c.isRecursive { + javaArgs = append(javaArgs, "--recursive") + } + if c.preserve { + javaArgs = append(javaArgs, "--preserve") + } + if c.threads != 0 { + javaArgs = append(javaArgs, "--thread", strconv.Itoa(c.threads)) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/fs.go b/cli/src/alluxio.org/cli/cmd/fs/fs.go new file mode 100644 index 000000000000..94fea1aeaf32 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/fs.go @@ -0,0 +1,66 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "fs", + Description: "Operations to interface with the Alluxio filesystem", + Documentation: `Operations to interface with the Alluxio filesystem +For commands that take Alluxio URIs as an argument such as ls or mkdir, the argument should be either +- A complete Alluxio URI, such as alluxio://:/ +- A path without its scheme header, such as /path, in order to use the default hostname and port set in alluxio-site.properties + +> Note: All fs commands require the Alluxio cluster to be running. + +Most of the commands which require path components allow wildcard arguments for ease of use. +For example, the command "bin/alluxio fs rm '/data/2014*'" deletes anything in the data directory with a prefix of 2014. + +Some shells will attempt to glob the input paths, causing strange errors. +As a workaround, you can disable globbing (depending on the shell type; for example, set -f) or by escaping wildcards +For example, the command "bin/alluxio fs cat /\\*" uses the escape backslash character twice. +This is because the shell script will eventually call a java program which should have the final escaped parameters "cat /\\*". +`, + Commands: Cmds(names.FileSystemShellJavaClass), +} + +func Cmds(className string) []env.Command { + var ret []env.Command + for _, c := range []func(string) env.Command{ + Cat, + CheckCached, + Checksum, + Chgrp, + Chmod, + Chown, + ConsistentHash, + Cp, + Head, + Location, + Ls, + Mkdir, + Mv, + Rm, + Stat, + Tail, + Test, + Touch, + } { + ret = append(ret, c(className)) + } + + return ret +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/head.go b/cli/src/alluxio.org/cli/cmd/fs/head.go new file mode 100644 index 000000000000..0ed5458177ca --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/head.go @@ -0,0 +1,64 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Head(className string) env.Command { + return &HeadCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "head", + JavaClassName: className, + Parameters: []string{"head"}, + }, + } +} + +type HeadCommand struct { + *env.BaseJavaCommand + + bytes string +} + +func (c *HeadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *HeadCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "head [path]", + Short: "Print the leading bytes from the specified file", + Long: `The head command prints the first 1KB of data of a file to the shell. +Specifying the -c flag sets the number of bytes to print.`, + Example: `# Print first 2048 bytes of a file +$ ./bin/alluxio fs head -c 2048 /output/part-00000`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVarP(&c.bytes, "bytes", "c", "", "Byte size to print") + return cmd +} + +func (c *HeadCommand) Run(args []string) error { + var javaArgs []string + if c.bytes != "" { + javaArgs = append(javaArgs, "-c", c.bytes) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/location.go b/cli/src/alluxio.org/cli/cmd/fs/location.go new file mode 100644 index 000000000000..c0b7e55dc675 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/location.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Location(className string) env.Command { + return &LocationCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "location", + JavaClassName: className, + Parameters: []string{"location"}, + }, + } +} + +type LocationCommand struct { + *env.BaseJavaCommand +} + +func (c *LocationCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LocationCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "location [path]", + Short: "Displays the list of hosts storing the specified file.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *LocationCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/ls.go b/cli/src/alluxio.org/cli/cmd/fs/ls.go new file mode 100644 index 000000000000..562ce2873ad3 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/ls.go @@ -0,0 +1,150 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Ls(className string) env.Command { + return &LsCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "ls", + JavaClassName: className, + Parameters: []string{"ls"}, + }, + } +} + +type LsCommand struct { + *env.BaseJavaCommand + + listDirAsFile bool + forceLoadMetadata bool + isHumanReadable bool + omitMountInfo bool + pinnedFileOnly bool + isRecursive bool + isReverse bool + sortBy string + timestamp string +} + +func (c *LsCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +var ( + sortOptions = []string{ + "creationTime", + "inMemoryPercentage", + "lastAccessTime", + "lastModificationTime", + "name", + "path", + "size", + } + timestampOptions = []string{ + "createdTime", + "lastAccessTime", + "lastModifiedTime", + } +) + +func (c *LsCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "ls [path]", + Short: "Prints information for files and directories at the given path", + Long: `The ls command lists all the immediate children in a directory and displays the file size, last modification time, and in memory status of the files. +Using ls on a file will only display the information for that specific file. + +The ls command will also load the metadata for any file or immediate children of a directory from the under storage system to Alluxio namespace if it does not exist in Alluxio. +It queries the under storage system for any file or directory matching the given path and creates a mirror of the file in Alluxio backed by that file. +Only the metadata, such as the file name and size, are loaded this way and no data transfer occurs.`, + Example: `# List and load metadata for all immediate children of /s3/data +$ ./bin/alluxio fs ls /s3/data + +# Force loading metadata of /s3/data +$ ./bin/alluxio fs ls -f /s3/data`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + // special case to overwrite the -h shorthand for --help flag for --human-readable + cmd.PersistentFlags().BoolP("help", "", false, "help for this command") + + cmd.Flags().BoolVarP(&c.listDirAsFile, "list-dir-as-file", "d", false, "List directories as files") + cmd.Flags().BoolVarP(&c.forceLoadMetadata, "load-metadata", "f", false, "Force load metadata for immediate children in a directory") + cmd.Flags().BoolVarP(&c.isHumanReadable, "human-readable", "h", false, "Print sizes in human readable format") + cmd.Flags().BoolVarP(&c.omitMountInfo, "omit-mount-info", "m", false, "Omit mount point related information such as the UFS path") + cmd.Flags().BoolVarP(&c.pinnedFileOnly, "pinned-files", "p", false, "Only show pinned files") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "List subdirectories recursively") + cmd.Flags().BoolVarP(&c.isReverse, "reverse", "r", false, "Reverse sorted order") + cmd.Flags().StringVar(&c.sortBy, "sort", "", fmt.Sprintf("Sort entries by column, one of {%v}", strings.Join(sortOptions, "|"))) + cmd.Flags().StringVar(&c.timestamp, "timestamp", "", fmt.Sprintf("Display specified timestamp of entry, one of {%v}", strings.Join(timestampOptions, "|"))) + return cmd +} + +func (c *LsCommand) Run(args []string) error { + var javaArgs []string + if c.listDirAsFile { + javaArgs = append(javaArgs, "-d") + } + if c.forceLoadMetadata { + javaArgs = append(javaArgs, "-f") + } + if c.isHumanReadable { + javaArgs = append(javaArgs, "-h") + } + if c.omitMountInfo { + javaArgs = append(javaArgs, "-m") + } + if c.pinnedFileOnly { + javaArgs = append(javaArgs, "-p") + } + if c.isRecursive { + javaArgs = append(javaArgs, "-R") + } + if c.isReverse { + javaArgs = append(javaArgs, "-r") + } + if c.sortBy != "" { + if err := checkAllowed(c.sortBy, sortOptions...); err != nil { + return stacktrace.Propagate(err, "error validating sort options") + } + javaArgs = append(javaArgs, "--sort", c.sortBy) + } + if c.timestamp != "" { + if err := checkAllowed(c.timestamp, timestampOptions...); err != nil { + return stacktrace.Propagate(err, "error validating timestamp options") + } + javaArgs = append(javaArgs, "--timestamp", c.timestamp) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} + +func checkAllowed(val string, allowed ...string) error { + for _, i := range allowed { + if val == i { + return nil + } + } + return stacktrace.NewError("value %v must be one of %v", val, strings.Join(allowed, ",")) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/mkdir.go b/cli/src/alluxio.org/cli/cmd/fs/mkdir.go new file mode 100644 index 000000000000..79c08f42a58e --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/mkdir.go @@ -0,0 +1,60 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Mkdir(className string) env.Command { + return &MkdirCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "mkdir", + JavaClassName: className, + Parameters: []string{"mkdir"}, + }, + } +} + +type MkdirCommand struct { + *env.BaseJavaCommand +} + +func (c *MkdirCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *MkdirCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "mkdir [path1 path2 ...]", + Short: "Create directories at the specified paths, creating the parent directory if not exists", + Long: `The mkdir command creates a new directory in the Alluxio filesystem. +It is recursive and will create any parent directories that do not exist. +Note that the created directory will not be created in the under storage system until a file in the directory is persisted to the underlying storage. +Using mkdir on an invalid or existing path will fail.`, + Example: `# Creating a folder structure +$ ./bin/alluxio fs mkdir /users +$ ./bin/alluxio fs mkdir /users/Alice +$ ./bin/alluxio fs mkdir /users/Bob`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *MkdirCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/mv.go b/cli/src/alluxio.org/cli/cmd/fs/mv.go new file mode 100644 index 000000000000..ee188ef01ede --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/mv.go @@ -0,0 +1,58 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Mv(className string) env.Command { + return &MoveCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "mv", + JavaClassName: className, + Parameters: []string{"mv"}, + }, + } +} + +type MoveCommand struct { + *env.BaseJavaCommand +} + +func (c *MoveCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *MoveCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "mv [srcPath] [dstPath]", + Short: "Rename a file or directory", + Long: `The mv command moves a file or directory to another path in Alluxio. +The destination path must not exist or be a directory. +If it is a directory, the file or directory will be placed as a child of the directory. +The command is purely a metadata operation and does not affect the data blocks of the file.`, + Example: `# Moving a file +$ ./bin/alluxio fs mv /data/2014 /data/archives/2014`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *MoveCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/rm.go b/cli/src/alluxio.org/cli/cmd/fs/rm.go new file mode 100644 index 000000000000..07da9b51be9d --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/rm.go @@ -0,0 +1,79 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Rm(className string) env.Command { + return &RmCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "rm", + JavaClassName: className, + Parameters: []string{"rm"}, + }, + } +} + +type RmCommand struct { + *env.BaseJavaCommand + + alluxioOnly bool + deleteMount bool + isRecursive bool + skipUfsCheck bool +} + +func (c *RmCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *RmCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "rm [path]", + Short: "Remove the specified file", + Long: `The rm command removes a file from Alluxio space and the under storage system. +The file will be unavailable immediately after this command returns, but the actual data may be deleted a while later.`, + Example: `# Remove a file from Alluxio and the under storage system +$ ./bin/alluxio fs rm /tmp/unused-file + +# Remove a file from Alluxio filesystem only +$ ./bin/alluxio fs rm --alluxio-only --skip-ufs-check /tmp/unused-file2 +# Note it is recommended to use both --alluxio-only and --skip-ufs-check together in this situation`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVar(&c.alluxioOnly, "alluxio-only", false, "True to only remove data and metadata from Alluxio cache") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "True to recursively remove files within the specified directory subtree") + cmd.Flags().BoolVarP(&c.skipUfsCheck, "skip-ufs-check", "U", false, "True to skip checking if corresponding UFS contents are in sync") + return cmd +} + +func (c *RmCommand) Run(args []string) error { + var javaArgs []string + if c.alluxioOnly { + javaArgs = append(javaArgs, "--alluxioOnly") + } + if c.isRecursive { + javaArgs = append(javaArgs, "-R") + } + if c.skipUfsCheck { + javaArgs = append(javaArgs, "-U") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/stat.go b/cli/src/alluxio.org/cli/cmd/fs/stat.go new file mode 100644 index 000000000000..21474e72342d --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/stat.go @@ -0,0 +1,96 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Stat(className string) env.Command { + return &StatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "stat", + JavaClassName: className, + Parameters: []string{"stat"}, + }, + } +} + +type StatCommand struct { + *env.BaseJavaCommand + + Path string + FileId string + Format string +} + +func (c *StatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *StatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "stat", + Short: "Displays info for the specified file or directory", + Long: `The stat command dumps the FileInfo representation of a file or a directory to the shell.`, + Example: `# Display file's stat +$ ./bin/alluxio fs stat /data/2015/logs-1.txt + +# Display directory's stat +$ ./bin/alluxio fs stat /data/2015 + +# Display the size of file +$ ./bin/alluxio fs stat -f %z /data/2015/logs-1.txt + +# Find the file by fileID and display the stat, useful in troubleshooting +$ ./bin/alluxio fs stat -fileId 12345678`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + const path, fileId = "path", "file-id" + cmd.Flags().StringVar(&c.Path, path, "", "Path to file or directory") + cmd.Flags().StringVar(&c.FileId, fileId, "", "File id of file") + cmd.Flags().StringVarP(&c.Format, "format", "f", "", `Display info in the given format: + "%N": name of the file + "%z": size of file in bytes + "%u": owner + "%g": group name of owner + "%i": file id of the file + "%y": modification time in UTC in 'yyyy-MM-dd HH:mm:ss' format + "%Y": modification time as Unix timestamp in milliseconds + "%b": Number of blocks allocated for file +`) + cmd.MarkFlagsMutuallyExclusive(path, fileId) + return cmd +} + +func (c *StatCommand) Run(_ []string) error { + if c.Path == "" && c.FileId == "" { + return stacktrace.NewError("must set one of --path or --file-id flags") + } + + var javaArgs []string + if c.Format != "" { + javaArgs = append(javaArgs, "-f", c.Format) + } + if c.Path != "" { + javaArgs = append(javaArgs, c.Path) + } else if c.FileId != "" { + javaArgs = append(javaArgs, "--file-id", c.FileId) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/tail.go b/cli/src/alluxio.org/cli/cmd/fs/tail.go new file mode 100644 index 000000000000..aa258881b231 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/tail.go @@ -0,0 +1,64 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Tail(className string) env.Command { + return &TailCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "tail", + JavaClassName: className, + Parameters: []string{"tail"}, + }, + } +} + +type TailCommand struct { + *env.BaseJavaCommand + + bytes string +} + +func (c *TailCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TailCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "tail [path]", + Short: "Print the trailing bytes from the specified file", + Long: `The tail command prints the last 1KB of data of a file to the shell. +Specifying the -c flag sets the number of bytes to print.`, + Example: `# Print last 2048 bytes of a file +$ ./bin/alluxio fs tail -c 2048 /output/part-00000`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.bytes, "bytes", "", "Byte size to print") + return cmd +} + +func (c *TailCommand) Run(args []string) error { + var javaArgs []string + if c.bytes != "" { + javaArgs = append(javaArgs, "-c", c.bytes) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/test.go b/cli/src/alluxio.org/cli/cmd/fs/test.go new file mode 100644 index 000000000000..6c9146ae7456 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/test.go @@ -0,0 +1,80 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Test(className string) env.Command { + return &TestCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "test", + JavaClassName: className, + Parameters: []string{"test"}, + }, + } +} + +type TestCommand struct { + *env.BaseJavaCommand + + isDirectory bool + isExists bool + isFile bool + isNotEmpty bool + isZeroLength bool +} + +func (c *TestCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "test [path]", + Short: "Test a property of a path, returning 0 if the property is true, or 1 otherwise", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVarP(&c.isDirectory, "dir", "d", false, "Test if path is a directory") + cmd.Flags().BoolVarP(&c.isExists, "exists", "e", false, "Test if path exists") + cmd.Flags().BoolVarP(&c.isFile, "file", "f", false, "Test if path is a file") + cmd.Flags().BoolVarP(&c.isFile, "not-empty", "s", false, "Test if path is not empty") + cmd.Flags().BoolVarP(&c.isFile, "zero", "z", false, "Test if path is zero length") + return cmd +} + +func (c *TestCommand) Run(args []string) error { + var javaArgs []string + if c.isDirectory { + javaArgs = append(javaArgs, "-d") + } + if c.isExists { + javaArgs = append(javaArgs, "-e") + } + if c.isFile { + javaArgs = append(javaArgs, "-f") + } + if c.isNotEmpty { + javaArgs = append(javaArgs, "-s") + } + if c.isZeroLength { + javaArgs = append(javaArgs, "-z") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/touch.go b/cli/src/alluxio.org/cli/cmd/fs/touch.go new file mode 100644 index 000000000000..c827cc57c3ca --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/touch.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Touch(className string) env.Command { + return &TouchCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "touch", + JavaClassName: className, + Parameters: []string{"touch"}, + }, + } +} + +type TouchCommand struct { + *env.BaseJavaCommand +} + +func (c *TouchCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TouchCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "touch [path]", + Short: "Create a 0 byte file at the specified path, which will also be created in the under file system", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *TouchCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/generate/doc_tables.go b/cli/src/alluxio.org/cli/cmd/generate/doc_tables.go new file mode 100644 index 000000000000..2abf73f6ba86 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/generate/doc_tables.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package generate + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var DocTables = &DocTablesCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "doc-tables", + JavaClassName: "alluxio.cli.DocGenerator", + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console")}, + }, +} + +type DocTablesCommand struct { + *env.BaseJavaCommand +} + +func (c *DocTablesCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *DocTablesCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: DocTables.CommandName, + Short: "Generate configuration and metric tables used in documentation", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *DocTablesCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/generate/generate.go b/cli/src/alluxio.org/cli/cmd/generate/generate.go new file mode 100644 index 000000000000..84d2a1231bd9 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/generate/generate.go @@ -0,0 +1,49 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package generate + +import ( + "alluxio.org/cli/env" + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" +) + +var Service = &env.Service{ + Name: "generate", + Description: "Generate files used in documentation", + Commands: []env.Command{ + &DocsCommand{}, + DocTables, + UserCliDoc, + }, +} + +type DocsCommand struct{} + +func (c *DocsCommand) ToCommand() *cobra.Command { + return &cobra.Command{ + Use: "docs", + Short: "Generate all documentation files", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + for _, c := range []env.Command{ + UserCliDoc, + DocTables, + } { + if err := c.ToCommand().RunE(cmd, args); err != nil { + return stacktrace.Propagate(err, "error running %v", c.ToCommand().Use) + } + } + return nil + }, + } +} diff --git a/cli/src/alluxio.org/cli/cmd/generate/user_cli.go b/cli/src/alluxio.org/cli/cmd/generate/user_cli.go new file mode 100644 index 000000000000..cb702046e901 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/generate/user_cli.go @@ -0,0 +1,177 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package generate + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "alluxio.org/cli/env" +) + +var UserCliDoc = &UserCliCommand{ + Dst: filepath.Join("docs", "en", "operation", "User-CLI.md"), +} + +type UserCliCommand struct { + Dst string +} + +func (c *UserCliCommand) ToCommand() *cobra.Command { + return &cobra.Command{ + Use: "user-cli", + Short: "Generate content for `operation/User-CLI.md`", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + var rootCmd *cobra.Command + for rootCmd = cmd; rootCmd.HasParent(); rootCmd = rootCmd.Parent() { + } + + f, err := os.Create(c.Dst) + if err != nil { + return stacktrace.Propagate(err, "error creating output file") + } + defer f.Close() + w := bufio.NewWriter(f) + fmt.Fprintln(w, + `--- +layout: global +title: User Command Line Interface +--- + +{% comment %} +This is a generated file created by running command "bin/alluxio generate user-cli" +The command parses the golang command definitions and descriptions to generate the markdown in this file +{% endcomment %} + +Alluxio's command line interface provides user access to various operations, such as: +- Start or stop processes +- Filesystem operations +- Administrative commands`) + fmt.Fprintln(w) + fmt.Fprintln(w, "Invoke the executable to view the possible subcommands:") + fmt.Fprintln(w, "```shell") + fmt.Fprintln(w, "$ ./bin/alluxio") + fmt.Fprintln(w, rootCmd.UsageString()) + fmt.Fprintln(w, "```") + fmt.Fprintln(w) + fmt.Fprintln(w, "To set JVM system properties as part of the command, set the `-D` flag in the form of `-Dproperty=value`.") + fmt.Fprintln(w) + fmt.Fprintln(w, "To attach debugging java options specified by `$ALLUXIO_USER_ATTACH_OPTS`, set the `--attach-debug` flag") + fmt.Fprintln(w) + fmt.Fprintln(w, "Note that, as a part of Alluxio deployment, the Alluxio shell will also take the configuration in `${ALLUXIO_HOME}/conf/alluxio-site.properties` when it is run from Alluxio installation at `${ALLUXIO_HOME}`.") + fmt.Fprintln(w) + + for _, serviceCmd := range rootCmd.Commands() { + if serviceCmd.Name() == "help" { + // help is a built in command from the library. avoid documenting it + continue + } + fmt.Fprint(w, "## ") + fmt.Fprintln(w, serviceCmd.Name()) + + desc := serviceCmd.Short + if serviceCmd.Long != "" { + desc = serviceCmd.Long + } + fmt.Fprintln(w, desc) + fmt.Fprintln(w) + + for _, opCmd := range serviceCmd.Commands() { + printCommandDocs(serviceCmd.Name(), opCmd, w) + } + } + w.Flush() + return nil + }, + } +} + +func printCommandDocs(serviceName string, opCmd *cobra.Command, w io.Writer) { + fmt.Fprintln(w, "###", serviceName, opCmd.Name()) + + // collect relevant flags defined for the command + inheritedFlags := opCmd.InheritedFlags() + definedFlags := pflag.NewFlagSet(fmt.Sprintf("%v_%v", serviceName, opCmd.Name()), pflag.ContinueOnError) + opCmd.Flags().VisitAll(func(f *pflag.Flag) { + if f.Hidden { + return + } + if f.Name == env.AttachDebugName || f.Name == env.JavaOptsName { + return + } + if inheritedFlags.Lookup(f.Name) == nil { + definedFlags.AddFlag(f) + } + }) + if definedFlags.HasFlags() { + fmt.Fprintf(w, "Usage: `%v`\n\n", opCmd.UseLine()) + } else { + // remove the [flags] part of the usage as there are no flags to mention + fmt.Fprintf(w, "Usage: `%v`\n\n", strings.Replace(opCmd.UseLine(), " [flags]", "", 1)) + } + + desc := opCmd.Short + if opCmd.Long != "" { + desc = opCmd.Long + } + fmt.Fprintln(w, desc) + fmt.Fprintln(w) + + if definedFlags.HasFlags() { + fmt.Fprintln(w, "Flags:") + definedFlags.VisitAll(func(f *pflag.Flag) { + fmt.Fprintf(w, "- `--%v`", f.Name) + if f.Shorthand != "" { + fmt.Fprintf(w, ",`-%v`", f.Shorthand) + } + _, required := f.Annotations[cobra.BashCompOneRequiredFlag] + _, usage := pflag.UnquoteUsage(f) + + // prepend the flag description with "(Required)" if required + // print default value if flag is not required + var requiredPrefix, defVal string + if required { + requiredPrefix = "(Required) " + } else { + v := f.DefValue + if f.Value.Type() == "string" { + // add quotes for string flags + v = fmt.Sprintf("%q", defVal) + } + defVal = fmt.Sprintf(" (Default: %v)", v) + } + fmt.Fprintf(w, ": %v%v%v\n", requiredPrefix, usage, defVal) + + }) + fmt.Fprintln(w) + } + + if opCmd.HasExample() { + fmt.Fprintln(w, "Examples:") + for _, ex := range strings.Split(opCmd.Example, "\n\n") { + fmt.Fprintln(w, "```shell") + fmt.Fprintln(w, ex) + fmt.Fprintln(w, "```") + fmt.Fprintln(w) + } + fmt.Fprintln(w) + } +} diff --git a/cli/src/alluxio.org/cli/cmd/info/cache.go b/cli/src/alluxio.org/cli/cmd/info/cache.go new file mode 100644 index 000000000000..9ebd5923a445 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/cache.go @@ -0,0 +1,71 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "strings" + + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Cache = &CacheCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cache", + JavaClassName: names.FileSystemAdminShellJavaClass, + Parameters: []string{"report", "capacity"}, + }, +} + +type CacheCommand struct { + *env.BaseJavaCommand + + liveWorkers bool + lostWorkers bool + workersList []string // list of worker hostnames or ip addresses +} + +func (c *CacheCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CacheCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: c.CommandName, + Short: "Reports worker capacity information", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + const live, lost, worker = "live", "lost", "worker" + cmd.Flags().BoolVar(&c.liveWorkers, live, false, "Only show live workers for capacity report") + cmd.Flags().BoolVar(&c.lostWorkers, lost, false, "Only show lost workers for capacity report") + cmd.Flags().StringSliceVar(&c.workersList, worker, nil, "Only show specified workers for capacity report, labeled by hostname or IP address") + cmd.MarkFlagsMutuallyExclusive(live, lost, worker) + return cmd +} + +func (c *CacheCommand) Run(_ []string) error { + // TODO: output all in a serializable format and filter/trim as specified by flags + var args []string + if c.liveWorkers { + args = append(args, "-live") + } else if c.lostWorkers { + args = append(args, "-lost") + } else if len(c.workersList) > 0 { + args = append(args, "-workers", strings.Join(c.workersList, ",")) + } + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/collect.go b/cli/src/alluxio.org/cli/cmd/info/collect.go new file mode 100644 index 000000000000..a646519594d5 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/collect.go @@ -0,0 +1,154 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Collect = &CollectCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "collect", + JavaClassName: "alluxio.cli.bundler.CollectInfo", + }, +} + +type CollectCommand struct { + *env.BaseJavaCommand + + AdditionalCommands map[string]string + additionalLogs []string + endTime string + excludeLogs []string + excludeWorkerMetrics bool + includeLogs []string + local bool + maxThreads int + outputDir string + startTime string +} + +func (c *CollectCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +const dateFormat = "2006-01-02T15:04:05" + +func (c *CollectCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [command]", c.CommandName), + Short: "Collects information such as logs, config, metrics, and more from the running Alluxio cluster and bundle into a single tarball", + Long: `Collects information such as logs, config, metrics, and more from the running Alluxio cluster and bundle into a single tarball + +[command] must be one of the following values: +- all: runs all the commands below +- cluster: runs a set of Alluxio commands to collect information about the Alluxio cluster +- conf: collects the configuration files under ${ALLUXIO_HOME}/config/ +- env: runs a set of linux commands to collect information about the cluster +- jvm: collects jstack from the JVMs +- log: collects the log files under ${ALLUXIO_HOME}/logs/ +- metrics: collects Alluxio system metrics + +> WARNING: This command MAY bundle credentials. Inspect the output tarball for any sensitive information and remove it before sharing with others.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringSliceVar(&c.additionalLogs, "additional-logs", nil, "Additional file name prefixes from ${ALLUXIO_HOME}/logs to include in the tarball, inclusive of the default log files") + cmd.Flags().StringVar(&c.endTime, "end-time", "", "Logs that do not contain entries before this time will be ignored, format must be like "+dateFormat) + cmd.Flags().StringSliceVar(&c.excludeLogs, "exclude-logs", nil, "File name prefixes from ${ALLUXIO_HOME}/logs to exclude; this is evaluated after adding files from --additional-logs") + cmd.Flags().BoolVar(&c.excludeWorkerMetrics, "exclude-worker-metrics", false, "True to skip worker metrics collection") + cmd.Flags().StringSliceVar(&c.includeLogs, "include-logs", nil, "File name prefixes from ${ALLUXIO_HOME}/logs to include in the tarball, ignoring the default log files; cannot be used with --exclude-logs or --additional-logs") + cmd.Flags().BoolVar(&c.local, "local", false, "True to only collect information from the local machine") + cmd.Flags().IntVar(&c.maxThreads, "max-threads", 1, "Parallelism of the command; use a smaller value to limit network I/O when transferring tarballs") + const outputDir = "output-dir" + cmd.Flags().StringVar(&c.outputDir, outputDir, "", "Output directory to write collect info tarball to") + cmd.MarkFlagRequired(outputDir) + cmd.Flags().StringVar(&c.startTime, "start-time", "", "Logs that do not contain entries after this time will be ignored, format must be like "+dateFormat) + return cmd +} + +func (c *CollectCommand) Run(args []string) error { + // TODO: use flags instead of arguments to parse user input + commands := map[string]string{ + "all": "all", + "cluster": "collectAlluxioInfo", + "conf": "collectConfig", + "env": "collectEnv", + "jvm": "collectJvmInfo", + "log": "collectLog", + "metrics": "collectMetrics", + } + for k, v := range c.AdditionalCommands { + commands[k] = v + } + commandArg, ok := commands[args[0]] + if !ok { + var cmds []string + for c := range commands { + cmds = append(cmds, c) + } + return stacktrace.NewError("first argument must be one of %v", strings.Join(cmds, ", ")) + } + + var javaArgs []string + if c.additionalLogs != nil { + if c.includeLogs != nil { + return stacktrace.NewError("cannot set both --include-logs and --additional-logs") + } + javaArgs = append(javaArgs, "--additional-logs", strings.Join(c.additionalLogs, ",")) + } + if c.endTime != "" { + if _, err := time.Parse(dateFormat, c.endTime); err != nil { + return stacktrace.Propagate(err, "could not parse end time %v", c.endTime) + } + javaArgs = append(javaArgs, "--end-time", c.endTime) + } + if c.excludeLogs != nil { + if c.includeLogs != nil { + return stacktrace.NewError("cannot set both --include-logs and --exclude-logs") + } + javaArgs = append(javaArgs, "--exclude-logs", strings.Join(c.excludeLogs, ",")) + } + if c.excludeWorkerMetrics { + javaArgs = append(javaArgs, "--exclude-worker-metrics") + } + if c.includeLogs != nil { + // already checked exclusivity with --additional-logs and --exclude-logs + javaArgs = append(javaArgs, "--include-logs", strings.Join(c.includeLogs, ",")) + } + if c.local { + javaArgs = append(javaArgs, "--local") + } + if c.maxThreads > 1 { + javaArgs = append(javaArgs, "--max-threads", strconv.Itoa(c.maxThreads)) + } + if c.startTime != "" { + if _, err := time.Parse(dateFormat, c.startTime); err != nil { + return stacktrace.Propagate(err, "could not parse start time %v", c.startTime) + } + javaArgs = append(javaArgs, "--start-time", c.startTime) + } + + javaArgs = append(javaArgs, "--output-dir", c.outputDir, commandArg) + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/doctor.go b/cli/src/alluxio.org/cli/cmd/info/doctor.go new file mode 100644 index 000000000000..3d16a92a2eb7 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/doctor.go @@ -0,0 +1,60 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "fmt" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Doctor = &DoctorCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "doctor", + JavaClassName: names.FileSystemAdminShellJavaClass, + Parameters: []string{"doctor"}, + }, +} + +type DoctorCommand struct { + *env.BaseJavaCommand +} + +func (c *DoctorCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *DoctorCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [type]", Doctor.CommandName), + Short: "Runs doctor configuration or storage command", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *DoctorCommand) Run(args []string) error { + var javaArgs []string + if args[0] == "configuration" || args[0] == "storage" { + javaArgs = append(javaArgs, args[0]) + } else if args[0] != "all" { + return stacktrace.NewError("Invalid doctor type. Valid types: [all, configuration, storage]") + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/info.go b/cli/src/alluxio.org/cli/cmd/info/info.go new file mode 100644 index 000000000000..80a09cd216fd --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/info.go @@ -0,0 +1,29 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "info", + Description: "Retrieve and/or display info about the running Alluxio cluster", + Commands: []env.Command{ + Cache, + Collect, + Doctor, + Nodes, + Report, + Version, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/info/nodes.go b/cli/src/alluxio.org/cli/cmd/info/nodes.go new file mode 100644 index 000000000000..aebf1c06e404 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/nodes.go @@ -0,0 +1,51 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Nodes = &NodesCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "nodes", + JavaClassName: names.FileSystemAdminShellJavaClass, + Parameters: []string{"nodes", "status"}, + }, +} + +type NodesCommand struct { + *env.BaseJavaCommand +} + +func (c *NodesCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *NodesCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: c.CommandName, + Short: "Show all registered workers' status", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *NodesCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/report.go b/cli/src/alluxio.org/cli/cmd/info/report.go new file mode 100644 index 000000000000..89c0631384a0 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/report.go @@ -0,0 +1,97 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "bytes" + "fmt" + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "io" + "os" + "strings" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/cmd/util" + "alluxio.org/cli/env" +) + +var Report = &ReportCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "report", + JavaClassName: names.FileSystemAdminShellJavaClass, + Parameters: []string{"report"}, + }, +} + +type ReportCommand struct { + *env.BaseJavaCommand + raw bool +} + +func (c *ReportCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ReportCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [arg]", c.CommandName), + Short: "Reports Alluxio running cluster information", + Long: `Reports Alluxio running cluster information +[arg] can be one of the following values: + jobservice: job service metrics information + metrics: metrics information + summary: cluster summary + ufs: under storage system information + +Defaults to summary if no arg is provided +`, + Args: cobra.RangeArgs(0, 1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVar(&c.raw, "raw", false, + "Output raw JSON data instead of human-readable format for bytes, datetime, and duration.") + return cmd +} + +func (c *ReportCommand) Run(args []string) error { + reportArg := "summary" + if len(args) == 1 { + options := map[string]struct{}{ + "jobservice": {}, + "metrics": {}, + "summary": {}, + "ufs": {}, + } + if _, ok := options[args[0]]; !ok { + var cmds []string + for c := range options { + cmds = append(cmds, c) + } + return stacktrace.NewError("first argument must be one of %v", strings.Join(cmds, ", ")) + } + reportArg = args[0] + } + + buf := &bytes.Buffer{} + if err := c.RunWithIO([]string{reportArg}, nil, buf, os.Stderr); err != nil { + io.Copy(os.Stdout, buf) + return err + } + + if err := util.PrintProcessedJsonBytes(buf.Bytes(), c.raw); err != nil { + return stacktrace.Propagate(err, "error formatting output to print") + } + return nil +} diff --git a/cli/src/alluxio.org/cli/cmd/info/version.go b/cli/src/alluxio.org/cli/cmd/info/version.go new file mode 100644 index 000000000000..76a3cd304640 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/version.go @@ -0,0 +1,49 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Version = &VersionCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "version", + JavaClassName: "alluxio.cli.Version", + }, +} + +type VersionCommand struct { + *env.BaseJavaCommand +} + +func (c *VersionCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *VersionCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: c.CommandName, + Short: "Print Alluxio version.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *VersionCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/initiate/clear_os_cache.go b/cli/src/alluxio.org/cli/cmd/initiate/clear_os_cache.go new file mode 100644 index 000000000000..06808f47be65 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/initiate/clear_os_cache.go @@ -0,0 +1,43 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package initiate + +import ( + "io/ioutil" + "os/exec" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" +) + +var ClearOSCache = &ClearOSCacheCommand{} + +type ClearOSCacheCommand struct{} + +func (c *ClearOSCacheCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "clear-os-cache", + Args: cobra.NoArgs, + Short: "Clear OS buffer cache of the machine", + Long: `The clear-os-cache command drops the OS buffer cache`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := exec.Command("sync").Run(); err != nil { + return stacktrace.Propagate(err, "error running sync") + } + if err := ioutil.WriteFile("/proc/sys/vm/drop_caches", []byte("3"), 0644); err != nil { + return stacktrace.Propagate(err, "error running drop_caches") + } + return nil + }, + } + return cmd +} diff --git a/cli/src/alluxio.org/cli/cmd/initiate/copy_dir.go b/cli/src/alluxio.org/cli/cmd/initiate/copy_dir.go new file mode 100644 index 000000000000..c7c1ba57df57 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/initiate/copy_dir.go @@ -0,0 +1,94 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package initiate + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/processes" + "alluxio.org/log" +) + +var CopyDir = &CopyDirCommand{} + +type CopyDirCommand struct{} + +func (c *CopyDirCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "copy-dir [path]", + Args: cobra.ExactArgs(1), + Short: "Copy a path to all master and worker nodes.", + Long: `The copy-dir command copies the directory at given path to all master nodes listed in conf/masters and all worker nodes listed in conf/workers. + +> Note: This command does not require the Alluxio cluster to be running.`, + Example: `# copy alluxio-site properties file to all nodes +$ ./bin/alluxio init copy-dir conf/alluxio-site.properties`, + RunE: func(cmd *cobra.Command, args []string) error { + // get list of masters or workers, or both + hosts, err := processes.GetHostnames([]string{processes.HostGroupMasters, processes.HostGroupWorkers}) + if err != nil { + return stacktrace.Propagate(err, "cannot read host names") + } + + // get name of current host, avoid rsync itself + myHost, err := os.Hostname() + if err != nil { + return stacktrace.Propagate(err, "cannot read local host name") + } + + // determine the absolute directory to copy + absolutePath, err := filepath.Abs(args[0]) + if err != nil { + return stacktrace.Propagate(err, "cannot find absolute path") + } + + var errs []error + for _, host := range hosts { + if host == myHost { + continue + } + log.Logger.Infof("RSYNCing directory %s to %s", args[0], host) + + // src path needs to end with a slash, but dst path needs to end without the slash + dstDir := strings.TrimRight(absolutePath, "/") + srcDir := dstDir + "/" + + cmd := exec.Command("rsync", "-az", srcDir, + fmt.Sprintf("%s:%s", host, dstDir)) + log.Logger.Infof("Command: %s", cmd) + if err := cmd.Run(); err != nil { + log.Logger.Error("Error: %s", err) + errs = append(errs, err) + } + } + + if len(errs) > 0 { + var errMsgs []string + for _, err := range errs { + errMsgs = append(errMsgs, err.Error()) + } + return stacktrace.NewError("At least one error encountered:\n%v", strings.Join(errMsgs, "\n")) + } + + log.Logger.Infof("Copying path %s successful on nodes: %s", args[0], hosts) + return nil + }, + } + return cmd +} diff --git a/cli/src/alluxio.org/cli/cmd/initiate/format.go b/cli/src/alluxio.org/cli/cmd/initiate/format.go new file mode 100644 index 000000000000..ed6b7f0e4e49 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/initiate/format.go @@ -0,0 +1,96 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package initiate + +import ( + "os/exec" + "path/filepath" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" + "alluxio.org/cli/processes" + "alluxio.org/log" +) + +var Format = &FormatCommand{} + +type FormatCommand struct { + localFileSystem bool + skipMaster bool + skipWorker bool +} + +func (c *FormatCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "format", + Args: cobra.NoArgs, + Short: "Format Alluxio master and all workers", + Long: `The format command formats the Alluxio master and all its workers. + +Running this command on an existing Alluxio cluster deletes everything persisted in Alluxio, including cached data and any metadata information. +Data in under storage will not be changed. + +> Warning: Formatting is required when you run Alluxio for the first time. +It should only be called while the cluster is not running. +`, + RunE: func(cmd *cobra.Command, args []string) error { + if c.localFileSystem { + // check if alluxio.master.mount.table.root.ufs set + if env.Env.EnvVar.GetString(env.ConfAlluxioMasterMountTableRootUfs.EnvVar) != "" { + return nil + } + } + + cliPath := filepath.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), names.BinAlluxio) + + if !c.skipWorker { + // run cache format on workers + workerArgs := []string{"cache", "format"} + if err := processes.RunSshCommand( + strings.Join(append([]string{cliPath}, workerArgs...), " "), + processes.HostGroupWorkers); err != nil { + return stacktrace.Propagate(err, "error formatting workers") + } + } + + if !c.skipMaster { + // run journal format on masters + journalArgs := []string{"journal", "format"} + if env.Env.EnvVar.GetString(env.ConfAlluxioMasterJournalType.EnvVar) == "EMBEDDED" { + if err := processes.RunSshCommand( + strings.Join(append([]string{cliPath}, journalArgs...), " "), + processes.HostGroupMasters); err != nil { + return stacktrace.Propagate(err, "error formatting masters") + } + } else { + if err := exec.Command(cliPath, journalArgs...).Run(); err != nil { + return stacktrace.Propagate(err, "error formatting master") + } + } + } + log.Logger.Infof("Format successful on master and workers.") + + return nil + }, + } + cmd.Flags().BoolVarP(&c.localFileSystem, "localFileSystem", "s", false, + "Only format if underfs is local and doesn't already exist") + const skipMaster, skipWorker = "skip-master", "skip-worker" + cmd.Flags().BoolVar(&c.skipMaster, skipMaster, false, "Skip formatting journal on all masters") + cmd.Flags().BoolVar(&c.skipWorker, skipWorker, false, "Skip formatting cache on all workers") + cmd.MarkFlagsMutuallyExclusive(skipMaster, skipWorker) + return cmd +} diff --git a/cli/src/alluxio.org/cli/cmd/initiate/initiate.go b/cli/src/alluxio.org/cli/cmd/initiate/initiate.go new file mode 100644 index 000000000000..f2e474d27533 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/initiate/initiate.go @@ -0,0 +1,28 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package initiate + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "init", + Description: "Initialization operations such as format and validate", + Commands: []env.Command{ + //ClearMetrics, + ClearOSCache, + CopyDir, + Format, + Validate, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/initiate/validate.go b/cli/src/alluxio.org/cli/cmd/initiate/validate.go new file mode 100644 index 000000000000..c4ebad271c5a --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/initiate/validate.go @@ -0,0 +1,66 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package initiate + +import ( + "fmt" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Validate = &ValidateCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "validate", + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console")}, + }, +} + +type ValidateCommand struct { + *env.BaseJavaCommand + validateType string +} + +func (c *ValidateCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ValidateCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: c.CommandName, + Short: "Validate Alluxio configuration or environment", + Example: `# Validate configuration +$ ./bin/alluxio init validate --type conf + +# Validate environment +$ ./bin/alluxio init validate --type env`, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.validateType, "type", "", + "Decide the type to validate. Valid inputs: [conf, env]") + return cmd +} + +func (c *ValidateCommand) Run(args []string) error { + if c.validateType == "conf" { + c.JavaClassName = "alluxio.cli.ValidateConf" + } else if c.validateType == "env" { + c.JavaClassName = "alluxio.cli.ValidateEnv" + } else { + return stacktrace.NewError("Invalid validate type. Valid inputs: [conf, env]") + } + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/job.go b/cli/src/alluxio.org/cli/cmd/job/job.go new file mode 100644 index 000000000000..3b168f6118bd --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/job.go @@ -0,0 +1,76 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "job", + Description: "Command line tool for interacting with the job service.", + Commands: []env.Command{ + Load, + }, +} + +const ( + progress = "progress" + stop = "stop" + submit = "submit" +) + +var operations = []string{ + progress, + stop, + submit, +} + +type BaseJobCommand struct { + *env.BaseJavaCommand + + isProgress bool + isStop bool + isSubmit bool + + progressFormat string + progressVerbose bool +} + +func (c *BaseJobCommand) AttachOperationFlags(cmd *cobra.Command) { + cmd.Flags().BoolVar(&c.isProgress, progress, false, "View progress of submitted job") + cmd.Flags().BoolVar(&c.isStop, stop, false, "Stop running job") + cmd.Flags().BoolVar(&c.isSubmit, submit, false, "Submit job") + cmd.MarkFlagsMutuallyExclusive(operations...) + + cmd.Flags().StringVar(&c.progressFormat, "format", "TEXT", "[progress] Format of output, either TEXT or JSON") + cmd.Flags().BoolVar(&c.progressVerbose, "verbose", false, "[progress] Verbose output") +} + +func (c *BaseJobCommand) OperationWithArgs() ([]string, error) { + // rely on MarkFlagsMutuallyExclusive to ensure there is at most one boolean switched to true + if c.isProgress { + ret := []string{"--" + progress, "--format", c.progressFormat} + if c.progressVerbose { + ret = append(ret, "--verbose") + } + return ret, nil + } else if c.isStop { + return []string{"--" + stop}, nil + } else if c.isSubmit { + return []string{"--" + submit}, nil + } + return nil, stacktrace.NewError("Did not specify an operation flag: %v", operations) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/load.go b/cli/src/alluxio.org/cli/cmd/job/load.go new file mode 100644 index 000000000000..dd4ef7d92f2b --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/load.go @@ -0,0 +1,113 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Load = &LoadCommand{ + BaseJobCommand: &BaseJobCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "load", + JavaClassName: names.FileSystemShellJavaClass, + }, + }, +} + +type LoadCommand struct { + *BaseJobCommand + path string + + bandwidth string + verify bool + partialListing bool + metadataOnly bool + skipIfExists bool + fileFilterRegx string +} + +func (c *LoadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LoadCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Load.CommandName, + Short: "Submit or manage load jobs", + Long: `The load command moves data from the under storage system into Alluxio storage. +For example, load can be used to prefetch data for analytics jobs. +If load is run on a directory, files in the directory will be recursively loaded.`, + Example: `# Submit a load job +$ ./bin/alluxio job load --path /path --submit + +# View the progress of a submitted job +$ ./bin/alluxio job load --path /path --progress +# Example output +Progress for loading path '/path': + Settings: bandwidth: unlimited verify: false + Job State: SUCCEEDED + Files Processed: 1000 + Bytes Loaded: 125.00MB + Throughput: 2509.80KB/s + Block load failure rate: 0.00% + Files Failed: 0 + +# Stop a submitted job +$ ./bin/alluxio job load --path /path --stop`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const path = "path" + cmd.Flags().StringVar(&c.path, path, "", "[all] Source path of load operation") + cmd.MarkFlagRequired(path) + c.AttachOperationFlags(cmd) + + cmd.Flags().StringVar(&c.bandwidth, "bandwidth", "", "[submit] Single worker read bandwidth limit") + cmd.Flags().BoolVar(&c.verify, "verify", false, "[submit] Run verification when load finishes and load new files if any") + cmd.Flags().BoolVar(&c.partialListing, "partial-listing", false, "[submit] Use partial directory listing, initializing load before reading the entire directory but cannot report on certain progress details") + cmd.Flags().BoolVar(&c.metadataOnly, "metadata-only", false, "[submit] Only load file metadata") + cmd.Flags().BoolVar(&c.skipIfExists, "skip-if-exists", false, "[submit] Skip existing fullly cached files") + cmd.Flags().StringVar(&c.fileFilterRegx, "file-filter-regx", "", "[submit] Skip files that match the regx pattern") + return cmd +} + +func (c *LoadCommand) Run(_ []string) error { + opWithArgs, err := c.OperationWithArgs() + if err != nil { + return stacktrace.Propagate(err, "error parsing operation") + } + javaArgs := []string{"load", c.path} + javaArgs = append(javaArgs, opWithArgs...) + if c.bandwidth != "" { + javaArgs = append(javaArgs, "--bandwidth", c.bandwidth) + } + if c.partialListing { + javaArgs = append(javaArgs, "--partial-listing") + } + if c.metadataOnly { + javaArgs = append(javaArgs, "--metadata-only") + } + if c.skipIfExists { + javaArgs = append(javaArgs, "--skip-if-exists") + } + if c.fileFilterRegx != "" { + javaArgs = append(javaArgs, "--file-filter-regx", c.fileFilterRegx) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/checkpoint.go b/cli/src/alluxio.org/cli/cmd/journal/checkpoint.go new file mode 100644 index 000000000000..9bed574c6e8c --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/checkpoint.go @@ -0,0 +1,54 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "alluxio.org/cli/cmd/names" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Checkpoint = &CheckpointCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "checkpoint", + JavaClassName: names.FileSystemAdminShellJavaClass, + Parameters: []string{"journal", "checkpoint"}, + }, +} + +type CheckpointCommand struct { + *env.BaseJavaCommand +} + +func (c *CheckpointCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CheckpointCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Checkpoint.CommandName, + Short: "Create a checkpoint in the leading Alluxio master journal", + Long: `The checkpoint command creates a checkpoint the leading Alluxio master's journal. +This command is mainly used for debugging and to avoid master journal logs from growing unbounded. +Checkpointing requires a pause in master metadata changes, so use this command sparingly to avoid interfering with other users of the system.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *CheckpointCommand) Run(_ []string) error { + return c.Base().Run(nil) +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/format.go b/cli/src/alluxio.org/cli/cmd/journal/format.go new file mode 100644 index 000000000000..c783fe71127b --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/format.go @@ -0,0 +1,72 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "bytes" + "fmt" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Format = &FormatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "format", + JavaClassName: "alluxio.cli.Format", + Parameters: []string{"master"}, + UseServerClasspath: true, + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console")}, + }, +} + +type FormatCommand struct { + *env.BaseJavaCommand +} + +func (c *FormatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *FormatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Format.CommandName, + Short: "Format the local Alluxio master journal", + Long: `The format command formats the local Alluxio master's journal. + +> Warning: Formatting should only be called while the cluster is not running.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *FormatCommand) Run(_ []string) error { + return c.Base().Run(nil) +} + +func (c *FormatCommand) Format() error { + cmd := c.RunJavaClassCmd(nil) + errBuf := &bytes.Buffer{} + cmd.Stderr = errBuf + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + return stacktrace.Propagate(err, "error running %v\nstderr: %v", c.CommandName, errBuf.String()) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/journal.go b/cli/src/alluxio.org/cli/cmd/journal/journal.go new file mode 100644 index 000000000000..8299bfd2289f --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/journal.go @@ -0,0 +1,26 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "journal", + Description: "Journal related operations", + Commands: []env.Command{ + Checkpoint, + Format, + Read, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/read.go b/cli/src/alluxio.org/cli/cmd/journal/read.go new file mode 100644 index 000000000000..2087cbf21fd1 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/read.go @@ -0,0 +1,96 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "fmt" + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Read = &ReadCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "read", + JavaClassName: "alluxio.master.journal.tool.JournalTool", + UseServerClasspath: true, + ShellJavaOpts: []string{fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console")}, + }, +} + +type ReadCommand struct { + *env.BaseJavaCommand + + end int + inputDir string + master string + outputDir string + start int +} + +func (c *ReadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ReadCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Read.CommandName, + Short: "Read an Alluxio journal file to a human-readable version", + Long: `The read command parses the current journal and outputs a human readable version to the local folder. +This command may take a while depending on the size of the journal. +> Note: This command requies that the Alluxio cluster is NOT running.`, + Example: `$ ./bin/alluxio readJournal +# output +Dumping journal of type EMBEDDED to /Users/alluxio/journal_dump-1602698211916 +2020-10-14 10:56:51,960 INFO RaftStorageDirectory - Lock on /Users/alluxio/alluxio/journal/raft/02511d47-d67c-49a3-9011-abb3109a44c1/in_use.lock acquired by nodename 78602@alluxio-user +2020-10-14 10:56:52,254 INFO RaftJournalDumper - Read 223 entries from log /Users/alluxio/alluxio/journal/raft/02511d47-d67c-49a3-9011-abb3109a44c1/current/log_0-222.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + cmd.Flags().IntVar(&c.end, "end", -1, "end log sequence number (exclusive)") + cmd.Flags().StringVar(&c.inputDir, "input-dir", "", "input directory on-disk to read the journal content from") + cmd.Flags().StringVar(&c.master, "master", "FileSystemMaster", "name of the master class") + cmd.Flags().StringVar(&c.outputDir, "output-dir", "", "output directory to write journal content to") + cmd.Flags().IntVar(&c.start, "start", 0, "start log sequence number (inclusive)") + return cmd +} + +func (c *ReadCommand) Run(_ []string) error { + var javaArgs []string + if c.start < 0 { + return stacktrace.NewError("start log sequence number must be non-negative but was %v", c.start) + } + if c.end < -1 { + return stacktrace.NewError("end log sequence number must be non-negative but was %v", c.end) + } + if c.end != -1 { + javaArgs = append(javaArgs, "-end", strconv.Itoa(c.end)) + } + if c.inputDir != "" { + javaArgs = append(javaArgs, "-inputDir", c.inputDir) + } + if c.master != "FileSystemMaster" { + javaArgs = append(javaArgs, "-master", c.master) + } + if c.outputDir != "" { + javaArgs = append(javaArgs, "-outputDir", c.outputDir) + } + if c.start > 0 { + javaArgs = append(javaArgs, "-start", strconv.Itoa(c.start)) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/names/names.go b/cli/src/alluxio.org/cli/cmd/names/names.go new file mode 100644 index 000000000000..4fbd59de6d38 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/names/names.go @@ -0,0 +1,18 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package names + +const ( + BinAlluxio = "bin/alluxio" + FileSystemAdminShellJavaClass = "alluxio.cli.fsadmin.FileSystemAdminShell" + FileSystemShellJavaClass = "alluxio.cli.fs.FileSystemShell" +) diff --git a/cli/src/alluxio.org/cli/cmd/process/process.go b/cli/src/alluxio.org/cli/cmd/process/process.go new file mode 100644 index 000000000000..b4ec935da8d1 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/process/process.go @@ -0,0 +1,28 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package process + +import ( + "alluxio.org/cli/env" +) + +var ( + Service = &env.Service{ + Name: "process", + Description: "Start/stop cluster processes or remove workers", + Commands: []env.Command{ + &env.StartProcessCommand{}, + &env.StopProcessCommand{}, + RemoveWorker, + }, + } +) diff --git a/cli/src/alluxio.org/cli/cmd/process/remove_worker.go b/cli/src/alluxio.org/cli/cmd/process/remove_worker.go new file mode 100644 index 000000000000..daf1dce578ff --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/process/remove_worker.go @@ -0,0 +1,57 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package process + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var RemoveWorker = &RemoveWorkerCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "remove-worker", + JavaClassName: names.FileSystemAdminShellJavaClass, + Parameters: []string{"nodes", "remove"}, + }, +} + +type RemoveWorkerCommand struct { + *env.BaseJavaCommand + workerId string +} + +func (c *RemoveWorkerCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *RemoveWorkerCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "remove-worker", + Short: "Remove a worker from ETCD membership", + Long: `Remove given worker from the cluster, so that clients and other workers will not consider the removed worker for services. +The worker must have been stopped before it can be safely removed from the cluster.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + } + const name = "name" + cmd.Flags().StringVarP(&c.workerId, name, "n", "", "Worker id") + cmd.MarkFlagRequired(name) + return cmd +} + +func (c *RemoveWorkerCommand) Run(_ []string) error { + return c.Base().Run([]string{"-n", c.workerId}) +} diff --git a/cli/src/alluxio.org/cli/cmd/util/format.go b/cli/src/alluxio.org/cli/cmd/util/format.go new file mode 100644 index 000000000000..ace83f125ed2 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/util/format.go @@ -0,0 +1,112 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package util + +import ( + "encoding/json" + "fmt" + "math" + "os" + "strconv" + "strings" + "time" + + "github.com/iancoleman/orderedmap" + "github.com/palantir/stacktrace" +) + +func PrintProcessedJsonBytes(jsonInput []byte, raw bool) error { + obj := orderedmap.New() + if err := obj.UnmarshalJSON(jsonInput); err != nil { + return stacktrace.Propagate(err, "error unmarshalling json from java command") + } + + if !raw { + newObj := orderedmap.New() + for _, key := range obj.Keys() { + if val, ok := obj.Get(key); ok { + k, v := processKeyValuePair(key, val) + newObj.Set(k, v) + } + } + obj = newObj + } + + prettyJson, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return stacktrace.Propagate(err, "error marshalling json to pretty format") + } + os.Stdout.Write(append(prettyJson, '\n')) + return nil +} + +func processKeyValuePair(key string, data interface{}) (string, interface{}) { + switch value := data.(type) { + case float64: + if strings.HasSuffix(key, "Time") { + return strings.TrimSuffix(key, "Time"), convertMsToDatetime(value) + } else if strings.HasSuffix(key, "Duration") { + return strings.TrimSuffix(key, "Duration"), convertMsToDuration(value) + } else if strings.HasSuffix(key, "Bytes") { + return strings.TrimSuffix(key, "Bytes"), convertBytesToString(value) + } else { + return key, value + } + case []interface{}: + array := make([]interface{}, len(value)) + for i, item := range value { + _, processedItem := processKeyValuePair(key, item) + array[i] = processedItem + } + return key, array + case orderedmap.OrderedMap: + processedMap := orderedmap.New() + for _, key := range value.Keys() { + if val, ok := value.Get(key); ok { + k, v := processKeyValuePair(key, val) + processedMap.Set(k, v) + } + } + return key, processedMap + default: + return key, value + } +} + +func convertMsToDatetime(timeMs float64) string { + tm := time.Unix(int64(timeMs)/1000, 0) + return tm.Format(time.RFC3339) +} + +func convertMsToDuration(timeMs float64) string { + duration := time.Duration(int64(timeMs)) * time.Millisecond + days := duration / (24 * time.Hour) + duration = duration % (24 * time.Hour) + hours := duration / time.Hour + duration = duration % time.Hour + minutes := duration / time.Minute + seconds := duration % time.Minute / time.Second + return fmt.Sprintf("%dd %02dh%02dm%02ds", days, hours, minutes, seconds) +} + +func convertBytesToString(bytes float64) string { + const unit = 1024 + suffixes := []string{"B", "KB", "MB", "GB", "TB", "PB"} + exp, n := 0, int64(bytes) + for n > 5*unit && exp < len(suffixes)-1 { + n /= unit + exp++ + } + value := bytes / math.Pow(unit, float64(exp)) + size := strconv.FormatFloat(value, 'f', 2, 64) + return size + suffixes[exp] +} diff --git a/cli/src/alluxio.org/cli/env/command.go b/cli/src/alluxio.org/cli/env/command.go new file mode 100644 index 000000000000..c8dab30c9498 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/command.go @@ -0,0 +1,156 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + "alluxio.org/log" +) + +type Command interface { + ToCommand() *cobra.Command +} + +type BaseJavaCommand struct { + CommandName string + JavaClassName string + Parameters []string + + DebugMode bool + + UseServerClasspath bool // defaults to ALLUXIO_CLIENT_CLASSPATH, use ALLUXIO_SERVER_CLASSPATH if true + InlineJavaOpts []string // java opts provided by the user as part of the inline command + ShellJavaOpts []string // default java opts encoded as part of the specific command +} + +const ( + AttachDebugName = "attach-debug" + JavaOptsName = "java-opts" +) + +func (c *BaseJavaCommand) InitRunJavaClassCmd(cmd *cobra.Command) *cobra.Command { + cmd.Flags().BoolVar(&c.DebugMode, AttachDebugName, false, fmt.Sprintf("True to attach debug opts specified by $%v", ConfAlluxioUserAttachOpts.EnvVar)) + cmd.Flags().StringSliceVarP(&c.InlineJavaOpts, JavaOptsName, "D", nil, `Alluxio properties to apply, ex. -Dkey=value`) + return cmd +} + +// RunJavaClassCmd constructs a java command with a predetermined order of variable opts +// ${JAVA} ${ALLUXIO_USER_ATTACH_OPTS} -cp ${ALLUXIO_CLIENT_CLASSPATH} ${ALLUXIO_USER_JAVA_OPTS} \ +// {command default opts} {user inline opts} {command java class} {command parameter} {user inline args} +// where: +// - ${ALLUXIO_USER_*} are environment variables set by the user in alluxio-env.sh +// - {command *} are encoded as part of the command's definition +// - {user inline *} are specified by the user when entering the command +func (c *BaseJavaCommand) RunJavaClassCmd(args []string) *exec.Cmd { + var cmdArgs []string + if c.DebugMode { + if opts := Env.EnvVar.GetString(ConfAlluxioUserAttachOpts.EnvVar); opts != "" { + cmdArgs = append(cmdArgs, strings.Split(opts, " ")...) + } + } + classpath := EnvAlluxioClientClasspath + if c.UseServerClasspath { + classpath = EnvAlluxioServerClasspath + } + cmdArgs = append(cmdArgs, "-cp", Env.EnvVar.GetString(classpath)) + + if opts := Env.EnvVar.GetString(ConfAlluxioUserJavaOpts.EnvVar); opts != "" { + cmdArgs = append(cmdArgs, strings.Split(opts, " ")...) + } + if len(c.ShellJavaOpts) > 0 { + cmdArgs = append(cmdArgs, c.ShellJavaOpts...) + } + for _, o := range c.InlineJavaOpts { + if opts := strings.TrimSpace(o); opts != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("-D%v", opts)) + } + } + cmdArgs = append(cmdArgs, c.JavaClassName) + if len(c.Parameters) > 0 { + cmdArgs = append(cmdArgs, c.Parameters...) + } + cmdArgs = append(cmdArgs, args...) + + ret := exec.Command(Env.EnvVar.GetString(ConfJava.EnvVar), cmdArgs...) + ret.Env = os.Environ() + for _, k := range Env.EnvVar.AllKeys() { + ret.Env = append(ret.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + return ret +} + +func (c *BaseJavaCommand) Run(args []string) error { + return c.RunWithIO(args, nil, os.Stdout, os.Stderr) +} + +func (c *BaseJavaCommand) RunWithIO(args []string, stdin io.Reader, stdout, stderr io.Writer) error { + cmd := c.RunJavaClassCmd(args) + + cmd.Stdin = stdin + cmd.Stdout = stdout + cmd.Stderr = stderr + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + return stacktrace.Propagate(err, "error running %v", c.CommandName) + } + return nil +} + +func (c *BaseJavaCommand) RunAndFormat(format string, stdin io.Reader, args []string) error { + switch strings.ToLower(format) { + case "json": + buf := &bytes.Buffer{} + if err := c.RunWithIO(args, stdin, buf, os.Stderr); err != nil { + io.Copy(os.Stdout, buf) + return err + } + var obj json.RawMessage + if err := json.Unmarshal(buf.Bytes(), &obj); err != nil { + return stacktrace.Propagate(err, "error unmarshalling json from java command") + } + prettyJson, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return stacktrace.Propagate(err, "error marshalling json to pretty format") + } + os.Stdout.Write(append(prettyJson, '\n')) + case "yaml": + buf := &bytes.Buffer{} + if err := c.RunWithIO(args, stdin, buf, os.Stderr); err != nil { + io.Copy(os.Stdout, buf) + return err + } + var obj interface{} + if err := json.Unmarshal(buf.Bytes(), &obj); err != nil { + return stacktrace.Propagate(err, "error unmarshalling json from java command") + } + if out, err := yaml.Marshal(obj); err == nil { + os.Stdout.Write(out) + } else { + return stacktrace.Propagate(err, "error marshalling yaml") + } + default: + return stacktrace.NewError("unfamiliar output format %s", format) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/env/env.go b/cli/src/alluxio.org/cli/env/env.go new file mode 100644 index 000000000000..b42a04ff54a4 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/env.go @@ -0,0 +1,298 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/palantir/stacktrace" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + + "alluxio.org/log" +) + +const ( + userLoggerType = "USER_LOGGER" +) + +// Env is a global instance of the Alluxio environment +// It is assumed that InitAlluxioEnv() is run before access +// The package level variable exists to allow access by cobra.Command.RunE functions +var Env *AlluxioEnv + +type AlluxioEnv struct { + RootPath string + Version string + EnvVar *viper.Viper +} + +func InitAlluxioEnv(rootPath string, jarEnvVars map[string]string, appendClasspathJars map[string]func(*viper.Viper, string) string) error { + /* + Note the precedence order of viper, where earlier is retrieved first: + - Set, env, config, default (https://pkg.go.dev/github.com/dvln/viper#section-readme) + Since env > config, explicitly set any values read from config to take precedence over env + - Use single instance of viper to set values and retrieve values within go logic + - Use local viper instances to read configuration files to avoid confusion in precedence order + This effectively negates the possibility of retrieving config values to simplify the precedence order to be + - Set, env, default + */ + envVar := viper.New() + envVar.AutomaticEnv() // override defaults with existing environment variable values + + // read VERSION from version.sh + var ver string + { + v := viper.New() + v.AddConfigPath(filepath.Join(rootPath, "libexec")) + v.SetConfigName("version.sh") + v.SetConfigType("env") + if err := v.ReadInConfig(); err != nil { + return stacktrace.Propagate(err, "error reading version config") + } + ver = v.GetString(envVersion) + envVar.Set(envVersion, ver) + } + + setEnvDefaults(envVar, ver, rootPath, jarEnvVars) + + // set user-specified environment variable values from alluxio-env.sh + { + // use ALLUXIO_CONF_DIR if set externally, otherwise default to above value + confPath := envVar.GetString(ConfAlluxioConfDir.EnvVar) + const alluxioEnv = "alluxio-env.sh" + log.Logger.Debugf("Loading %v configuration from directory %v", alluxioEnv, confPath) + v := viper.New() + if _, err := os.Stat(filepath.Join(confPath, alluxioEnv)); err == nil { + v.AddConfigPath(confPath) + v.SetConfigName(alluxioEnv) + v.SetConfigType("env") + v.AllowEmptyEnv(true) + if err := v.ReadInConfig(); err != nil { + return stacktrace.Propagate(err, "error reading alluxio-env config") + } + // for each entry found in alluxio-env.sh, explicitly set to global viper + for _, k := range v.AllKeys() { + key, val := strings.ToUpper(k), v.Get(k) + log.Logger.Debugf("Setting user provided %v=%v", key, val) + envVar.Set(key, val) + } + } + } + + setClasspathVariables(envVar) + + for envVarName, envF := range appendClasspathJars { + envVar.Set(envVarName, envF(envVar, ver)) + } + + // check java executable and version + if err := checkAndSetJava(envVar); err != nil { + return stacktrace.Propagate(err, "error finding installed java") + } + if err := checkJavaVersion(envVar.GetString(ConfJava.EnvVar)); err != nil { + return stacktrace.Propagate(err, "error checking java version compatibility") + } + + setJavaOpts(envVar) + + if log.Logger.IsLevelEnabled(logrus.DebugLevel) { + keys := envVar.AllKeys() + sort.Strings(keys) + log.Logger.Debugln("Environment variables:") + for _, k := range keys { + log.Logger.Debugf("%v %v", strings.ToUpper(k), envVar.Get(k)) + } + log.Logger.Debugln() + } + + Env = &AlluxioEnv{ + RootPath: rootPath, + Version: ver, + EnvVar: envVar, + } + return nil +} + +func checkAndSetJava(envVar *viper.Viper) error { + if envVar.Get(ConfJava.EnvVar) != nil { + return nil + } + // check JAVA_HOME + if javaHome := envVar.Get(confJavaHome.EnvVar); javaHome != nil { + javaHomeStr, ok := javaHome.(string) + if !ok { + return stacktrace.NewError("error casting %v to string", javaHome) + } + javaHomeBinJava := filepath.Join(javaHomeStr, "bin", "java") + // check if JAVA_HOME exists with valid $JAVA_HOME/bin/java binary + if _, err := os.Stat(javaHomeBinJava); err == nil { + envVar.Set(ConfJava.EnvVar, javaHomeBinJava) + return nil + } + } + // check if java is available via `PATH` using `which` + whichJavaPath, err := exec.Command("which", "java").Output() + if err == nil { + envVar.Set(ConfJava.EnvVar, strings.TrimSpace(string(whichJavaPath))) + return nil + } + // cannot find java + // - ${JAVA} is not set + // - ${JAVA_HOME}/bin/java is not a valid path + // - java is not found as part of ${PATH} + return stacktrace.NewError(`Error: Cannot find 'java' on path or under $JAVA_HOME/bin/. Please set %v in alluxio-env.sh or user bash profile.`, confJavaHome.EnvVar) +} + +var ( + // matches the first 2 numbers in the version string encapsulated by double quotes, ex. "11.0.19" -> 11.0 + javaVersionRe = regexp.MustCompile(`.*"(?P\d+\.\d+)[\w.-]*".*`) + // must be either java 8 or 11 + requiredVersionRegex = regexp.MustCompile(`1\.8|11\.0`) +) + +func checkJavaVersion(javaPath string) error { + cmd := exec.Command(javaPath, "-version") + javaVer, err := cmd.CombinedOutput() + if err != nil { + return stacktrace.Propagate(err, "error finding java version from `%v -version`", javaPath) + } + matches := javaVersionRe.FindStringSubmatch(string(javaVer)) + if len(matches) != 2 { + return stacktrace.NewError("java version output does not match expected regex pattern %v\n%v", javaVersionRe.String(), string(javaVer)) + } + if !requiredVersionRegex.MatchString(matches[1]) { + return stacktrace.NewError("Error: Alluxio requires Java 1.8 or 11.0, currently Java %v found.", matches[1]) + } + return nil +} + +func InitEnvForTesting(alluxioEnv map[string]string) { + envVar := viper.New() + ver := "300-testing" + envVar.Set(envVersion, ver) + + rootPath := "/opt/alluxio" + setEnvDefaults(envVar, ver, rootPath, map[string]string{ + EnvAlluxioAssemblyClientJar: filepath.Join("assembly", "client", "target", "alluxio-assembly-client-%v-jar-with-dependencies.jar"), + EnvAlluxioAssemblyServerJar: filepath.Join("assembly", "server", "target", "alluxio-assembly-server-%v-jar-with-dependencies.jar"), + }) + + // set test specified env + for k, v := range alluxioEnv { + envVar.Set(k, v) + } + + setClasspathVariables(envVar) + + // set dummy java + envVar.Set(ConfJava.EnvVar, "/usr/lib/java") + + setJavaOpts(envVar) + + Env = &AlluxioEnv{ + RootPath: rootPath, + Version: ver, + EnvVar: envVar, + } +} + +func setEnvDefaults(envVar *viper.Viper, ver, rootPath string, jarEnvVars map[string]string) { + // set default values + for k, v := range map[string]string{ + ConfAlluxioHome.EnvVar: rootPath, + ConfAlluxioConfDir.EnvVar: filepath.Join(rootPath, "conf"), + ConfAlluxioLogsDir.EnvVar: filepath.Join(rootPath, "logs"), + confAlluxioUserLogsDir.EnvVar: filepath.Join(rootPath, "logs", "user"), + } { + envVar.SetDefault(k, v) + } + // set jar env vars + for envVarName, jarPathFormat := range jarEnvVars { + envVar.SetDefault(envVarName, filepath.Join(rootPath, fmt.Sprintf(jarPathFormat, ver))) + } +} + +func setClasspathVariables(envVar *viper.Viper) { + // set classpath variables which are dependent on user configurable values + envVar.Set(EnvAlluxioClientClasspath, strings.Join([]string{ + envVar.GetString(ConfAlluxioConfDir.EnvVar) + "/", + envVar.GetString(confAlluxioClasspath.EnvVar), + envVar.GetString(EnvAlluxioAssemblyClientJar), + }, ":")) + envVar.Set(EnvAlluxioServerClasspath, strings.Join([]string{ + envVar.GetString(ConfAlluxioConfDir.EnvVar) + "/", + envVar.GetString(confAlluxioClasspath.EnvVar), + envVar.GetString(EnvAlluxioAssemblyServerJar), + }, ":")) +} + +func setJavaOpts(envVar *viper.Viper) { + // append default opts to ALLUXIO_JAVA_OPTS + alluxioJavaOpts := ConfAlluxioJavaOpts.JavaOptsToArgs(envVar) + if opts := strings.Join(alluxioJavaOpts, " "); opts != "" { + // warn about setting configuration through java opts that should be set through environment variables instead + for _, c := range []*AlluxioConfigEnvVar{ + ConfAlluxioConfDir, + ConfAlluxioLogsDir, + confAlluxioUserLogsDir, + } { + if strings.Contains(opts, c.configKey) { + log.Logger.Warnf("Setting %v through %v will be ignored. Use environment variable %v instead.", c.configKey, ConfAlluxioJavaOpts.EnvVar, c.EnvVar) + } + } + } + + for _, c := range []*AlluxioConfigEnvVar{ + ConfAlluxioHome, + ConfAlluxioConfDir, + ConfAlluxioLogsDir, + confAlluxioUserLogsDir, + } { + alluxioJavaOpts = append(alluxioJavaOpts, c.ConfigToJavaOpts(envVar, true)...) // mandatory java opts + } + + for _, c := range []*AlluxioConfigEnvVar{ + confAlluxioRamFolder, + confAlluxioMasterHostname, + ConfAlluxioMasterMountTableRootUfs, + ConfAlluxioMasterJournalType, + confAlluxioWorkerRamdiskSize, + } { + alluxioJavaOpts = append(alluxioJavaOpts, c.ConfigToJavaOpts(envVar, false)...) // optional user provided java opts + } + alluxioJavaOpts = append(alluxioJavaOpts, + fmt.Sprintf(JavaOptFormat, "log4j.configuration", "file:"+filepath.Join(envVar.GetString(ConfAlluxioConfDir.EnvVar), "log4j.properties")), + fmt.Sprintf(JavaOptFormat, "org.apache.jasper.compiler.disablejsr199", true), + fmt.Sprintf(JavaOptFormat, "java.net.preferIPv4Stack", true), + fmt.Sprintf(JavaOptFormat, "org.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads", false), + ) + + envVar.Set(ConfAlluxioJavaOpts.EnvVar, strings.Join(alluxioJavaOpts, " ")) + + for _, p := range ProcessRegistry { + p.SetEnvVars(envVar) + } + + // also set user environment variables, as they are not associated with a particular process + // ALLUXIO_USER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} {user provided opts} + userJavaOpts := []string{fmt.Sprintf(JavaOptFormat, ConfAlluxioLoggerType, userLoggerType)} + userJavaOpts = append(userJavaOpts, ConfAlluxioJavaOpts.JavaOptsToArgs(envVar)...) + userJavaOpts = append(userJavaOpts, ConfAlluxioUserJavaOpts.JavaOptsToArgs(envVar)...) + envVar.Set(ConfAlluxioUserJavaOpts.EnvVar, strings.Join(userJavaOpts, " ")) +} diff --git a/cli/src/alluxio.org/cli/env/env_vars.go b/cli/src/alluxio.org/cli/env/env_vars.go new file mode 100644 index 000000000000..9908d645a473 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/env_vars.go @@ -0,0 +1,150 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "strings" + + "github.com/spf13/viper" +) + +const ( + JavaOptFormat = "-D%v=%v" +) + +// alluxioEnvVarsTemplate lists the environment variables to include in conf/alluxio-env.sh.template +var alluxioEnvVarsTemplate = map[string]*AlluxioConfigEnvVar{} + +type AlluxioConfigEnvVar struct { + EnvVar string // environment variable read by startup script + description string // describes the environment variable in conf/alluxio-env.sh.template file + + // optional + IsJavaOpts bool // true if environment variable's value is a java opts string + configKey string // corresponding property key as defined in java + additionalAlluxioJavaOpts map[string]string // additional java opts to append if non-empty java opt is added +} + +func RegisterTemplateEnvVar(v *AlluxioConfigEnvVar) *AlluxioConfigEnvVar { + if _, ok := alluxioEnvVarsTemplate[v.EnvVar]; ok { + panic("Environment variable already registered: " + v.EnvVar) + } + alluxioEnvVarsTemplate[v.EnvVar] = v + return v +} + +const ( + envVersion = "VERSION" // VERSION is specified by libexec/version.sh and should not be overridden by the user + EnvAlluxioAssemblyClientJar = "ALLUXIO_ASSEMBLY_CLIENT_JAR" + EnvAlluxioAssemblyServerJar = "ALLUXIO_ASSEMBLY_SERVER_JAR" + EnvAlluxioClientClasspath = "ALLUXIO_CLIENT_CLASSPATH" + EnvAlluxioServerClasspath = "ALLUXIO_SERVER_CLASSPATH" +) + +var ( + ConfAlluxioConfDir = &AlluxioConfigEnvVar{ + configKey: "alluxio.conf.dir", + EnvVar: "ALLUXIO_CONF_DIR", + } + ConfAlluxioHome = &AlluxioConfigEnvVar{ + configKey: "alluxio.home", + EnvVar: "ALLUXIO_HOME", + } +) + +// environment variables that belong to the templatized alluxio-env.sh +// those with configKeys set will be appended to ALLUXIO_JAVA_OPTS +var ( + confJavaHome = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "JAVA_HOME", + }) + ConfJava = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "JAVA", + }) + ConfAlluxioLogsDir = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + configKey: "alluxio.logs.dir", + EnvVar: "ALLUXIO_LOGS_DIR", + }) + confAlluxioUserLogsDir = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + configKey: "alluxio.user.logs.dir", + EnvVar: "ALLUXIO_USER_LOGS_DIR", + }) + ConfAlluxioJavaOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JAVA_OPTS", + IsJavaOpts: true, + }) + confAlluxioClasspath = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_CLASSPATH", + }) + ConfAlluxioUserJavaOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_USER_JAVA_OPTS", + IsJavaOpts: true, + }) + ConfAlluxioUserAttachOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_USER_ATTACH_OPTS", + IsJavaOpts: true, + }) +) + +// environment variables with corresponding configKeys to append to ALLUXIO_JAVA_OPTS +var ( + confAlluxioRamFolder = &AlluxioConfigEnvVar{ + configKey: "alluxio.worker.tieredstore.level0.dirs.path", + EnvVar: "ALLUXIO_RAM_FOLDER", + additionalAlluxioJavaOpts: map[string]string{ + "alluxio.worker.tieredstore.level0.alias": "MEM", + }, + } + confAlluxioMasterHostname = &AlluxioConfigEnvVar{ + configKey: "alluxio.master.hostname", + EnvVar: "ALLUXIO_MASTER_HOSTNAME", + } + ConfAlluxioMasterMountTableRootUfs = &AlluxioConfigEnvVar{ + configKey: "alluxio.master.mount.table.root.ufs", + EnvVar: "ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS", + } + ConfAlluxioMasterJournalType = &AlluxioConfigEnvVar{ + configKey: "alluxio.master.journal.type", + EnvVar: "ALLUXIO_MASTER_JOURNAL_TYPE", + } + confAlluxioWorkerRamdiskSize = &AlluxioConfigEnvVar{ + configKey: "alluxio.worker.ramdisk.size", + EnvVar: "ALLUXIO_WORKER_RAMDISK_SIZE", + } +) + +func (a *AlluxioConfigEnvVar) ConfigToJavaOpts(env *viper.Viper, required bool) []string { + v := env.Get(a.EnvVar) + if v == nil { + if required { + panic("No value set for required environment variable: " + a.EnvVar) + } + return nil + } + ret := []string{fmt.Sprintf(JavaOptFormat, a.configKey, v)} + for k2, v2 := range a.additionalAlluxioJavaOpts { + ret = append(ret, fmt.Sprintf(JavaOptFormat, k2, v2)) + } + return ret +} + +func (a *AlluxioConfigEnvVar) JavaOptsToArgs(env *viper.Viper) []string { + if !a.IsJavaOpts { + panic(fmt.Sprintf("cannot convert to args because %v is not declared to be a JAVA_OPT environment variable", a.EnvVar)) + } + v := strings.TrimSpace(env.GetString(a.EnvVar)) + if v == "" { + return nil + } + return strings.Split(v, " ") +} diff --git a/cli/src/alluxio.org/cli/env/process.go b/cli/src/alluxio.org/cli/env/process.go new file mode 100644 index 000000000000..347184e29a9f --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process.go @@ -0,0 +1,245 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "os" + "os/exec" + "os/user" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/palantir/stacktrace" + "github.com/shirou/gopsutil/process" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/log" +) + +var ProcessRegistry = map[string]Process{} + +func RegisterProcess(p Process) Process { + name := p.Base().Name + if _, ok := ProcessRegistry[name]; ok { + panic(fmt.Sprintf("Process %v is already registered", name)) + } + ProcessRegistry[name] = p + return p +} + +type Process interface { + Base() *BaseProcess + SetEnvVars(*viper.Viper) + + Start(*StartProcessCommand) error + StartCmd(*cobra.Command) *cobra.Command + + Stop(*StopProcessCommand) error + StopCmd(*cobra.Command) *cobra.Command +} + +type BaseProcess struct { + Name string + JavaClassName string + JavaOpts *AlluxioConfigEnvVar + + // start + ProcessOutFile string + + // monitor + MonitorJavaClassName string +} + +func (p *BaseProcess) Launch(start *StartProcessCommand, args []string) error { + logsDir := Env.EnvVar.GetString(ConfAlluxioLogsDir.EnvVar) + if err := os.MkdirAll(logsDir, 0755); err != nil { + return stacktrace.Propagate(err, "error creating log directory at %v", logsDir) + } + + startCmd := exec.Command("nohup", args...) + for _, k := range Env.EnvVar.AllKeys() { + startCmd.Env = append(startCmd.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + + outFile := filepath.Join(Env.EnvVar.GetString(ConfAlluxioLogsDir.EnvVar), p.ProcessOutFile) + f, err := os.OpenFile(outFile, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return stacktrace.Propagate(err, "error opening file at %v", outFile) + } + startCmd.Stdout = f + startCmd.Stderr = f + + log.Logger.Infof("Starting %v", p.Name) + log.Logger.Debugf("%v > %v 2>&1 &", startCmd.String(), outFile) + if err := startCmd.Start(); err != nil { + return stacktrace.Propagate(err, "error starting %v", p.Name) + } + + if start.AsyncStart { + log.Logger.Warnf("Skipping monitor checks for %v", p.Name) + } else { + // wait a little for main process to start before starting monitor process + time.Sleep(500 * time.Millisecond) + if err := p.Monitor(); err != nil { + return stacktrace.Propagate(err, "error monitoring process after launching") + } + } + return nil +} + +var debugOptsToRemove = []*regexp.Regexp{ + regexp.MustCompile(`-agentlib:jdwp=transport=dt_socket.*address=[0-9]*`), + regexp.MustCompile(`-Dcom.sun.management.jmxremote.port=[0-9]*`), +} + +// Monitor runs the corresponding monitoring java class for the process, running the command +// ${JAVA} -cp ${ALLUXIO_CLIENT_CLASSPATH} ${ALLUXIO_MONITOR_JAVA_OPTS} {monitor java class} +// where ${ALLUXIO_MONITOR_JAVA_OPTS} is equivalent to the process specific JAVA_OPTS +// but with several debugging related options removed +func (p *BaseProcess) Monitor() error { + cmdArgs := []string{"-cp", Env.EnvVar.GetString(EnvAlluxioClientClasspath)} + + javaOpts := strings.TrimSpace(Env.EnvVar.GetString(p.JavaOpts.EnvVar)) + // remove debugging options from java opts for monitor process + for _, re := range debugOptsToRemove { + javaOpts = re.ReplaceAllString(javaOpts, "") + } + cmdArgs = append(cmdArgs, strings.Split(javaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.MonitorJavaClassName) + + cmd := exec.Command(Env.EnvVar.GetString(ConfJava.EnvVar), cmdArgs...) + for _, k := range Env.EnvVar.AllKeys() { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + log.Logger.Errorf("[ FAILED ] The %v process is not serving requests", p.Name) + return stacktrace.Propagate(err, "error running monitor command for %v", p.Name) + } + log.Logger.Infof("[ OK ] The %v process is in a healthy state", p.Name) + return nil +} + +func (p *BaseProcess) Stop(cmd *StopProcessCommand) error { + processes, err := process.Processes() + if err != nil { + return stacktrace.Propagate(err, "error listing processes") + } + currentUser, err := user.Current() + if err != nil { + return stacktrace.Propagate(err, "error fetching the current user") + } + uid, err := strconv.Atoi(currentUser.Uid) + if err != nil { + return stacktrace.Propagate(err, "error converting UID to int") + } + + var matchingProcesses []*process.Process + for _, ps := range processes { + procUIDs, err := ps.Uids() + if err != nil { + continue + } + // match uid first + if len(procUIDs) > 0 && int32(uid) == procUIDs[0] { + cmdline, err := ps.Cmdline() + if err != nil { + // process may have been terminated since listing + continue + } + if strings.Contains(cmdline, p.JavaClassName) { + matchingProcesses = append(matchingProcesses, ps) + } + } + } + if len(matchingProcesses) == 0 { + log.Logger.Warnf("No process to stop because could not find running process matching %v", p.JavaClassName) + return nil + } + log.Logger.Infof("Found %v running process(es) matching %v", len(matchingProcesses), p.JavaClassName) + wg, errs := &sync.WaitGroup{}, make([]error, len(matchingProcesses)) + for i, ps := range matchingProcesses { + i, ps := i, ps + wg.Add(1) + go func() { + defer wg.Done() + pid := int(ps.Pid) + proc, err := os.FindProcess(pid) + if err != nil { + errs[i] = stacktrace.Propagate(err, "error finding process with pid %v", pid) + return + } + if err := proc.Signal(syscall.SIGTERM); err != nil { // syscall.SIGTERM = kill -15 + errs[i] = stacktrace.Propagate(err, "error sending TERM signal to process for %v with pid %v", p.JavaClassName, pid) + return + } + // wait for process to exit + const killTimeoutSec = 120 + if ok := waitForProcessExit(proc, killTimeoutSec); !ok { + if cmd.SoftKill { + errs[i] = fmt.Errorf("Process for %v with pid %v did not terminate after %v seconds", p.JavaClassName, pid, killTimeoutSec) + return + } else { + log.Logger.Warnf("Force killing process for %v with pid %v", p.JavaClassName, pid) + if err := proc.Signal(syscall.SIGKILL); err != nil { // syscall.SIGKILL = kill -9 + errs[i] = stacktrace.Propagate(err, "error sending KILL signal to process for %v with pid %v", p.JavaClassName, pid) + return + } + } + } + }() + } + wg.Wait() + + var errMsg []string + for _, err := range errs { + if err != nil { + errMsg = append(errMsg, fmt.Sprintf("Failed to kill process: %v", err.Error())) + } + } + log.Logger.Infof("Successfully killed %v process(es)", len(matchingProcesses)-len(errMsg)) + if len(errMsg) != 0 { + log.Logger.Errorf("Failed to kill %v process(es):", len(errMsg)) + for _, msg := range errMsg { + log.Logger.Errorln(msg) + } + } + + return nil +} + +func waitForProcessExit(proc *os.Process, killTimeoutSec int) bool { + timeout := time.After(time.Duration(killTimeoutSec) * time.Second) + tick := time.Tick(time.Second * 10) + for { + select { + case <-timeout: + return false + case <-tick: + if err := proc.Signal(syscall.Signal(0)); err != nil { + // process is not found, done + return true + } + } + } +} diff --git a/cli/src/alluxio.org/cli/env/process_start.go b/cli/src/alluxio.org/cli/env/process_start.go new file mode 100644 index 000000000000..22d1d59d1a81 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process_start.go @@ -0,0 +1,49 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "github.com/spf13/cobra" +) + +const StartProcessName = "start" + +type StartProcessCommand struct { + AsyncStart bool + SkipKillOnStart bool +} + +func (c *StartProcessCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: StartProcessName, + Short: "Starts a process locally or a group of similar processes across the cluster", + Long: `Starts a single process locally or a group of similar processes across the cluster. +For starting a group, it is assumed the local host has passwordless SSH access to other nodes in the cluster. +The command will parse the hostnames to run on by reading the conf/masters and conf/workers files, depending on the process type.`, + } + cmd.PersistentFlags().BoolVarP(&c.SkipKillOnStart, "skip-kill-prev", "N", false, "Avoid killing previous running processes when starting") + cmd.PersistentFlags().BoolVarP(&c.AsyncStart, "async", "a", false, "Asynchronously start processes without monitoring for start completion") + + for _, p := range ProcessRegistry { + p := p + cmd.AddCommand(p.StartCmd(&cobra.Command{ + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if !c.SkipKillOnStart { + _ = p.Stop(&StopProcessCommand{}) + } + return p.Start(c) + }, + })) + } + return cmd +} diff --git a/cli/src/alluxio.org/cli/env/process_stop.go b/cli/src/alluxio.org/cli/env/process_stop.go new file mode 100644 index 000000000000..391a237483d4 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process_stop.go @@ -0,0 +1,42 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import "github.com/spf13/cobra" + +const StopProcessName = "stop" + +type StopProcessCommand struct { + SoftKill bool +} + +func (c *StopProcessCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: StopProcessName, + Short: "Stops a process locally or a group of similar processes across the cluster", + Long: `Stops a single process locally or a group of similar processes across the cluster. +For stopping a group, it is assumed the local host has passwordless SSH access to other nodes in the cluster. +The command will parse the hostnames to run on by reading the conf/masters and conf/workers files, depending on the process type.`, + } + cmd.PersistentFlags().BoolVarP(&c.SoftKill, "soft", "s", false, "Soft kill only, don't forcibly kill the process") + + for _, p := range ProcessRegistry { + p := p + cmd.AddCommand(p.StopCmd(&cobra.Command{ + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return p.Stop(c) + }, + })) + } + return cmd +} diff --git a/cli/src/alluxio.org/cli/env/props.go b/cli/src/alluxio.org/cli/env/props.go new file mode 100644 index 000000000000..875310de5074 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/props.go @@ -0,0 +1,17 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +const ( + ConfAlluxioConfValidationEnabled = "alluxio.conf.validation.enabled" + ConfAlluxioLoggerType = "alluxio.logger.type" +) diff --git a/cli/src/alluxio.org/cli/env/service.go b/cli/src/alluxio.org/cli/env/service.go new file mode 100644 index 000000000000..13e14124bbe7 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/service.go @@ -0,0 +1,59 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var serviceRegistry = map[string]*Service{} + +func RegisterService(p *Service) *Service { + if _, ok := serviceRegistry[p.Name]; ok { + panic(fmt.Sprintf("Service %v is already registered", p.Name)) + } + serviceRegistry[p.Name] = p + return p +} + +func InitServiceCommandTree(rootCmd *cobra.Command) { + for _, p := range serviceRegistry { + p.InitCommandTree(rootCmd) + } +} + +type Service struct { + Name string + Description string + Documentation string + Commands []Command +} + +func (s *Service) InitCommandTree(rootCmd *cobra.Command) { + cmd := &cobra.Command{ + Use: s.Name, + Short: s.Description, + Long: s.Documentation, + } + rootCmd.AddCommand(cmd) + + for _, c := range s.Commands { + cmd.AddCommand(c.ToCommand()) + } +} + +func (s *Service) AddCommands(c ...Command) *Service { + s.Commands = append(s.Commands, c...) + return s +} diff --git a/cli/src/alluxio.org/cli/launch/launch.go b/cli/src/alluxio.org/cli/launch/launch.go new file mode 100644 index 000000000000..6a3f0b6ecba6 --- /dev/null +++ b/cli/src/alluxio.org/cli/launch/launch.go @@ -0,0 +1,73 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package launch + +import ( + "path/filepath" + + "github.com/palantir/stacktrace" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +type Launcher struct { + JarEnvVars map[bool]map[string]string + AppendClasspathJars map[string]func(*viper.Viper, string) string + + RootPath string + DebugLog bool + IsDeployed bool +} + +func (l *Launcher) AddFlags(cmd *cobra.Command) error { + const rootPathName = "rootPath" + cmd.PersistentFlags().StringVar(&l.RootPath, rootPathName, "", "Path to root of Alluxio installation") + if err := cmd.MarkPersistentFlagRequired(rootPathName); err != nil { + return stacktrace.Propagate(err, "error marking %v flag required", rootPathName) + } + if err := cmd.PersistentFlags().MarkHidden(rootPathName); err != nil { + return stacktrace.Propagate(err, "error marking %v flag hidden", rootPathName) + } + cmd.PersistentFlags().BoolVar(&l.DebugLog, "debug-log", false, "True to enable debug logging") + const deployedEnv = "deployed-env" + cmd.PersistentFlags().BoolVar(&l.IsDeployed, deployedEnv, false, "True to set paths to be compatible with a deployed environment") + cmd.PersistentFlags().MarkHidden(deployedEnv) + return nil +} + +func (l *Launcher) GetPreRunFunc() func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + if l.DebugLog { + log.Logger.SetLevel(logrus.DebugLevel) + } + if err := env.InitAlluxioEnv(filepath.Clean(l.RootPath), l.JarEnvVars[l.IsDeployed], l.AppendClasspathJars); err != nil { + return stacktrace.Propagate(err, "error defining alluxio environment") + } + return nil + } +} + +func (l *Launcher) Run() error { + rootCmd := &cobra.Command{ + Use: "bin/alluxio", + } + rootCmd.CompletionOptions.DisableDefaultCmd = true + l.AddFlags(rootCmd) + rootCmd.PersistentPreRunE = l.GetPreRunFunc() + env.InitServiceCommandTree(rootCmd) + + return rootCmd.Execute() +} diff --git a/cli/src/alluxio.org/cli/main.go b/cli/src/alluxio.org/cli/main.go new file mode 100644 index 000000000000..7eb2a5a1ac7b --- /dev/null +++ b/cli/src/alluxio.org/cli/main.go @@ -0,0 +1,96 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/viper" + + "alluxio.org/cli/cmd/cache" + "alluxio.org/cli/cmd/conf" + "alluxio.org/cli/cmd/exec" + "alluxio.org/cli/cmd/fs" + "alluxio.org/cli/cmd/generate" + "alluxio.org/cli/cmd/info" + "alluxio.org/cli/cmd/initiate" + "alluxio.org/cli/cmd/job" + "alluxio.org/cli/cmd/journal" + "alluxio.org/cli/cmd/process" + "alluxio.org/cli/env" + "alluxio.org/cli/launch" + "alluxio.org/cli/processes" +) + +func main() { + for _, p := range []env.Process{ + processes.All, + processes.JobMaster, + processes.JobMasters, + processes.JobWorker, + processes.JobWorkers, + processes.Local, + processes.Master, + processes.Masters, + processes.Proxy, + processes.Proxies, + processes.Worker, + processes.Workers, + } { + env.RegisterProcess(p) + } + + for _, c := range []*env.Service{ + cache.Service, + conf.Service, + exec.Service, + fs.Service, + generate.Service, + initiate.Service, + info.Service, + job.Service, + journal.Service, + process.Service, + } { + env.RegisterService(c) + } + + // isDeployed bool -> env var key -> relative path from root with version placeholder + launcher := &launch.Launcher{ + JarEnvVars: map[bool]map[string]string{ + false: { + env.EnvAlluxioAssemblyClientJar: filepath.Join("assembly", "client", "target", "alluxio-assembly-client-%v-jar-with-dependencies.jar"), + env.EnvAlluxioAssemblyServerJar: filepath.Join("assembly", "server", "target", "alluxio-assembly-server-%v-jar-with-dependencies.jar"), + }, + true: { + env.EnvAlluxioAssemblyClientJar: filepath.Join("assembly", "alluxio-client-%v.jar"), + env.EnvAlluxioAssemblyServerJar: filepath.Join("assembly", "alluxio-server-%v.jar"), + }, + }, + AppendClasspathJars: map[string]func(*viper.Viper, string) string{ + env.EnvAlluxioClientClasspath: func(envVar *viper.Viper, ver string) string { + return strings.Join([]string{ + envVar.GetString(env.EnvAlluxioClientClasspath), + filepath.Join(envVar.GetString(env.ConfAlluxioHome.EnvVar), "lib", fmt.Sprintf("alluxio-integration-tools-validation-%v.jar", ver)), + }, ":") + + }, + }, + } + + if err := launcher.Run(); err != nil { + os.Exit(1) + } +} diff --git a/cli/src/alluxio.org/cli/processes/all.go b/cli/src/alluxio.org/cli/processes/all.go new file mode 100644 index 000000000000..a63e1726d2ae --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/all.go @@ -0,0 +1,73 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var All = &AllProcess{ + BaseProcess: &env.BaseProcess{ + Name: "all", + }, + Processes: []env.Process{ + Masters, + Workers, + }, +} + +type AllProcess struct { + *env.BaseProcess + Processes []env.Process +} + +func (p *AllProcess) SetEnvVars(_ *viper.Viper) { + return +} + +func (p *AllProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *AllProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *AllProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *AllProcess) Start(cmd *env.StartProcessCommand) error { + for i := 0; i < len(p.Processes); i++ { + subprocess := p.Processes[i] + if err := subprocess.Start(cmd); err != nil { + return stacktrace.Propagate(err, "error starting subprocesses for %s", subprocess.Base().Name) + } + } + return nil +} + +func (p *AllProcess) Stop(cmd *env.StopProcessCommand) error { + for i := len(p.Processes) - 1; i >= 0; i-- { + subprocess := p.Processes[i] + if err := subprocess.Stop(cmd); err != nil { + return stacktrace.Propagate(err, "error stopping subprocesses for %s", subprocess.Base().Name) + } + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/common.go b/cli/src/alluxio.org/cli/processes/common.go new file mode 100644 index 000000000000..50108a815930 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/common.go @@ -0,0 +1,268 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "os" + "os/user" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/palantir/stacktrace" + "golang.org/x/crypto/ssh" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" + "alluxio.org/log" +) + +type HostnamesFile struct { + Name string + Mutex *sync.Mutex + Once *sync.Once + Hostnames []string +} + +func NewHostnamesFile(name string) *HostnamesFile { + return &HostnamesFile{ + Name: name, + Mutex: &sync.Mutex{}, + Once: &sync.Once{}, + } +} + +// RunSshCommand parses hostnames from the hosts files such as `conf/masters` and `conf/workers` +// For each hostname, create an SSH client and run the given command +func RunSshCommand(command string, hostGroups ...string) error { + // prepare client config, ssh port info + config, port, err := prepareCommand() + if err != nil { + return stacktrace.Propagate(err, "prepare command failed") + } + + // get list of masters or workers, or both + hosts, err := GetHostnames(hostGroups) + if err != nil { + return stacktrace.Propagate(err, "cannot read host names") + } + + // create wait group and channels + var wg sync.WaitGroup + errs := make(chan error) + for _, h := range hosts { + h := h + wg.Add(1) + + go func() { + defer wg.Done() + // dial on target remote address with given host, config and ssh port + dialAddr := fmt.Sprintf("%s:%d", h, port) + conn, err := ssh.Dial("tcp", dialAddr, config) + if err != nil { + errs <- stacktrace.Propagate(err, "dial failed to %v", h) + return + } + defer func(conn *ssh.Client) { + if err := conn.Close(); err != nil { + log.Logger.Warnf("connection to %s closed, error: %s", h, err) + } else { + log.Logger.Debugf("connection to %s closed", h) + } + }(conn) + + // create and set up a session + session, err := conn.NewSession() + if err != nil { + errs <- stacktrace.Propagate(err, "cannot create session at %v.", h) + return + } + defer func(session *ssh.Session) { + if err := session.Close(); err != nil && err != io.EOF { + log.Logger.Warnf("session at %s closed, error: %s", h, err) + } else { + log.Logger.Debugf("session at %s closed", h) + } + }(session) + + session.Stdout = os.Stdout + session.Stderr = os.Stderr + + // run command on the session, output errors + if err = session.Run(command); err != nil { + errs <- stacktrace.Propagate(err, "run command %v failed at %v", command, h) + return + } + }() + } + + go func() { + wg.Wait() + close(errs) + }() + + var errMsgs []string + for err := range errs { + errMsgs = append(errMsgs, err.Error()) + } + if len(errMsgs) > 0 { + return stacktrace.NewError("At least one error encountered:\n%v", strings.Join(errMsgs, "\n")) + } + log.Logger.Infof("Command %s successful on nodes: %s", command, hosts) + return nil +} + +func prepareCommand() (*ssh.ClientConfig, int, error) { + // get the current user + cu, err := user.Current() + if err != nil { + return nil, 0, stacktrace.Propagate(err, "cannot find current user") + } + cuName := cu.Username + log.Logger.Debugf("current user: %v", cuName) + + // get public key + signer, err := getSigner() + if err != nil { + return nil, 0, stacktrace.Propagate(err, "cannot get private key") + } + + // find default ssh port + port, e := net.LookupPort("tcp", "ssh") + if e != nil { + return nil, 0, stacktrace.Propagate(err, "cannot find default ssh port") + } + + // set client config with current user and signer + config := &ssh.ClientConfig{ + User: cuName, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + Timeout: 5 * time.Second, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + return config, port, nil +} + +func addStartFlags(cmd *env.StartProcessCommand, arguments ...string) string { + cliPath := filepath.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), names.BinAlluxio) + var command []string + command = append(command, cliPath) + command = append(command, arguments...) + if cmd.AsyncStart { + command = append(command, "-a") + } + if cmd.SkipKillOnStart { + command = append(command, "-N") + } + return strings.Join(command, " ") +} + +func addStopFlags(cmd *env.StopProcessCommand, arguments ...string) string { + cliPath := filepath.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), names.BinAlluxio) + var command []string + command = append(command, cliPath) + command = append(command, arguments...) + if cmd.SoftKill { + command = append(command, "-s") + } + return strings.Join(command, " ") +} + +const ( + HostGroupMasters = "masters" + HostGroupWorkers = "workers" +) + +func GetHostnames(hostGroups []string) ([]string, error) { + var hosts []string + for _, hostGroup := range hostGroups { + switch hostGroup { + case HostGroupMasters, HostGroupWorkers: + hostnames, err := NewHostnamesFile(hostGroup).getHostnames() + if err != nil { + return nil, stacktrace.Propagate(err, "error listing hostnames from %v", hostGroup) + } + hosts = append(hosts, hostnames...) + default: + return nil, stacktrace.NewError("unknown hosts file: %v", hostGroup) + } + + } + return hosts, nil +} + +func (f *HostnamesFile) getHostnames() ([]string, error) { + f.Mutex.Lock() + defer f.Mutex.Unlock() + var parseErr error + f.Once.Do(func() { + p := filepath.Join(env.Env.EnvVar.GetString(env.ConfAlluxioConfDir.EnvVar), f.Name) + file, err := ioutil.ReadFile(p) + if err != nil { + parseErr = err + } + for _, line := range strings.Split(string(file), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + if strings.TrimSpace(line) != "" { + f.Hostnames = append(f.Hostnames, line) + } + } + }) + if parseErr != nil { + return nil, stacktrace.Propagate(parseErr, "error parsing hostnames file") + } + return f.Hostnames, nil +} + +func getSigner() (ssh.Signer, error) { + // get private key + homePath, err := os.UserHomeDir() + if err != nil { + return nil, stacktrace.Propagate(err, "user home directory not found at %v", homePath) + } + privateKeyFile := filepath.Join(homePath, ".ssh", "id_rsa") + + f, err := os.Open(privateKeyFile) + if err != nil { + return nil, stacktrace.Propagate(err, "error reading file %v", privateKeyFile) + } + defer f.Close() + privateKey, err := ioutil.ReadAll(f) + if err != nil { + return nil, stacktrace.Propagate(err, "error reading contents of %v", privateKeyFile) + } + parsedPrivateKey, err := ssh.ParsePrivateKey(privateKey) + if err != nil { + return nil, stacktrace.Propagate(err, "cannot parse public key at %v", privateKeyFile) + } + return parsedPrivateKey, nil +} + +func argsContainsOpt(args []string, optPrefixes ...string) bool { + for _, arg := range args { + for _, prefix := range optPrefixes { + if strings.HasPrefix(arg, prefix) { + return true + } + } + } + return false +} diff --git a/cli/src/alluxio.org/cli/processes/job_master.go b/cli/src/alluxio.org/cli/processes/job_master.go new file mode 100644 index 000000000000..0b6b0d4d1b5e --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/job_master.go @@ -0,0 +1,96 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var JobMaster = &JobMasterProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_master", + JavaClassName: "alluxio.master.AlluxioJobMaster", + JavaOpts: ConfAlluxioJobMasterJavaOpts, + ProcessOutFile: "job_master.out", + MonitorJavaClassName: "alluxio.master.job.AlluxioJobMasterMonitor", + }, +} + +const ( + confAlluxioJobMasterAuditLoggerType = "alluxio.job.master.audit.logger.type" + envAlluxioAuditJobMasterLogger = "ALLUXIO_AUDIT_JOB_MASTER_LOGGER" + envAlluxioJobMasterLogger = "ALLUXIO_JOB_MASTER_LOGGER" + jobMasterAuditLoggerType = "JOB_MASTER_AUDIT_LOGGER" + jobMasterLoggerType = "JOB_MASTER_LOGGER" +) + +var ( + ConfAlluxioJobMasterJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_MASTER_JAVA_OPTS", + IsJavaOpts: true, + }) + confAlluxioJobMasterAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_MASTER_ATTACH_OPTS", + IsJavaOpts: true, + }) +) + +type JobMasterProcess struct { + *env.BaseProcess +} + +func (p *JobMasterProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *JobMasterProcess) SetEnvVars(envVar *viper.Viper) { + envVar.SetDefault(envAlluxioJobMasterLogger, jobMasterLoggerType) + envVar.SetDefault(envAlluxioAuditJobMasterLogger, jobMasterAuditLoggerType) + // ALLUXIO_JOB_MASTER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_JOB_MASTER_JAVA_OPTS} + javaOpts := []string{ + fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioJobMasterLogger)), + fmt.Sprintf(env.JavaOptFormat, confAlluxioJobMasterAuditLoggerType, envVar.Get(envAlluxioAuditJobMasterLogger)), + } + javaOpts = append(javaOpts, env.ConfAlluxioJavaOpts.JavaOptsToArgs(envVar)...) + javaOpts = append(javaOpts, p.JavaOpts.JavaOptsToArgs(envVar)...) + envVar.Set(p.JavaOpts.EnvVar, strings.Join(javaOpts, " ")) +} + +func (p *JobMasterProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobMasterProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + cmdArgs = append(cmdArgs, confAlluxioJobMasterAttachOpts.JavaOptsToArgs(env.Env.EnvVar)...) + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + cmdArgs = append(cmdArgs, p.JavaOpts.JavaOptsToArgs(env.Env.EnvVar)...) + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *JobMasterProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/job_worker.go b/cli/src/alluxio.org/cli/processes/job_worker.go new file mode 100644 index 000000000000..f94f5ddd6459 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/job_worker.go @@ -0,0 +1,91 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var JobWorker = &WorkerProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_worker", + JavaClassName: "alluxio.worker.AlluxioJobWorker", + JavaOpts: ConfAlluxioJobWorkerJavaOpts, + ProcessOutFile: "job_worker.out", + MonitorJavaClassName: "alluxio.worker.job.AlluxioJobWorkerMonitor", + }, +} + +const ( + envAlluxioJobWorkerLogger = "ALLUXIO_JOB_WORKER_LOGGER" + jobWorkerLoggerType = "JOB_WORKER_LOGGER" +) + +var ( + ConfAlluxioJobWorkerJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_WORKER_JAVA_OPTS", + IsJavaOpts: true, + }) + confAlluxioJobWorkerAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_WORKER_ATTACH_OPTS", + IsJavaOpts: true, + }) +) + +type JobWorkerProcess struct { + *env.BaseProcess +} + +func (p *JobWorkerProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *JobWorkerProcess) SetEnvVars(envVar *viper.Viper) { + envVar.SetDefault(envAlluxioJobWorkerLogger, jobWorkerLoggerType) + // ALLUXIO_JOB_WORKER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_WORKER_JAVA_OPTS} + javaOpts := []string{ + fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioJobWorkerLogger)), + } + javaOpts = append(javaOpts, env.ConfAlluxioJavaOpts.JavaOptsToArgs(envVar)...) + javaOpts = append(javaOpts, p.JavaOpts.JavaOptsToArgs(envVar)...) + envVar.Set(p.JavaOpts.EnvVar, strings.Join(javaOpts, " ")) +} + +func (p *JobWorkerProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobWorkerProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + cmdArgs = append(cmdArgs, confAlluxioJobWorkerAttachOpts.JavaOptsToArgs(env.Env.EnvVar)...) + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + cmdArgs = append(cmdArgs, p.JavaOpts.JavaOptsToArgs(env.Env.EnvVar)...) + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *JobWorkerProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/local.go b/cli/src/alluxio.org/cli/processes/local.go new file mode 100644 index 000000000000..3729a35c232a --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/local.go @@ -0,0 +1,73 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Local = &LocalProcess{ + BaseProcess: &env.BaseProcess{ + Name: "local", + }, + Processes: []env.Process{ + Master, + Worker, + }, +} + +type LocalProcess struct { + *env.BaseProcess + Processes []env.Process +} + +func (p *LocalProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *LocalProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *LocalProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *LocalProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *LocalProcess) Start(cmd *env.StartProcessCommand) error { + for i := 0; i < len(p.Processes); i++ { + subProcess := p.Processes[i] + if err := subProcess.Start(cmd); err != nil { + return stacktrace.Propagate(err, "Error starting subprocesses for %s", p.Processes[i]) + } + } + return nil +} + +func (p *LocalProcess) Stop(cmd *env.StopProcessCommand) error { + for i := len(p.Processes) - 1; i >= 0; i-- { + subProcess := p.Processes[i] + if err := subProcess.Stop(cmd); err != nil { + return stacktrace.Propagate(err, "Error stopping subprocesses for %s", p.Processes[i]) + } + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/master.go b/cli/src/alluxio.org/cli/processes/master.go new file mode 100644 index 000000000000..a71442477ea1 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/master.go @@ -0,0 +1,141 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "os" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/cmd/conf" + "alluxio.org/cli/cmd/journal" + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Master = &MasterProcess{ + BaseProcess: &env.BaseProcess{ + Name: "master", + JavaClassName: "alluxio.master.AlluxioMaster", + JavaOpts: ConfAlluxioMasterJavaOpts, + ProcessOutFile: "master.out", + MonitorJavaClassName: "alluxio.master.AlluxioMasterMonitor", + }, +} + +const ( + confAlluxioMasterAuditLoggerType = "alluxio.master.audit.logger.type" + confAlluxioMasterJournalFolder = "alluxio.master.journal.folder" + + envAlluxioAuditMasterLogger = "ALLUXIO_AUDIT_MASTER_LOGGER" + envAlluxioMasterLogger = "ALLUXIO_MASTER_LOGGER" + masterAuditLoggerType = "MASTER_AUDIT_LOGGER" + masterLoggerType = "MASTER_LOGGER" +) + +var ( + ConfAlluxioMasterJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_MASTER_JAVA_OPTS", + IsJavaOpts: true, + }) + confAlluxioMasterAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_MASTER_ATTACH_OPTS", + IsJavaOpts: true, + }) +) + +type MasterProcess struct { + *env.BaseProcess +} + +func (p *MasterProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *MasterProcess) SetEnvVars(envVar *viper.Viper) { + envVar.SetDefault(envAlluxioMasterLogger, masterLoggerType) + envVar.SetDefault(envAlluxioAuditMasterLogger, masterAuditLoggerType) + // ALLUXIO_MASTER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_MASTER_JAVA_OPTS} + javaOpts := []string{ + fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioMasterLogger)), + fmt.Sprintf(env.JavaOptFormat, confAlluxioMasterAuditLoggerType, envVar.Get(envAlluxioAuditMasterLogger)), + } + javaOpts = append(javaOpts, env.ConfAlluxioJavaOpts.JavaOptsToArgs(envVar)...) + javaOpts = append(javaOpts, p.JavaOpts.JavaOptsToArgs(envVar)...) + envVar.Set(p.JavaOpts.EnvVar, strings.Join(javaOpts, " ")) +} + +func (p *MasterProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MasterProcess) Start(cmd *env.StartProcessCommand) error { + if err := p.checkJournal(); err != nil { + return stacktrace.Propagate(err, "error validating journal") + } + + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + cmdArgs = append(cmdArgs, confAlluxioMasterAttachOpts.JavaOptsToArgs(env.Env.EnvVar)...) + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + cmdArgs = append(cmdArgs, p.JavaOpts.JavaOptsToArgs(env.Env.EnvVar)...) + + // specify a default of -Xmx8g if no memory setting is specified + const xmxOpt = "-Xmx" + if !argsContainsOpt(cmdArgs, xmxOpt, "MaxRAMPercentage") { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v8g", xmxOpt)) + } + // specify a default of -XX:MetaspaceSize=256M if not set + const metaspaceSizeOpt = "-XX:MetaspaceSize" + if !argsContainsOpt(cmdArgs, metaspaceSizeOpt) { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v=256M", metaspaceSizeOpt)) + } + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *MasterProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MasterProcess) checkJournal() error { + journalDir, err := conf.Get.FetchValue(confAlluxioMasterJournalFolder) + if err != nil { + return stacktrace.Propagate(err, "error fetching value for %v", confAlluxioMasterJournalFolder) + } + stat, err := os.Stat(journalDir) + if os.IsNotExist(err) { + log.Logger.Info("Journal directory does not exist, formatting") + if err := journal.Format.Format(); err != nil { + return stacktrace.Propagate(err, "error formatting journal") + } + return nil + } + if err != nil { + return stacktrace.Propagate(err, "error listing path at %v", journalDir) + } + if !stat.IsDir() { + return stacktrace.NewError("Journal location %v is not a directory. Please remove the file and retry.", journalDir) + } + // journal folder path exists and is a directory + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/multi.go b/cli/src/alluxio.org/cli/processes/multi.go new file mode 100644 index 000000000000..c2095f1751fc --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/multi.go @@ -0,0 +1,100 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/cmd/process" + "alluxio.org/cli/env" +) + +var ( + JobMasters = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_masters", + }, + HostnameFiles: []string{HostGroupMasters}, + SingleProcessName: JobMaster.Name, + } + JobWorkers = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_workers", + }, + HostnameFiles: []string{HostGroupWorkers}, + SingleProcessName: JobWorker.Name, + } + Masters = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "masters", + }, + HostnameFiles: []string{HostGroupMasters}, + SingleProcessName: Master.Name, + } + Proxies = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "proxies", + }, + HostnameFiles: []string{HostGroupMasters, HostGroupWorkers}, + SingleProcessName: Proxy.Name, + } + Workers = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "workers", + }, + HostnameFiles: []string{HostGroupWorkers}, + SingleProcessName: Worker.Name, + } +) + +type MultiProcess struct { + *env.BaseProcess + HostnameFiles []string + SingleProcessName string +} + +func (p *MultiProcess) SetEnvVars(_ *viper.Viper) { + return +} + +func (p *MultiProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *MultiProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MultiProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MultiProcess) Start(cmd *env.StartProcessCommand) error { + return RunSshCommand( + addStartFlags(cmd, + process.Service.Name, + env.StartProcessName, + p.SingleProcessName, + ), p.HostnameFiles...) +} + +func (p *MultiProcess) Stop(cmd *env.StopProcessCommand) error { + return RunSshCommand( + addStopFlags(cmd, + process.Service.Name, + env.StopProcessName, + p.SingleProcessName, + ), p.HostnameFiles...) +} diff --git a/cli/src/alluxio.org/cli/processes/proxy.go b/cli/src/alluxio.org/cli/processes/proxy.go new file mode 100644 index 000000000000..81b51f8ccaae --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/proxy.go @@ -0,0 +1,96 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Proxy = &ProxyProcess{ + BaseProcess: &env.BaseProcess{ + Name: "proxy", + JavaClassName: "alluxio.proxy.AlluxioProxy", + JavaOpts: ConfAlluxioProxyJavaOpts, + ProcessOutFile: "proxy.out", + MonitorJavaClassName: "alluxio.proxy.AlluxioProxyMonitor", + }, +} + +const ( + confAlluxioProxyAuditLoggerType = "alluxio.proxy.audit.logger.type" + envAlluxioAuditProxyLogger = "ALLUXIO_AUDIT_PROXY_LOGGER" + envAlluxioProxyLogger = "ALLUXIO_PROXY_LOGGER" + proxyAuditLoggerType = "PROXY_AUDIT_LOGGER" + proxyLoggerType = "PROXY_LOGGER" +) + +var ( + ConfAlluxioProxyJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_PROXY_JAVA_OPTS", + IsJavaOpts: true, + }) + confAlluxioProxyAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_PROXY_ATTACH_OPTS", + IsJavaOpts: true, + }) +) + +type ProxyProcess struct { + *env.BaseProcess +} + +func (p *ProxyProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *ProxyProcess) SetEnvVars(envVar *viper.Viper) { + envVar.SetDefault(envAlluxioProxyLogger, proxyLoggerType) + envVar.SetDefault(envAlluxioAuditProxyLogger, proxyAuditLoggerType) + // ALLUXIO_PROXY_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_PROXY_JAVA_OPTS} + javaOpts := []string{ + fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioProxyLogger)), + fmt.Sprintf(env.JavaOptFormat, confAlluxioProxyAuditLoggerType, envVar.Get(envAlluxioAuditProxyLogger)), + } + javaOpts = append(javaOpts, env.ConfAlluxioJavaOpts.JavaOptsToArgs(envVar)...) + javaOpts = append(javaOpts, p.JavaOpts.JavaOptsToArgs(envVar)...) + envVar.Set(p.JavaOpts.EnvVar, strings.Join(javaOpts, " ")) +} + +func (p *ProxyProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *ProxyProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + cmdArgs = append(cmdArgs, confAlluxioProxyAttachOpts.JavaOptsToArgs(env.Env.EnvVar)...) + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + cmdArgs = append(cmdArgs, p.JavaOpts.JavaOptsToArgs(env.Env.EnvVar)...) + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *ProxyProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/worker.go b/cli/src/alluxio.org/cli/processes/worker.go new file mode 100644 index 000000000000..de32a7b3dec1 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/worker.go @@ -0,0 +1,104 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Worker = &WorkerProcess{ + BaseProcess: &env.BaseProcess{ + Name: "worker", + JavaClassName: "alluxio.worker.AlluxioWorker", + JavaOpts: ConfAlluxioWorkerJavaOpts, + ProcessOutFile: "worker.out", + MonitorJavaClassName: "alluxio.worker.AlluxioWorkerMonitor", + }, +} + +const ( + envAlluxioWorkerLogger = "ALLUXIO_WORKER_LOGGER" + workerLoggerType = "WORKER_LOGGER" +) + +var ( + ConfAlluxioWorkerJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_WORKER_JAVA_OPTS", + IsJavaOpts: true, + }) + confAlluxioWorkerAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_WORKER_ATTACH_OPTS", + IsJavaOpts: true, + }) +) + +type WorkerProcess struct { + *env.BaseProcess +} + +func (p *WorkerProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *WorkerProcess) SetEnvVars(envVar *viper.Viper) { + envVar.SetDefault(envAlluxioWorkerLogger, workerLoggerType) + // ALLUXIO_WORKER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_WORKER_JAVA_OPTS} + javaOpts := []string{ + fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioWorkerLogger)), + } + javaOpts = append(javaOpts, env.ConfAlluxioJavaOpts.JavaOptsToArgs(envVar)...) + javaOpts = append(javaOpts, p.JavaOpts.JavaOptsToArgs(envVar)...) + envVar.Set(p.JavaOpts.EnvVar, strings.Join(javaOpts, " ")) +} + +func (p *WorkerProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *WorkerProcess) startCmdArgs() []string { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + cmdArgs = append(cmdArgs, confAlluxioWorkerAttachOpts.JavaOptsToArgs(env.Env.EnvVar)...) + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + cmdArgs = append(cmdArgs, p.JavaOpts.JavaOptsToArgs(env.Env.EnvVar)...) + + // specify a default of -Xmx4g if no memory setting is specified + const xmxOpt = "-Xmx" + if !argsContainsOpt(cmdArgs, xmxOpt, "MaxRAMPercentage") { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v4g", xmxOpt)) + } + // specify a default of -XX:MaxDirectMemorySize=4g if not set + const maxDirectMemorySize = "-XX:MaxDirectMemorySize" + if !argsContainsOpt(cmdArgs, maxDirectMemorySize) { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v=4g", maxDirectMemorySize)) + } + return append(cmdArgs, p.JavaClassName) +} + +func (p *WorkerProcess) Start(cmd *env.StartProcessCommand) error { + if err := p.Launch(cmd, p.startCmdArgs()); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *WorkerProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/worker_test.go b/cli/src/alluxio.org/cli/processes/worker_test.go new file mode 100644 index 000000000000..3fdadb500c13 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/worker_test.go @@ -0,0 +1,86 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "strings" + "testing" + + "alluxio.org/cli/env" +) + +func TestWorkerProcess_startCmdArgs(t *testing.T) { + env.RegisterProcess(Worker) + for name, tc := range map[string]struct { + alluxioEnv map[string]string + expectNumArgs int + }{ + "none": { + expectNumArgs: 15, + }, + "2 alluxio java opts": { + alluxioEnv: map[string]string{ + env.ConfAlluxioJavaOpts.EnvVar: "-DalluxioJava1=opts1 -DalluxioJava2=opts2", + }, + expectNumArgs: 17, + }, + "2 alluxio java opts with spaces": { + alluxioEnv: map[string]string{ + env.ConfAlluxioJavaOpts.EnvVar: " -DalluxioJava1=opts1 -DalluxioJava2=opts2 ", + }, + expectNumArgs: 17, + }, + "2 worker java opts with spaces": { + alluxioEnv: map[string]string{ + ConfAlluxioWorkerJavaOpts.EnvVar: " -DaworkerJava1=opts1 -DworkerJava2=opts2 ", + }, + expectNumArgs: 17, + }, + "2 worker attach java opts with spaces": { + alluxioEnv: map[string]string{ + confAlluxioWorkerAttachOpts.EnvVar: " -DaworkerAttach1=opts1 -DworkerAttach2=opts2 ", + }, + expectNumArgs: 17, + }, + "1 each of alluxio java, worker java, worker attach": { + alluxioEnv: map[string]string{ + env.ConfAlluxioJavaOpts.EnvVar: "-DalluxioJava1=opts1", + ConfAlluxioWorkerJavaOpts.EnvVar: "-DaworkerJava1=opts1", + confAlluxioWorkerAttachOpts.EnvVar: "-DaworkerAttach1=opts1", + }, + expectNumArgs: 18, + }, + "1 each of alluxio java, worker java, worker attach with spaces": { + alluxioEnv: map[string]string{ + env.ConfAlluxioJavaOpts.EnvVar: " -DalluxioJava1=opts1 ", + ConfAlluxioWorkerJavaOpts.EnvVar: " -DaworkerJava1=opts1 ", + confAlluxioWorkerAttachOpts.EnvVar: " -DaworkerAttach1=opts1 ", + }, + expectNumArgs: 18, + }, + } { + env.InitEnvForTesting(tc.alluxioEnv) + args := Worker.startCmdArgs() + if len(args) != tc.expectNumArgs { + t.Errorf("case %v: expected %v args but got %v:\n%v", + name, tc.expectNumArgs, len(args), strings.Join(args, "\n")) + } + for i, arg := range args { + if arg == "" { + t.Errorf("case %v: %dth argument is empty", name, i) + } + if arg != strings.TrimSpace(arg) { + t.Errorf("case %v: %dth argument has leading or trailing spaces:\n%v", name, i, arg) + } + } + } +} diff --git a/cli/src/alluxio.org/go.mod b/cli/src/alluxio.org/go.mod new file mode 100644 index 000000000000..3bf63f7b1dc2 --- /dev/null +++ b/cli/src/alluxio.org/go.mod @@ -0,0 +1,17 @@ +module alluxio.org + +go 1.15 + +require ( + github.com/iancoleman/orderedmap v0.3.0 + github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 + github.com/shirou/gopsutil v3.21.11+incompatible + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/cobra v1.7.0 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.6.0 + github.com/tklauser/go-sysconf v0.3.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/cli/src/alluxio.org/go.sum b/cli/src/alluxio.org/go.sum new file mode 100644 index 000000000000..873de4c60a46 --- /dev/null +++ b/cli/src/alluxio.org/go.sum @@ -0,0 +1,192 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= +github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 h1:nRlQD0u1871kaznCnn1EvYiMbum36v7hw1DLPEjds4o= +github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177/go.mod h1:ao5zGxj8Z4x60IOVYZUbDSmt3R8Ddo080vEgPosHpak= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.6.0 h1:qSjVKzM2dmqQLutPN4Y0SEzDpAf7T6HHIT3E2Xr75Gg= +github.com/spf13/viper v1.6.0/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tklauser/go-sysconf v0.3.0 h1:vqcXSDUzrLuLJN6olUN62tT629h1uwcW3i5juA7TwSU= +github.com/tklauser/go-sysconf v0.3.0/go.mod h1:h54uFIrVIJBr8RXt3F5JJdxVkmFeallWuXajbMhn2O8= +github.com/tklauser/numcpus v0.1.0 h1:BQeAuOQKZtytv8qblsFbi9mhTfXQdFFkyX0vaV6B4tc= +github.com/tklauser/numcpus v0.1.0/go.mod h1:i3up9VjARpkV00NkBbexuDd0RAVQz4rV5Gl8miYgd5k= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/cli/src/alluxio.org/log/log.go b/cli/src/alluxio.org/log/log.go new file mode 100644 index 000000000000..86e50b31187b --- /dev/null +++ b/cli/src/alluxio.org/log/log.go @@ -0,0 +1,30 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package log + +import ( + "os" + + "github.com/sirupsen/logrus" +) + +var Logger *logrus.Logger + +func init() { + l := logrus.New() + l.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + }) + l.SetOutput(os.Stdout) + + Logger = l +} diff --git a/common/pom.xml b/common/pom.xml new file mode 100644 index 000000000000..7662cfdbd3b6 --- /dev/null +++ b/common/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + org.alluxio + alluxio-parent + 314-SNAPSHOT + + alluxio-common + pom + Alluxio Common + Parent POM for Alluxio common + + + + ${project.parent.basedir}/build + + + + transport + + diff --git a/common/transport/pom.xml b/common/transport/pom.xml new file mode 100644 index 000000000000..f92f91762f5d --- /dev/null +++ b/common/transport/pom.xml @@ -0,0 +1,180 @@ + + + + 4.0.0 + + org.alluxio + alluxio-common + 314-SNAPSHOT + + alluxio-core-transport + jar + Alluxio Core - Transport + Protobuf shared in Alluxio core modules + + + + + ${project.parent.parent.basedir}/build + false + + alluxio.core.transport + false + + + + + com.google.guava + guava + + + com.google.protobuf + protobuf-java + + + io.grpc + grpc-core + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + ${skip.protoc} + + + + + compile + compile-custom + + + + + + + + com.salesforce.servicelibs + proto-backwards-compatibility + 1.0.7 + + ${basedir}/src/main/proto + + + + + + + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + true + + + + + com.mycila + license-maven-plugin + + true + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + + kr.motd.maven + os-maven-plugin + 1.6.2 + + + + + + + generate + + false + + + + + + org.apache.maven.plugins + maven-clean-plugin + + + + src/main/java/alluxio/proto + false + + **/*.java + + + + src/main/java/alluxio/grpc + false + + **/*.java + + + + + + + + + + java11 + + 11 + + + + javax.annotation + javax.annotation-api + + + + + diff --git a/core/transport/src/main/proto/grpc/block_master.proto b/common/transport/src/main/proto/grpc/block_master.proto similarity index 87% rename from core/transport/src/main/proto/grpc/block_master.proto rename to common/transport/src/main/proto/grpc/block_master.proto index 1740e49a834b..4324cd3338de 100644 --- a/core/transport/src/main/proto/grpc/block_master.proto +++ b/common/transport/src/main/proto/grpc/block_master.proto @@ -59,7 +59,7 @@ message GetUsedBytesPResponse { } message WorkerInfo { - optional int64 id = 1; + optional int64 id = 1 [deprecated = true]; // deprecated in favor of WorkerIdentity optional grpc.WorkerNetAddress address = 2; optional int32 lastContactSec = 3; optional string state = 4; @@ -69,6 +69,7 @@ message WorkerInfo { map capacityBytesOnTiers = 8; map usedBytesOnTiers = 9; optional BuildVersion buildVersion = 10; + optional WorkerIdentity identity = 11; } enum WorkerRange { @@ -109,16 +110,24 @@ message WorkerLostStorageInfo { /** a map from tier alias to the lost storage paths */ map lostStorage = 2; } -message RemoveDecommissionedWorkerPOptions { - optional string workerName = 1; +message RemoveDisabledWorkerPOptions { + required string workerHostname = 1; + optional int64 workerWebPort = 2; } -message RemoveDecommissionedWorkerPResponse {} +message RemoveDisabledWorkerPResponse {} message GetWorkerLostStoragePOptions {} message GetWorkerLostStoragePResponse { repeated WorkerLostStorageInfo workerLostStorageInfo = 1; } +message DecommissionWorkerPResponse {} +message DecommissionWorkerPOptions { + required string workerHostname = 1; + optional int64 workerWebPort = 2; + optional bool canRegisterAgain = 3; +} + /** * This interface contains block master service endpoints for Alluxio clients. */ @@ -149,12 +158,17 @@ service BlockMasterClientService { */ rpc GetWorkerInfoList(GetWorkerInfoListPOptions) returns (GetWorkerInfoListPResponse); + /** + * Returns a list of workers information. + */ + rpc GetLostWorkerList(GetWorkerInfoListPOptions) returns (GetWorkerInfoListPResponse); + /** * If target worker is in the decommissioned worker set, * return true, remove target worker from decommissioned worker set; else, return false. */ - rpc RemoveDecommissionedWorker(RemoveDecommissionedWorkerPOptions) - returns (RemoveDecommissionedWorkerPResponse); + rpc RemoveDisabledWorker(RemoveDisabledWorkerPOptions) + returns (RemoveDisabledWorkerPResponse); /** * Returns a list of workers information for report CLI. @@ -165,6 +179,11 @@ service BlockMasterClientService { * Returns a list of worker lost storage information */ rpc GetWorkerLostStorage(GetWorkerLostStoragePOptions) returns (GetWorkerLostStoragePResponse); + + /** + * Decommission the specific worker from Alluxio. + */ + rpc DecommissionWorker(DecommissionWorkerPOptions) returns (DecommissionWorkerPResponse); } message TierList { @@ -229,16 +248,16 @@ message CommitBlockPRequest { } message CommitBlockPOptions {} -message CommitBlockInUfsPRequest { - /** the id of the worker */ - optional int64 blockId = 1; - /** the space used in bytes on the target tier */ - optional int64 length = 2; - optional CommitBlockInUfsPOptions options = 3; -} -message CommitBlockInUfsPOptions {} -message CommitBlockInUfsPResponse {} +message NotifyWorkerIdPOptions {} +message NotifyWorkerIdPRequest { + optional int64 workerId = 1; + /** the worker network address */ + optional grpc.WorkerNetAddress workerNetAddress = 2; + optional NotifyWorkerIdPOptions options = 3; +} +message NotifyWorkerIdPResponse { +} message GetWorkerIdPOptions {} message GetWorkerIdPRequest { @@ -263,12 +282,6 @@ message GetRegisterLeasePResponse { optional GetRegisterLeasePOptions options = 4; } -message BuildVersion { - /** the project version of the worker */ - optional string version = 1; - /** the git revision at the time of building the worker */ - optional string revision = 2; -} message RegisterWorkerPOptions { repeated grpc.ConfigProperty configs = 1; /** the worker version to display in info pages (useful for rolling upgrades) */ @@ -308,16 +321,16 @@ service BlockMasterWorkerService { */ rpc CommitBlock(CommitBlockPRequest) returns(CommitBlockPResponse); - /** - * Marks the given block as committed which resides in UFS. - */ - rpc CommitBlockInUfs(CommitBlockInUfsPRequest) returns(CommitBlockInUfsPResponse); - /** * Returns a worker id for the given network address. */ rpc GetWorkerId(GetWorkerIdPRequest) returns (GetWorkerIdPResponse); + /** + * Notify all masters about the worker ID. + */ + rpc NotifyWorkerId(NotifyWorkerIdPRequest) returns (NotifyWorkerIdPResponse); + /** * Registers a worker. */ diff --git a/common/transport/src/main/proto/grpc/block_worker.proto b/common/transport/src/main/proto/grpc/block_worker.proto new file mode 100644 index 000000000000..c8bada319ecf --- /dev/null +++ b/common/transport/src/main/proto/grpc/block_worker.proto @@ -0,0 +1,386 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_package = "alluxio.grpc"; +option java_outer_classname = "BlockWorkerProto"; + +package alluxio.grpc.block; + +import "proto/dataserver/protocol.proto"; +import "grpc/common.proto"; +import "grpc/file_system_master.proto"; + +// The block worker service +service BlockWorker { + /** + * Returns the status of the file or directory. + */ + rpc GetStatus (grpc.file.GetStatusPRequest) returns (grpc.file.GetStatusPResponse); + /** + * If the path points to a file, the method returns a singleton with its file information. + * If the path points to a directory, the method returns a list with file information for the + * directory contents. + */ + rpc ListStatus(grpc.file.ListStatusPRequest) returns (stream grpc.file.ListStatusPResponse); + + + rpc ReadBlock (stream ReadRequest) returns (stream ReadResponse); + rpc WriteBlock (stream WriteRequest) returns (stream WriteResponse); + + rpc AsyncCache (AsyncCacheRequest) returns (AsyncCacheResponse); + rpc Cache (CacheRequest) returns (CacheResponse); + rpc Load(LoadRequest)returns (LoadResponse); + rpc LoadFile(LoadFileRequest)returns (LoadFileResponse); + rpc Copy(CopyRequest)returns (CopyResponse); + rpc RemoveBlock (RemoveBlockRequest) returns (RemoveBlockResponse); + rpc MoveBlock (MoveBlockRequest) returns (MoveBlockResponse); + + rpc Move(MoveRequest) returns (MoveResponse); + + // TODO(lu) Move to metrics worker + rpc ClearMetrics (ClearMetricsRequest) returns (ClearMetricsResponse); + + rpc FreeWorker (FreeWorkerRequest) returns (FreeWorkerResponse); + + /** + * Creates a file. + */ + rpc CreateFile(grpc.file.CreateFilePRequest) returns (grpc.file.CreateFilePResponse); + + /** + * Marks a file as completed. + */ + rpc CompleteFile(grpc.file.CompleteFilePRequest) returns (grpc.file.CompleteFilePResponse); + + /** + * Deletes a file or a directory and returns whether the remove operation succeeded. + */ + rpc Remove(grpc.file.DeletePRequest) returns (grpc.file.DeletePResponse); + + /** + * Renames a file or a directory. + */ + rpc Rename(grpc.file.RenamePRequest) returns (grpc.file.RenamePResponse); + + /** + * Creates a directory. + */ + rpc CreateDirectory(grpc.file.CreateDirectoryPRequest) returns (grpc.file.CreateDirectoryPResponse); + + /** + * Checks the existence of a file or directory. + */ + rpc Exists(grpc.file.ExistsPRequest) returns (grpc.file.ExistsPResponse); + + /** + * Sets file or directory attributes. + */ + rpc SetAttribute(grpc.file.SetAttributePRequest) returns (grpc.file.SetAttributePResponse); + + /** + * Caches data from UFS. + */ + rpc CacheData(CacheDataRequest) returns (CacheDataResponse); +} + +message UfsStatus { + optional string name = 1; + optional bool is_directory = 2; + optional int64 last_modified_time_ms = 3; + optional string owner = 4; + optional string group = 5; + optional int32 mode = 6; + map xattr = 7; + optional string ufs_full_path = 8; + + optional UfsFileStatus ufs_file_status = 100; +} + +message UfsFileStatus { + optional string content_hash = 1; + optional int64 content_length = 2; + optional int64 block_size = 3; +} + +// The check request +message CheckRequest {} + +// The check response +message CheckResponse {} + +// The data chunk. +// next available id: 2 +message Chunk { + optional bytes data = 1; +} + +// The read/write request type. It can either be an Alluxio block operation or a UFS file operation. +// next available id: 3 +enum RequestType { + ALLUXIO_BLOCK = 0; + UFS_FILE = 1; +} + +// The read request. +// next available id: 9 +message ReadRequest { + optional int64 block_id = 1; + optional int64 offset = 2; + optional int64 length = 3; + // Whether the block should be promoted before reading + optional bool promote = 4; + optional int64 chunk_size = 5; + + // This is only set for UFS block read. + optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 6; + + // Read receipt + optional int64 offset_received = 7; + + // Is position read to a small buffer + optional bool position_short = 8; +} + +// The read response. +// next available id: 2 +message ReadResponse { + optional Chunk chunk = 1; +} + +// The write request command. +// next available id: 11 +message WriteRequestCommand { + optional RequestType type = 1; + // The block ID or UFS file ID. + optional int64 id = 2; + optional int64 offset = 3; + // This is only applicable for block write. + optional int32 tier = 4; + optional bool flush = 5; + // Cancel, close and error will be handled by standard gRPC stream APIs. + optional alluxio.proto.dataserver.CreateUfsFileOptions create_ufs_file_options = 6; + optional string medium_type = 8; + optional bool pin_on_create = 9; + optional int64 space_to_reserve = 10; +} + +// The write request. +// next available id: 3 +message WriteRequest { + oneof value { + WriteRequestCommand command = 1; + Chunk chunk = 2; + } +} + +// The write response. +// next available id: 3 +message WriteResponse { + optional int64 offset = 1; + // Errors will be handled by standard gRPC stream APIs. + optional string contentHash = 2; +} + +// Request for caching a block asynchronously +// Deprecated and will be removed in v3.0 +message AsyncCacheRequest { + optional int64 block_id = 1; + // TODO(calvin): source host and port should be replace with WorkerNetAddress + optional string source_host = 2; + optional int32 source_port = 3; + optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 4; + optional int64 length = 5; +} + +// Request for caching a block synchronously/asynchronously +// next available id: 7 +message CacheRequest { + optional int64 block_id = 1; + // TODO(calvin): source host and port should be replace with WorkerNetAddress + optional string source_host = 2; + optional int32 source_port = 3; + optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 4; + optional int64 length = 5; + optional bool async = 6; +} + +message CacheDataRequest { + optional string ufs_path = 1; + optional int64 pos = 2; + optional int64 length = 3; + optional bool async = 4; +} + +message CacheDataResponse {} + +// Request for load a block into alluxio +// next available id: 3 +message LoadRequest { + repeated Block blocks = 1; + required UfsReadOptions options = 2; +} + +message UfsReadOptions{ + required string tag = 1; + // is position short or not, used for HDFS performance optimization. + // When the client buffer size is large ( > 2MB) and reads are guaranteed to be somewhat + // sequential, the `pread` API to HDFS is not as efficient as simple `read`. + // We introduce a heuristic to choose which API to use. + required bool position_short = 2; + optional int64 bandwidth = 3; + optional string user = 4; +} + +message Block{ + optional int64 block_id = 1; + // The block length. + required int64 length = 2; + optional string ufs_path = 3; + // The offset of the block in within ufs the file. + optional int64 offset_in_file = 4; + optional int64 mountId = 5; + optional UfsStatus ufs_status = 6; +} + +message LoadResponse { + required TaskStatus status = 1; + repeated BlockStatus block_status = 2; +} + +enum TaskStatus { + SUCCESS = 0; + FAILURE = 1; + PARTIAL_FAILURE = 2; +} + +message LoadFileRequest { + repeated LoadSubTask subtasks = 1; + required UfsReadOptions options = 2; + optional bool skip_if_exists = 3; +} + +// A subtask of a load file request. either a load data or load metadata. +message LoadSubTask { + oneof task { + LoadDataSubTask load_data_subtask = 1; + LoadMetadataSubTask load_metadata_subtask = 2; + } +} + +message LoadDataSubTask { + required int64 length = 1; + optional string ufs_path = 2; + optional int64 offset_in_file = 3; + optional UfsStatus ufs_status = 4; + optional WorkerNetAddress main_worker = 5; +} + +message LoadMetadataSubTask { + optional UfsStatus ufs_status = 1; +} + +message File{ + optional string alluxio_path = 1; + optional string ufs_path = 2; + // file length + optional int64 length = 3; + optional int64 mountId = 4; +} + +message LoadFileResponse { + required TaskStatus status = 1; + repeated LoadFailure failures = 2; + optional int32 num_skipped = 3; + optional int64 bytes_skipped = 4; +} + +message FreeWorkerRequest{} + +message FreeWorkerResponse{} + +message BlockStatus { + required Block block = 1; + // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + required int32 code = 2; + // A developer-facing error message + optional string message = 3; + optional bool retryable = 4; +} + +// load subtask failure, either metadata or load block we only need to input one of them +message LoadFailure { + optional LoadSubTask subtask = 1; + // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + optional int32 code = 2; + // A developer-facing error message + optional string message = 3; + optional bool retryable = 4; +} + +// Response for an async cache request +message AsyncCacheResponse {} + +// Response for an async cache request +message CacheResponse {} + +message CopyRequest { + repeated Route routes = 1; + optional UfsReadOptions ufs_read_options = 2; + required WriteOptions write_options= 3; +} + +message Route { + required string src = 1; + required string dst = 2; + optional int64 length = 3; +} + +message CopyResponse { + required TaskStatus status = 1; + repeated RouteFailure failures = 2; +} + +message WriteOptions { + optional bool overwrite = 1; + optional grpc.file.WritePType write_type = 2 [default = CACHE_THROUGH]; + optional bool check_content = 3; +} + +message RouteFailure { + required Route route = 1; + // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + required int32 code = 2; + // A developer-facing error message + optional string message = 3; + optional bool retryable = 4; + optional bool is_skip = 5; +} + +// next available id: 2 +message RemoveBlockRequest { + optional int64 block_id = 1; +} + +message RemoveBlockResponse {} + +message MoveBlockRequest { + optional int64 block_id = 1; + optional string medium_type = 2; +} + +message MoveBlockResponse {} + +message ClearMetricsRequest {} + +message ClearMetricsResponse {} + +message MoveRequest { + repeated Route routes = 1; + optional UfsReadOptions ufs_read_options = 2; + required WriteOptions write_options= 3; +} + +message MoveResponse { + required TaskStatus status = 1; + repeated RouteFailure failures = 2; +} diff --git a/core/transport/src/main/proto/grpc/common.proto b/common/transport/src/main/proto/grpc/common.proto similarity index 79% rename from core/transport/src/main/proto/grpc/common.proto rename to common/transport/src/main/proto/grpc/common.proto index de020d5063ae..b3ae89aeda6d 100644 --- a/core/transport/src/main/proto/grpc/common.proto +++ b/common/transport/src/main/proto/grpc/common.proto @@ -65,6 +65,8 @@ enum MetricType { // METER represents a metric value at a _rate_. The value of the metric varies with the time over which events are // recorded METER = 2; + // HISTOGRAM gives statistics about the value of past occurrences of an event. + HISTOGRAM = 5; // TIMER represents a histogram of the rate of the specified events. TIMER = 3; // EXECUTOR_SERVICE represents an executor service. @@ -78,6 +80,8 @@ enum CommandType { Free = 3; // Ask the worker to free files. Delete = 4; // Ask the worker to delete files. Persist = 5; // Ask the worker to persist a file for lineage + Decommissioned = 6; // Notify the worker that it has been decommissioned + Disabled = 7; // Notify the worker that it has been disabled } message ConfigProperty { @@ -89,6 +93,7 @@ message ConfigProperty { enum TtlAction { DELETE = 0; // Delete the file after TTL expires. FREE = 1; // Free the file after TTL expires. + DELETE_ALLUXIO = 2; // Delete the data and metadata in Alluxio after TTL expires. } message Command { @@ -96,15 +101,6 @@ message Command { repeated int64 data = 2; } -message LocalityTier { - optional string tierName = 1; - optional string value = 2; -} - -message TieredIdentity { - repeated LocalityTier tiers = 1; -} - /** * Address information about masters. */ @@ -122,7 +118,6 @@ message WorkerNetAddress { optional int32 dataPort = 3; optional int32 webPort = 4; optional string domainSocketPath = 5; - optional TieredIdentity tieredIdentity = 6; optional string containerHost = 7; } @@ -144,3 +139,18 @@ enum ErrorType { Internal = 1; External = 2; } + +message BuildVersion { + /** the project version */ + optional string version = 1; + /** the git revision at the time of building */ + optional string revision = 2; +} + +message WorkerIdentity { + /** opaque identifier that uniquely identifies a worker */ + /** exactly how this byte sequence is interpreted depends on the version of its definition */ + optional bytes identifier = 1; + /** version of the definition of the opaque identifier */ + optional int32 version = 2; +} diff --git a/core/transport/src/main/proto/grpc/file_system_master.proto b/common/transport/src/main/proto/grpc/file_system_master.proto similarity index 84% rename from core/transport/src/main/proto/grpc/file_system_master.proto rename to common/transport/src/main/proto/grpc/file_system_master.proto index 211adc60a34f..ed9987976956 100644 --- a/core/transport/src/main/proto/grpc/file_system_master.proto +++ b/common/transport/src/main/proto/grpc/file_system_master.proto @@ -11,11 +11,11 @@ import "grpc/fscommon.proto"; import "proto/journal/file.proto"; enum WritePType { - MUST_CACHE = 1; - TRY_CACHE = 2; + MUST_CACHE = 1 [deprecated = true]; + TRY_CACHE = 2 [deprecated = true]; CACHE_THROUGH = 3; THROUGH = 4; - ASYNC_THROUGH = 5; + ASYNC_THROUGH = 5 [deprecated = true]; NONE = 6; } @@ -82,11 +82,13 @@ message CompleteFilePOptions { optional int64 ufsLength = 1; optional ScheduleAsyncPersistencePOptions asyncPersistOptions = 2; optional FileSystemMasterCommonPOptions commonOptions = 3; + optional string contentHash = 4; } message CompleteFilePRequest { /** the path of the file */ optional string path = 1; optional CompleteFilePOptions options = 2; + optional string uuid = 3; } message OpenFilePOptions { @@ -94,6 +96,11 @@ message OpenFilePOptions { optional int32 maxUfsReadConcurrency = 2; optional FileSystemMasterCommonPOptions commonOptions = 3; optional bool updateLastAccessTime = 4 [default = true]; + // If specified and the blocks are not cached in any worker, + // the data will be read and cached to the certain worker. + // If the blocks have been cached in some alluxio workers, + // this field will be ignored. + optional grpc.WorkerNetAddress ufsReadWorkerLocation = 15; } // XAttrPropagationStrategy controls the behaviour for assigning xAttr @@ -110,10 +117,11 @@ message CreateDirectoryPOptions { optional bool recursive = 1; optional bool allowExists = 2; optional PMode mode = 3; - optional WritePType writeType = 4; + optional WritePType writeType = 4 [default = CACHE_THROUGH]; optional FileSystemMasterCommonPOptions commonOptions = 5; map xattr = 6; optional XAttrPropagationStrategy xattrPropStrat = 7 [default = NEW_PATHS]; + optional bool checkS3BucketPath = 8; } message CreateDirectoryPRequest { /** the path of the directory */ @@ -123,6 +131,8 @@ message CreateDirectoryPRequest { message CreateFilePResponse { optional FileInfo fileInfo = 1; + /** uuid of its open file handle */ + optional string uuid = 2; } message CreateFilePOptions { optional int64 blockSizeBytes = 1; @@ -132,11 +142,17 @@ message CreateFilePOptions { optional int32 replicationMin = 5; optional int32 replicationDurable = 6; optional int32 writeTier = 7; - optional WritePType writeType = 8; + optional WritePType writeType = 8 [default = CACHE_THROUGH]; optional FileSystemMasterCommonPOptions commonOptions = 9; optional int64 persistenceWaitTime = 10; map xattr = 11; optional XAttrPropagationStrategy xattrPropStrat = 12 [default = NEW_PATHS]; + optional bool overwrite = 13; + optional bool checkS3BucketPath = 14; + // If specified, the data will be written to the certain worker + optional grpc.WorkerNetAddress workerLocation = 15; + optional bool isAtomicWrite = 16; + optional bool useMultipartUpload = 17; } message CreateFilePRequest { /** the path of the file */ @@ -150,6 +166,7 @@ message DeletePOptions { optional bool alluxioOnly = 2; optional bool unchecked = 3; optional FileSystemMasterCommonPOptions commonOptions = 4; + optional bool syncParentNextTime = 5; optional bool deleteMountPoint = 6; } message DeletePRequest { @@ -190,6 +207,7 @@ message GetStatusPOptions { optional FileSystemMasterCommonPOptions commonOptions = 2; optional Bits accessMode = 3; optional bool updateTimestamps = 4 [default = true]; + optional bool includeRealContentHash = 5; } message GetStatusPRequest { /** the path of the file or directory */ @@ -208,16 +226,6 @@ enum SyncPointStatus { Initially_Synced = 2; } -message SyncPointInfo { - optional string syncPointUri = 1; - optional SyncPointStatus syncStatus = 2; -} - -message GetSyncPathListPResponse { - repeated SyncPointInfo syncPaths= 1; -} -message GetSyncPathListPRequest {} - message ListStatusPResponse { repeated FileInfo fileInfos = 1; } @@ -229,6 +237,17 @@ message ListStatusPOptions { optional bool recursive = 4; // No data will be transferred. optional bool loadMetadataOnly = 5; + // Setting this to true will disable checking during metadata sync to see if the children + // of a directory has been loaded. This will avoid a costly full traversal of the file + // system during recursive listings, but may result in the children of directories not + // being loaded. It is recommended to set this to true after the first call of a + // recursive partial listing. + optional bool disableAreDescendantsLoadedCheck = 6; + // Mount info will be excluded from the list status response if this field is set to true. + // Resolving a path and obtain the mount info is an expensive operation. + // For clients that do not need this information such as hadoop-compatible clients, + // excluding mount info improves the endpoint performance. + optional bool excludeMountInfo = 7; } message ListStatusPRequest { /** the path of the file or directory */ @@ -372,6 +391,8 @@ message FileInfo { optional int64 lastAccessTimeMs = 31; map xattr = 32; repeated string mediumType = 33; + optional string contentHash = 34; + optional string ufsType = 35; } message GetFilePathPResponse { @@ -442,6 +463,7 @@ message RenamePOptions { optional FileSystemMasterCommonPOptions commonOptions = 1; optional bool persist = 2; optional S3SyntaxOptions s3SyntaxOptions = 3; + optional bool overwrite = 4; } message RenamePRequest { /** the source path of the file or directory */ @@ -457,7 +479,12 @@ message ReverseResolvePRequest { message ReverseResolvePResponse { optional string alluxioPath = 1; } - +message GetMountIdPRequest { + required string ufsUri = 1; +} +message GetMountIdPResponse { + required int64 mountId = 1; +} message SetAttributePResponse {} message SetAttributePOptions { @@ -473,6 +500,7 @@ message SetAttributePOptions { repeated string pinnedMedia = 10; map xattr = 11; optional alluxio.proto.journal.XAttrUpdateStrategy xattrUpdateStrategy = 12; + optional bool directChildrenLoaded = 13; } message SetAttributePRequest { /** the path of the file */ @@ -515,24 +543,6 @@ message ScheduleAsyncPersistencePRequest { optional ScheduleAsyncPersistencePOptions options = 2; } -message StartSyncPResponse {} -message StartSyncPOptions { - optional FileSystemMasterCommonPOptions commonOptions = 1; -} -message StartSyncPRequest { - optional string path = 1; - optional StartSyncPOptions options = 2; -} - -message StopSyncPResponse {} -message StopSyncPOptions { - optional FileSystemMasterCommonPOptions commonOptions = 1; -} -message StopSyncPRequest { - optional string path = 1; - optional StopSyncPOptions options = 2; -} - message UnmountPResponse {} message UnmountPOptions { optional FileSystemMasterCommonPOptions commonOptions = 1; @@ -585,6 +595,74 @@ message NeedsSyncRequest { message NeedsSyncResponse {} +message SubmitJobPRequest{ + optional bytes request_body = 1; +} + +message SubmitJobPResponse { + optional string jobId = 1; +} + +message LoadJobPOptions { + optional int64 bandwidth = 1; + optional bool verify = 2; + optional bool partialListing = 3; + optional bool loadMetadataOnly = 4; + optional bool skipIfExists = 5; + optional string fileFilterRegx = 6; + optional int32 replicas = 7; +} + +message CopyJobPOptions { + optional int64 bandwidth = 1; + optional bool verify = 2; + optional bool partialListing = 3; + optional bool overwrite = 4; + optional WritePType writeType = 5 [default = CACHE_THROUGH]; + optional bool check_content = 6; +} + +message MoveJobPOptions { + optional int64 bandwidth = 1; + optional bool verify = 2; + optional bool partialListing = 3; + optional bool overwrite = 4; + optional WritePType writeType = 5 [default = CACHE_THROUGH]; + optional bool check_content = 6; +} + +message StopJobPRequest { + required JobDescription jobDescription = 1; +} + +message StopJobPResponse { + optional bool jobStopped = 1; +} + +enum JobProgressReportFormat { + TEXT = 1; + JSON = 2; +} + +message JobDescription{ + required string type = 1; + optional string path = 2; +} + +message JobProgressPOptions { + optional JobProgressReportFormat format = 1; + optional bool verbose = 2; +} + +message GetJobProgressPRequest { + required JobDescription jobDescription = 1; + optional JobProgressPOptions options = 2; +} + +message GetJobProgressPResponse { + optional string progressReport = 1; + optional JobProgressReportFormat format = 2; +} /** * This interface contains file system master service endpoints for Alluxio clients. @@ -636,11 +714,6 @@ service FileSystemMasterClientService { */ rpc GetMountTable(GetMountTablePRequest) returns (GetMountTablePResponse); - /** - * Returns a list of paths that are being actively synced by Alluxio - */ - rpc GetSyncPathList(GetSyncPathListPRequest) returns (GetSyncPathListPResponse); - /** * Generates a new block id for the given file. */ @@ -686,6 +759,11 @@ service FileSystemMasterClientService { */ rpc ReverseResolve(ReverseResolvePRequest) returns (ReverseResolvePResponse); + /** + * Reverse resolve a ufs path and get mount id. + */ + rpc GetMountId(GetMountIdPRequest) returns (GetMountIdPResponse); + /** * Schedules async persistence. */ @@ -701,16 +779,6 @@ service FileSystemMasterClientService { */ rpc SetAttribute(SetAttributePRequest) returns (SetAttributePResponse); - /** - * Start the active syncing of the directory or file - */ - rpc StartSync(StartSyncPRequest) returns (StartSyncPResponse); - - /** - * Start the active syncing of the directory or file - */ - rpc StopSync(StopSyncPRequest) returns (StopSyncPResponse); - /** * Deletes an existing "mount point", voiding the Alluxio namespace at the given path. The path * should correspond to an existing mount point. Any files in its subtree that are backed by UFS @@ -731,6 +799,15 @@ service FileSystemMasterClientService { rpc GetStateLockHolders(GetStateLockHoldersPRequest) returns (GetStateLockHoldersPResponse); rpc NeedsSync(NeedsSyncRequest) returns (NeedsSyncResponse); + + /** + * Load a directory into Alluxio. + */ + rpc submitJob(SubmitJobPRequest) returns (SubmitJobPResponse); + + rpc StopJob(StopJobPRequest) returns (StopJobPResponse); + + rpc GetJobProgress(GetJobProgressPRequest) returns (GetJobProgressPResponse); } message FileSystemHeartbeatPResponse { @@ -801,6 +878,11 @@ service FileSystemMasterWorkerService { * Returns the UFS information for the given mount point identified by its id. **/ rpc GetUfsInfo(GetUfsInfoPRequest) returns (GetUfsInfoPResponse); + + /** + * Reverse resolve a ufs path and get mount id. + */ + rpc GetMountId(GetMountIdPRequest) returns (GetMountIdPResponse); } /** diff --git a/core/transport/src/main/proto/grpc/fscommon.proto b/common/transport/src/main/proto/grpc/fscommon.proto similarity index 100% rename from core/transport/src/main/proto/grpc/fscommon.proto rename to common/transport/src/main/proto/grpc/fscommon.proto diff --git a/core/transport/src/main/proto/grpc/job_master.proto b/common/transport/src/main/proto/grpc/job_master.proto similarity index 78% rename from core/transport/src/main/proto/grpc/job_master.proto rename to common/transport/src/main/proto/grpc/job_master.proto index ad5f553cf7de..7f2ee9be8182 100644 --- a/core/transport/src/main/proto/grpc/job_master.proto +++ b/common/transport/src/main/proto/grpc/job_master.proto @@ -107,6 +107,7 @@ message JobWorkerHealth { optional int32 taskPoolSize = 5; optional int32 numActiveTasks = 6; optional int32 unfinishedTasks = 7; + optional grpc.BuildVersion version = 8; } message JobCommand { @@ -197,6 +198,21 @@ message GetAllWorkerHealthPResponse { repeated JobWorkerHealth workerHealths = 1; } +message JobMasterStatus { + optional string state = 1; + optional grpc.NetAddress masterAddress = 2; + optional int64 startTime = 3; + optional grpc.BuildVersion version = 4; +} + +message GetAllMasterStatusPOptions {} +message GetAllMasterStatusPRequest { + optional GetAllMasterStatusPOptions options = 1; +} +message GetAllMasterStatusPResponse { + repeated JobMasterStatus jobMasterStatus = 1; +} + message SubmitOptions {} message SubmitRequest { optional bytes cmdConfig = 1; @@ -275,6 +291,11 @@ service JobMasterClientService { */ rpc GetAllWorkerHealth(GetAllWorkerHealthPRequest) returns (GetAllWorkerHealthPResponse); + /** + * Lists all job master status. + */ + rpc GetAllMasterStatus(GetAllMasterStatusPRequest) returns (GetAllMasterStatusPResponse); + /** * Submit a CMD job, return a jobControlId. */ @@ -305,6 +326,7 @@ message RegisterJobWorkerPOptions {} message RegisterJobWorkerPRequest { optional grpc.WorkerNetAddress workerNetAddress = 1; optional RegisterJobWorkerPOptions options = 2; + optional grpc.BuildVersion version = 3; } message RegisterJobWorkerPResponse { optional int64 id = 1; @@ -325,3 +347,62 @@ service JobMasterWorkerService { */ rpc RegisterJobWorker(RegisterJobWorkerPRequest) returns (RegisterJobWorkerPResponse); } + +message GetJobMasterIdPOptions {} +message GetJobMasterIdPRequest { + optional grpc.NetAddress masterAddress = 1; + optional GetJobMasterIdPOptions options = 2; +} +message GetJobMasterIdPResponse { + optional int64 masterId = 1; +} + +enum JobMasterMetaCommand { + MetaCommand_Unknown = 0; + MetaCommand_Nothing = 1; + MetaCommand_Register = 2; // Ask the standby master to re-register. +} + +message RegisterJobMasterPOptions { + optional int64 startTimeMs = 2; + optional int64 losePrimacyTimeMs = 3; + optional grpc.BuildVersion version = 4; +} + +message RegisterJobMasterPRequest { + optional int64 jobMasterId = 1; + optional RegisterJobMasterPOptions options = 2; +} +message RegisterJobMasterPResponse {} + +message JobMasterHeartbeatPOptions { +} +message JobMasterHeartbeatPRequest { + optional int64 masterId = 1; + optional JobMasterHeartbeatPOptions options = 2; +} +message JobMasterHeartbeatPResponse { + optional JobMasterMetaCommand command = 1; +} + +/** + * This interface contains meta master service endpoints for Alluxio standby masters. + */ +service JobMasterMasterService { + + /** + * Returns a master id for the given master address. + */ + rpc GetMasterId(GetJobMasterIdPRequest) returns (GetJobMasterIdPResponse); + + /** + * Registers a master. + */ + rpc RegisterMaster(RegisterJobMasterPRequest) returns (RegisterJobMasterPResponse); + + /** + * Heartbeats to indicate the master is lost or not. + */ + rpc MasterHeartbeat(JobMasterHeartbeatPRequest) returns (JobMasterHeartbeatPResponse); +} + diff --git a/core/transport/src/main/proto/grpc/journal_master.proto b/common/transport/src/main/proto/grpc/journal_master.proto similarity index 100% rename from core/transport/src/main/proto/grpc/journal_master.proto rename to common/transport/src/main/proto/grpc/journal_master.proto diff --git a/core/transport/src/main/proto/grpc/messaging_transport.proto b/common/transport/src/main/proto/grpc/messaging_transport.proto similarity index 100% rename from core/transport/src/main/proto/grpc/messaging_transport.proto rename to common/transport/src/main/proto/grpc/messaging_transport.proto diff --git a/core/transport/src/main/proto/grpc/meta_master.proto b/common/transport/src/main/proto/grpc/meta_master.proto similarity index 80% rename from core/transport/src/main/proto/grpc/meta_master.proto rename to common/transport/src/main/proto/grpc/meta_master.proto index c21022e66378..f896b34f6d5e 100644 --- a/core/transport/src/main/proto/grpc/meta_master.proto +++ b/common/transport/src/main/proto/grpc/meta_master.proto @@ -22,6 +22,8 @@ message GetConfigurationPResponse{ map pathConfigs = 2; optional string clusterConfigHash = 3; optional string pathConfigHash = 4; + optional int64 clusterConfigLastUpdateTime = 5; + optional int64 pathConfigLastUpdateTime = 6; } enum ConfigStatus { @@ -77,6 +79,13 @@ message MasterInfo { optional string clusterId = 11; optional bool raftJournal = 12; repeated string raftAddress = 13; + repeated MasterVersion masterVersions = 14; +} + +message MasterVersion { + optional grpc.NetAddress addresses = 1; + optional string version = 2; + optional string state = 3; } enum MasterInfoField { @@ -93,6 +102,7 @@ enum MasterInfoField { CLUSTER_ID = 10; RAFT_JOURNAL = 11; RAFT_ADDRESSES = 12; + MASTER_VERSION = 13; } message GetMasterInfoPOptions { @@ -140,6 +150,25 @@ message BackupStatusPRequest { optional string backupId = 1; } +message ProxyStatus { + optional grpc.NetAddress address = 1; + optional string state = 2; + optional int64 startTime = 3; + optional int64 lastHeartbeatTime = 4; + optional grpc.BuildVersion version = 5; +} + +message ListProxyStatusPRequest { + optional ListProxyStatusPOptions options = 1; +} + +message ListProxyStatusPOptions { +} + +message ListProxyStatusPResponse { + repeated ProxyStatus proxyStatuses = 1; +} + /** * This interface contains meta master service endpoints for Alluxio clients. */ @@ -168,6 +197,11 @@ service MetaMasterClientService { * Creates a checkpoint in the primary master journal system. */ rpc Checkpoint(CheckpointPOptions) returns (CheckpointPResponse); + + /** + * Returns the status of all known Proxy instances in the cluster. + */ + rpc ListProxyStatus(ListProxyStatusPRequest) returns (ListProxyStatusPResponse); } message SetPathConfigurationPOptions {} @@ -237,6 +271,10 @@ enum MetaCommand { message RegisterMasterPOptions { repeated grpc.ConfigProperty configs = 1; + optional int64 startTimeMs = 2; + optional int64 losePrimacyTimeMs = 3; + optional string version = 4; + optional string revision = 5; } message RegisterMasterPRequest { optional int64 masterId = 1; @@ -244,7 +282,10 @@ message RegisterMasterPRequest { } message RegisterMasterPResponse {} -message MasterHeartbeatPOptions {} +message MasterHeartbeatPOptions { + optional int64 lastCheckpointTime = 1; + optional int64 journalEntriesSinceCheckpoint = 2; +} message MasterHeartbeatPRequest { optional int64 masterId = 1; optional MasterHeartbeatPOptions options = 2; @@ -281,3 +322,24 @@ service MetaMasterMasterService { */ rpc MasterHeartbeat(MasterHeartbeatPRequest) returns (MasterHeartbeatPResponse); } + +message ProxyHeartbeatPOptions { + optional grpc.NetAddress proxyAddress = 1; + optional int64 startTime = 2; + optional grpc.BuildVersion version = 3; +} +message ProxyHeartbeatPRequest { + optional ProxyHeartbeatPOptions options = 1; +} +message ProxyHeartbeatPResponse { +} + +/** + * This interface contains meta master service endpoints for Alluxio Proxy instances. + */ +service MetaMasterProxyService { + /** + * Stateless heartbeat from proxy instances to report the current status. + */ + rpc ProxyHeartbeat(ProxyHeartbeatPRequest) returns (ProxyHeartbeatPResponse); +} diff --git a/core/transport/src/main/proto/grpc/metric_master.proto b/common/transport/src/main/proto/grpc/metric_master.proto similarity index 100% rename from core/transport/src/main/proto/grpc/metric_master.proto rename to common/transport/src/main/proto/grpc/metric_master.proto diff --git a/common/transport/src/main/proto/grpc/policy_master.proto b/common/transport/src/main/proto/grpc/policy_master.proto new file mode 100644 index 000000000000..9d817b95f4b6 --- /dev/null +++ b/common/transport/src/main/proto/grpc/policy_master.proto @@ -0,0 +1,141 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_package = "alluxio.grpc"; +option java_outer_classname = "PolicyMasterProto"; + +package alluxio.grpc.policy; + +// next available id: 5 +message PolicyInfo { + optional int64 id = 1; + optional string name = 2; + optional int64 created_at = 3; + optional AddPolicyPOptions options = 4; +} + +message ListPolicyPResponse { + repeated PolicyInfo policy = 1; +} +message ListPolicyPOptions {} +message ListPolicyPRequest { + optional ListPolicyPOptions options = 1; +} + +message AddPolicyPResponse { + optional PolicyInfo policy = 1; +} +message AddPolicyPOptions { + optional string operation = 1; + optional string srcPath = 2; + optional string dstPath = 3; + optional string timeExpression = 4; + optional string filter = 5; + optional string file_date_pattern = 6; +} +message AddPolicyPRequest { + optional string policyName = 1; + optional AddPolicyPOptions options = 2; +} + +message RemovePolicyPResponse {} +message RemovePolicyPOptions {} +message RemovePolicyPRequest { + optional string policyName = 1; + optional RemovePolicyPOptions options = 2; +} + +// next available id: 12 +message PolicyStatus { + // scan status + optional int64 scanStartedMs = 1; + optional int64 scanTotal = 2; + optional int64 scanPendingAction = 3; + optional int64 scanCompleted = 4; + optional bool isScanning = 11; + // metrics + optional int64 actionsScheduled = 5; + optional int64 actionsRunning = 6; + optional int64 actionsSucceeded = 7; + optional int64 actionsCanceled = 8; + optional int64 actionsFailed = 9; + map errors = 10; +} + + + +// next available id: 7 +message ActionInfo { + optional string actionId = 1; + optional string actionType = 2; + optional string actionStatus = 3; + optional string filePath = 4; + optional int64 lastUpdatedTime = 5; + optional string errorType = 6; + optional string errorMessage = 7; +} + +message GetActionInfoResponse { + repeated ActionInfo infos = 1; +} + +message GetActionInfoOptions {} + +message GetActionInfoRequest { + optional string policyName = 1; + optional string actionId = 2; + optional string filePath = 3; + optional GetPolicyStatusPOptions options = 4; +} + + +message GetPolicyStatusPResponse { + optional PolicyStatus status = 1; +} + +message GetPolicyStatusPOptions {} +message GetPolicyStatusPRequest { + optional string policyName = 1; + optional GetPolicyStatusPOptions options = 2; +} + +message TriggerPolicyRequest { + optional string policyName = 1; +} + +message TriggerPolicyResponse {} + +/** + * This interface contains policy master service endpoints for Alluxio clients. + */ +service PolicyMasterClientService { + /** + * Returns the list of policies + */ + rpc ListPolicy(ListPolicyPRequest) returns (ListPolicyPResponse); + + /** + * Adds a new policy definition + */ + rpc AddPolicy(AddPolicyPRequest) returns (AddPolicyPResponse); + + /** + * Removes a policy definition + */ + rpc RemovePolicy(RemovePolicyPRequest) returns (RemovePolicyPResponse); + + /** + * Gets the status of a policy + */ + rpc GetPolicyStatus(GetPolicyStatusPRequest) returns (GetPolicyStatusPResponse); + + /** + * Gets the information of a action + */ + rpc GetActionInfo(GetActionInfoRequest) returns (GetActionInfoResponse); + + /** + * Trigger policy scan for all policies + */ + rpc TriggerPolicy(TriggerPolicyRequest) returns (TriggerPolicyResponse); +} diff --git a/common/transport/src/main/proto/grpc/raft_journal.proto b/common/transport/src/main/proto/grpc/raft_journal.proto new file mode 100644 index 000000000000..dee7f92cddbe --- /dev/null +++ b/common/transport/src/main/proto/grpc/raft_journal.proto @@ -0,0 +1,104 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_package = "alluxio.grpc"; +option java_outer_classname = "RaftJournalProto"; + +package alluxio.grpc.meta; + +import "grpc/common.proto"; + +message JournalQueryRequest { + optional GetSnapshotInfoRequest snapshotInfoRequest = 1 [deprecated = true]; + optional GetSnapshotRequest snapshotRequest = 2 [deprecated = true]; + optional AddQuorumServerRequest addQuorumServerRequest = 3; +} + +message JournalQueryResponse { + option deprecated = true; + optional GetSnapshotInfoResponse snapshotInfoResponse = 1 [deprecated = true]; +} + +message AddQuorumServerRequest { + optional NetAddress serverAddress = 1; +} + +message GetSnapshotInfoRequest { + option deprecated = true; + optional SnapshotMetadata snapshotInfo = 1 [deprecated = true]; +} + +message GetSnapshotInfoResponse { + option deprecated = true; + optional SnapshotMetadata latest = 1 [deprecated = true]; +} + +message GetSnapshotRequest { + option deprecated = true; +} + +message SnapshotMetadata { + optional int64 snapshotTerm = 1; + optional int64 snapshotIndex = 2; + optional bool exists = 3; +} + +message SnapshotData { + optional int64 snapshotTerm = 1; + optional int64 snapshotIndex = 2; + optional bytes chunk = 3; + optional int64 offset = 4 [deprecated = true]; + optional bool eof = 5 [deprecated = true]; +} + +message UploadSnapshotPRequest { + option deprecated = true; + optional SnapshotData data = 1 [deprecated = true]; +} + +message UploadSnapshotPResponse { + option deprecated = true; + optional int64 offsetReceived = 1 [deprecated = true]; +} + +message DownloadSnapshotPRequest { + option deprecated = true; + optional int64 offsetReceived = 1 [deprecated = true]; +} + +message DownloadSnapshotPResponse { + option deprecated = true; + optional SnapshotData data = 1 [deprecated = true]; +} + +message LatestSnapshotInfoPRequest {} + +/** + * This interface contains raft service endpoints for Alluxio masters. + */ +service RaftJournalService { + + /** + * Uploads a snapshot to primary master. + */ + rpc UploadSnapshot (stream UploadSnapshotPRequest) returns (stream UploadSnapshotPResponse) { + option deprecated = true; + }; + + /** + * Downloads a snapshot from primary master. + */ + rpc DownloadSnapshot (stream DownloadSnapshotPRequest) returns (stream DownloadSnapshotPResponse) { + option deprecated = true; + }; + + /** + * Requests information about snapshots on a particular machine. + */ + rpc RequestLatestSnapshotInfo(LatestSnapshotInfoPRequest) returns (SnapshotMetadata) {} + + /** + * Request to download the snapshot information from a particular machine. + */ + rpc RequestLatestSnapshotData(SnapshotMetadata) returns (stream SnapshotData) {} +} diff --git a/core/transport/src/main/proto/grpc/sasl_server.proto b/common/transport/src/main/proto/grpc/sasl_server.proto similarity index 100% rename from core/transport/src/main/proto/grpc/sasl_server.proto rename to common/transport/src/main/proto/grpc/sasl_server.proto diff --git a/core/transport/src/main/proto/grpc/table/layout/hive/hive.proto b/common/transport/src/main/proto/grpc/table/layout/hive/hive.proto similarity index 91% rename from core/transport/src/main/proto/grpc/table/layout/hive/hive.proto rename to common/transport/src/main/proto/grpc/table/layout/hive/hive.proto index 538fbe02a0e0..8faf0c73b88b 100644 --- a/core/transport/src/main/proto/grpc/table/layout/hive/hive.proto +++ b/common/transport/src/main/proto/grpc/table/layout/hive/hive.proto @@ -6,9 +6,10 @@ option java_outer_classname = "HiveLayoutProto"; package alluxio.grpc.table.layout; -import "grpc/common.proto"; import "grpc/table/table_master.proto"; +// TODO(binfan): remove this proto file. +// it is no longer used but only to keep old Presto/Trino build with Alluxio 3.x client. message StorageFormat { optional string serde = 1; optional string input_format = 2; diff --git a/core/transport/src/main/proto/grpc/table/table_master.proto b/common/transport/src/main/proto/grpc/table/table_master.proto similarity index 98% rename from core/transport/src/main/proto/grpc/table/table_master.proto rename to common/transport/src/main/proto/grpc/table/table_master.proto index e607311e7709..a3d78f42c860 100644 --- a/core/transport/src/main/proto/grpc/table/table_master.proto +++ b/common/transport/src/main/proto/grpc/table/table_master.proto @@ -6,9 +6,10 @@ option java_outer_classname = "TableMasterProto"; package alluxio.grpc.table; -import "grpc/common.proto"; import "grpc/job_master.proto"; +// TODO(binfan): remove this proto file. +// it is no longer used but only to keep old Presto/Trino build with Alluxio 3.x client. message FieldSchema { optional uint32 id = 1; optional string name = 2; diff --git a/common/transport/src/main/proto/grpc/version.proto b/common/transport/src/main/proto/grpc/version.proto new file mode 100644 index 000000000000..27f8f55ec8a5 --- /dev/null +++ b/common/transport/src/main/proto/grpc/version.proto @@ -0,0 +1,77 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_package = "alluxio.grpc"; +option java_outer_classname = "VersionProto"; + +package alluxio.grpc.version; + + +enum ServiceType { + // FILE_SYSTEM_WORKER_WORKER_SERVICE is replaced by BLOCK_WORKER_CLIENT_SERVICE + // as a clearer type name + reserved 12; + reserved "FILE_SYSTEM_WORKER_WORKER_SERVICE"; + + UNKNOWN_SERVICE = 0; + FILE_SYSTEM_MASTER_CLIENT_SERVICE = 1; + FILE_SYSTEM_MASTER_WORKER_SERVICE = 2; + FILE_SYSTEM_MASTER_JOB_SERVICE = 3; + BLOCK_MASTER_CLIENT_SERVICE = 4; + BLOCK_MASTER_WORKER_SERVICE = 5; + META_MASTER_CONFIG_SERVICE = 6; + META_MASTER_CLIENT_SERVICE = 7; + META_MASTER_MASTER_SERVICE = 8; + META_MASTER_PROXY_SERVICE = 18; + METRICS_MASTER_CLIENT_SERVICE = 9; + JOB_MASTER_CLIENT_SERVICE = 10; + JOB_MASTER_WORKER_SERVICE = 11; + JOB_MASTER_MASTER_SERVICE = 19; + JOURNAL_MASTER_CLIENT_SERVICE = 13; + TABLE_MASTER_CLIENT_SERVICE = 14; + META_MASTER_BACKUP_MESSAGING_SERVICE = 15; + RAFT_JOURNAL_SERVICE = 16; + BLOCK_WORKER_CLIENT_SERVICE = 17; + PRIVILEGE_MASTER_CLIENT_SERVICE = 1001; + POLICY_MASTER_CLIENT_SERVICE = 1002; + ENCRYPTION_MASTER_CLIENT_SERVICE = 1003; + CROSS_CLUSTER_MASTER_CLIENT_SERVICE = 1004; + SECURITY_SERVER_CLIENT_SERVICE = 1005; +} + +message GetServiceVersionPRequest { + optional ServiceType serviceType = 1; + // The purpose of this field is to make grpc service on standby masters work without + // making client changes and keeps backwards compatibility. + // This requests to this endpoint will be rejected on standby masters by default, + // unless this field is set. + // Two places use this request: + // 1. PollingMasterInquireClient uses this endpoint to tell who is the primary master. + // 2. AbstractClient uses this endpoint to verify the version before it RPCs with the master. + // + // Behaviors: + // 1. old clients -> new cluster standby masters + // PollingMasterInquireClient does not set this field and is able to tell which one is primary master because + // the request will be rejected on the standby master. + // AbstractClient does not set this field. + // Old clients only connects to primary so this doesn't break the existing behavior. + // + // 2. new clients -> new cluster standby masters + // PollingMasterInquireClient does not set this field and is able to tell which one is primary master because + // the request will be rejected on the standby master. + // AbstractClient sets this field to true. Rpcs to standby masters can go through and pass the version verification. + + optional bool allowedOnStandbyMasters = 2; +} +message GetServiceVersionPResponse { + optional int64 version = 1; +} + +service ServiceVersionClientService { + + /** + * Returns the version of the master service. + * NOTE: The version should be updated every time a backwards incompatible API change occurs. + */ + rpc getServiceVersion(GetServiceVersionPRequest) returns (GetServiceVersionPResponse); +} diff --git a/core/transport/src/main/proto/proto.lock b/common/transport/src/main/proto/proto.lock similarity index 87% rename from core/transport/src/main/proto/proto.lock rename to common/transport/src/main/proto/proto.lock index 03c42fd97bd6..0160492985c9 100644 --- a/core/transport/src/main/proto/proto.lock +++ b/common/transport/src/main/proto/proto.lock @@ -365,17 +365,22 @@ ] }, { - "name": "RemoveDecommissionedWorkerPOptions", + "name": "RemoveDisabledWorkerPOptions", "fields": [ { "id": 1, - "name": "workerName", + "name": "workerHostname", "type": "string" + }, + { + "id": 2, + "name": "workerWebPort", + "type": "int64" } ] }, { - "name": "RemoveDecommissionedWorkerPResponse" + "name": "RemoveDisabledWorkerPResponse" }, { "name": "GetWorkerLostStoragePOptions" @@ -391,6 +396,29 @@ } ] }, + { + "name": "DecommissionWorkerPResponse" + }, + { + "name": "DecommissionWorkerPOptions", + "fields": [ + { + "id": 1, + "name": "workerHostname", + "type": "string" + }, + { + "id": 2, + "name": "workerWebPort", + "type": "int64" + }, + { + "id": 3, + "name": "canRegisterAgain", + "type": "bool" + } + ] + }, { "name": "TierList", "fields": [ @@ -595,6 +623,32 @@ { "name": "CommitBlockInUfsPResponse" }, + { + "name": "NotifyWorkerIdPOptions" + }, + { + "name": "NotifyWorkerIdPRequest", + "fields": [ + { + "id": 1, + "name": "workerId", + "type": "int64" + }, + { + "id": 2, + "name": "workerNetAddress", + "type": "grpc.WorkerNetAddress" + }, + { + "id": 3, + "name": "options", + "type": "NotifyWorkerIdPOptions" + } + ] + }, + { + "name": "NotifyWorkerIdPResponse" + }, { "name": "GetWorkerIdPOptions" }, @@ -666,21 +720,6 @@ } ] }, - { - "name": "BuildVersion", - "fields": [ - { - "id": 1, - "name": "version", - "type": "string" - }, - { - "id": 2, - "name": "revision", - "type": "string" - } - ] - }, { "name": "RegisterWorkerPOptions", "fields": [ @@ -792,9 +831,9 @@ "out_type": "GetWorkerInfoListPResponse" }, { - "name": "RemoveDecommissionedWorker", - "in_type": "RemoveDecommissionedWorkerPOptions", - "out_type": "RemoveDecommissionedWorkerPResponse" + "name": "RemoveDisabledWorker", + "in_type": "RemoveDisabledWorkerPOptions", + "out_type": "RemoveDisabledWorkerPResponse" }, { "name": "GetWorkerReport", @@ -805,6 +844,11 @@ "name": "GetWorkerLostStorage", "in_type": "GetWorkerLostStoragePOptions", "out_type": "GetWorkerLostStoragePResponse" + }, + { + "name": "DecommissionWorker", + "in_type": "DecommissionWorkerPOptions", + "out_type": "DecommissionWorkerPResponse" } ] }, @@ -831,6 +875,11 @@ "in_type": "GetWorkerIdPRequest", "out_type": "GetWorkerIdPResponse" }, + { + "name": "NotifyWorkerId", + "in_type": "NotifyWorkerIdPRequest", + "out_type": "NotifyWorkerIdPResponse" + }, { "name": "RegisterWorker", "in_type": "RegisterWorkerPRequest", @@ -1061,6 +1110,11 @@ "id": 1, "name": "offset", "type": "int64" + }, + { + "id": 2, + "name": "contentHash", + "type": "string" } ] }, @@ -1162,6 +1216,11 @@ "id": 3, "name": "bandwidth", "type": "int64" + }, + { + "id": 4, + "name": "user", + "type": "string" } ] }, @@ -1211,6 +1270,53 @@ } ] }, + { + "name": "LoadFileRequest", + "fields": [ + { + "id": 1, + "name": "files", + "type": "File", + "is_repeated": true + } + ] + }, + { + "name": "File", + "fields": [ + { + "id": 1, + "name": "ufs_path", + "type": "string" + }, + { + "id": 2, + "name": "length", + "type": "int64" + }, + { + "id": 3, + "name": "mountId", + "type": "int64" + } + ] + }, + { + "name": "LoadFileResponse", + "fields": [ + { + "id": 1, + "name": "status", + "type": "TaskStatus" + }, + { + "id": 2, + "name": "files", + "type": "FileFailure", + "is_repeated": true + } + ] + }, { "name": "FreeWorkerRequest" }, @@ -1242,12 +1348,154 @@ } ] }, + { + "name": "FileFailure", + "fields": [ + { + "id": 1, + "name": "file", + "type": "File" + }, + { + "id": 2, + "name": "code", + "type": "int32" + }, + { + "id": 3, + "name": "message", + "type": "string" + }, + { + "id": 4, + "name": "retryable", + "type": "bool" + } + ] + }, { "name": "AsyncCacheResponse" }, { "name": "CacheResponse" }, + { + "name": "CopyRequest", + "fields": [ + { + "id": 1, + "name": "routes", + "type": "Route", + "is_repeated": true + }, + { + "id": 2, + "name": "ufs_read_options", + "type": "UfsReadOptions" + }, + { + "id": 3, + "name": "write_options", + "type": "WriteOptions" + }, + { + "id": 4, + "name": "ufs_only", + "type": "bool" + } + ] + }, + { + "name": "Route", + "fields": [ + { + "id": 1, + "name": "src", + "type": "string" + }, + { + "id": 2, + "name": "dst", + "type": "string" + }, + { + "id": 3, + "name": "length", + "type": "int64" + }, + { + "id": 4, + "name": "src_ufs_address", + "type": "string" + }, + { + "id": 5, + "name": "dst_ufs_address", + "type": "string" + } + ] + }, + { + "name": "CopyResponse", + "fields": [ + { + "id": 1, + "name": "status", + "type": "TaskStatus" + }, + { + "id": 2, + "name": "file_status", + "type": "FileStatus", + "is_repeated": true + } + ] + }, + { + "name": "WriteOptions", + "fields": [ + { + "id": 1, + "name": "overwrite", + "type": "bool" + }, + { + "id": 2, + "name": "write_type", + "type": "grpc.file.WritePType" + }, + { + "id": 3, + "name": "check_content", + "type": "bool" + } + ] + }, + { + "name": "FileStatus", + "fields": [ + { + "id": 1, + "name": "route", + "type": "Route" + }, + { + "id": 2, + "name": "code", + "type": "int32" + }, + { + "id": 3, + "name": "message", + "type": "string" + }, + { + "id": 4, + "name": "retryable", + "type": "bool" + } + ] + }, { "name": "OpenLocalBlockRequest", "fields": [ @@ -1365,6 +1613,17 @@ { "name": "BlockWorker", "rpcs": [ + { + "name": "GetStatus", + "in_type": "grpc.file.GetStatusPRequest", + "out_type": "grpc.file.GetStatusPResponse" + }, + { + "name": "ListStatus", + "in_type": "grpc.file.ListStatusPRequest", + "out_type": "grpc.file.ListStatusPResponse", + "out_streamed": true + }, { "name": "ReadBlock", "in_type": "ReadRequest", @@ -1408,6 +1667,16 @@ "in_type": "LoadRequest", "out_type": "LoadResponse" }, + { + "name": "LoadFile", + "in_type": "LoadFileRequest", + "out_type": "LoadFileResponse" + }, + { + "name": "Copy", + "in_type": "CopyRequest", + "out_type": "CopyResponse" + }, { "name": "RemoveBlock", "in_type": "RemoveBlockRequest", @@ -1437,6 +1706,9 @@ }, { "path": "grpc/common.proto" + }, + { + "path": "grpc/file_system_master.proto" } ], "package": { @@ -1513,6 +1785,10 @@ "name": "METER", "integer": 2 }, + { + "name": "HISTOGRAM", + "integer": 5 + }, { "name": "TIMER", "integer": 3 @@ -1548,6 +1824,14 @@ { "name": "Persist", "integer": 5 + }, + { + "name": "Decommissioned", + "integer": 6 + }, + { + "name": "Disabled", + "integer": 7 } ] }, @@ -1560,6 +1844,10 @@ { "name": "FREE", "integer": 1 + }, + { + "name": "DELETE_ALLUXIO", + "integer": 2 } ] }, @@ -1724,37 +2012,11 @@ ] }, { - "name": "LocalityTier", + "name": "NetAddress", "fields": [ { "id": 1, - "name": "tierName", - "type": "string" - }, - { - "id": 2, - "name": "value", - "type": "string" - } - ] - }, - { - "name": "TieredIdentity", - "fields": [ - { - "id": 1, - "name": "tiers", - "type": "LocalityTier", - "is_repeated": true - } - ] - }, - { - "name": "NetAddress", - "fields": [ - { - "id": 1, - "name": "host", + "name": "host", "type": "string" }, { @@ -1838,6 +2100,21 @@ "type": "ErrorType" } ] + }, + { + "name": "BuildVersion", + "fields": [ + { + "id": 1, + "name": "version", + "type": "string" + }, + { + "id": 2, + "name": "revision", + "type": "string" + } + ] } ], "package": { @@ -2038,6 +2315,19 @@ "integer": 3 } ] + }, + { + "name": "JobProgressReportFormat", + "enum_fields": [ + { + "name": "TEXT", + "integer": 1 + }, + { + "name": "JSON", + "integer": 2 + } + ] } ], "messages": [ @@ -2195,6 +2485,11 @@ "id": 3, "name": "commonOptions", "type": "FileSystemMasterCommonPOptions" + }, + { + "id": 4, + "name": "contentHash", + "type": "string" } ] }, @@ -2241,6 +2536,11 @@ "value": "true" } ] + }, + { + "id": 15, + "name": "ufsReadWorkerLocation", + "type": "grpc.WorkerNetAddress" } ] }, @@ -2285,6 +2585,11 @@ "value": "NEW_PATHS" } ] + }, + { + "id": 8, + "name": "checkS3BucketPath", + "type": "bool" } ], "maps": [ @@ -2386,6 +2691,21 @@ "value": "NEW_PATHS" } ] + }, + { + "id": 13, + "name": "overwrite", + "type": "bool" + }, + { + "id": 14, + "name": "checkS3BucketPath", + "type": "bool" + }, + { + "id": 15, + "name": "workerLocation", + "type": "grpc.WorkerNetAddress" } ], "maps": [ @@ -2440,6 +2760,11 @@ "name": "commonOptions", "type": "FileSystemMasterCommonPOptions" }, + { + "id": 5, + "name": "syncParentNextTime", + "type": "bool" + }, { "id": 6, "name": "deleteMountPoint", @@ -2573,6 +2898,11 @@ "value": "true" } ] + }, + { + "id": 5, + "name": "includeRealContentHash", + "type": "bool" } ] }, @@ -2673,6 +3003,16 @@ "id": 5, "name": "loadMetadataOnly", "type": "bool" + }, + { + "id": 6, + "name": "disableAreDescendantsLoadedCheck", + "type": "bool" + }, + { + "id": 7, + "name": "excludeMountInfo", + "type": "bool" } ] }, @@ -3269,6 +3609,21 @@ } ] }, + { + "name": "S3SyntaxOptions", + "fields": [ + { + "id": 1, + "name": "overwrite", + "type": "bool" + }, + { + "id": 2, + "name": "isMultipartUpload", + "type": "bool" + } + ] + }, { "name": "RenamePResponse" }, @@ -3284,6 +3639,11 @@ "id": 2, "name": "persist", "type": "bool" + }, + { + "id": 3, + "name": "s3SyntaxOptions", + "type": "S3SyntaxOptions" } ] }, @@ -3675,6 +4035,161 @@ { "name": "NeedsSyncResponse" }, + { + "name": "SubmitJobPRequest", + "fields": [ + { + "id": 1, + "name": "request_body", + "type": "bytes" + } + ] + }, + { + "name": "SubmitJobPResponse", + "fields": [ + { + "id": 1, + "name": "jobId", + "type": "string" + } + ] + }, + { + "name": "LoadJobPOptions", + "fields": [ + { + "id": 1, + "name": "bandwidth", + "type": "int64" + }, + { + "id": 2, + "name": "verify", + "type": "bool" + }, + { + "id": 3, + "name": "partialListing", + "type": "bool" + } + ] + }, + { + "name": "CopyJobPOptions", + "fields": [ + { + "id": 1, + "name": "bandwidth", + "type": "int64" + }, + { + "id": 2, + "name": "verify", + "type": "bool" + }, + { + "id": 3, + "name": "partialListing", + "type": "bool" + }, + { + "id": 4, + "name": "overwrite", + "type": "bool" + }, + { + "id": 5, + "name": "writeType", + "type": "WritePType" + }, + { + "id": 6, + "name": "ufs_only", + "type": "bool" + } + ] + }, + { + "name": "StopJobPRequest", + "fields": [ + { + "id": 1, + "name": "jobDescription", + "type": "JobDescription" + } + ] + }, + { + "name": "StopJobPResponse", + "fields": [ + { + "id": 1, + "name": "jobStopped", + "type": "bool" + } + ] + }, + { + "name": "JobDescription", + "fields": [ + { + "id": 1, + "name": "type", + "type": "string" + }, + { + "id": 2, + "name": "path", + "type": "string" + } + ] + }, + { + "name": "JobProgressPOptions", + "fields": [ + { + "id": 1, + "name": "format", + "type": "JobProgressReportFormat" + }, + { + "id": 2, + "name": "verbose", + "type": "bool" + } + ] + }, + { + "name": "GetJobProgressPRequest", + "fields": [ + { + "id": 1, + "name": "jobDescription", + "type": "JobDescription" + }, + { + "id": 2, + "name": "options", + "type": "JobProgressPOptions" + } + ] + }, + { + "name": "GetJobProgressPResponse", + "fields": [ + { + "id": 1, + "name": "progressReport", + "type": "string" + }, + { + "id": 2, + "name": "format", + "type": "JobProgressReportFormat" + } + ] + }, { "name": "FileSystemHeartbeatPResponse", "fields": [ @@ -3942,6 +4457,21 @@ "name": "NeedsSync", "in_type": "NeedsSyncRequest", "out_type": "NeedsSyncResponse" + }, + { + "name": "submitJob", + "in_type": "SubmitJobPRequest", + "out_type": "SubmitJobPResponse" + }, + { + "name": "StopJob", + "in_type": "StopJobPRequest", + "out_type": "StopJobPResponse" + }, + { + "name": "GetJobProgress", + "in_type": "GetJobProgressPRequest", + "out_type": "GetJobProgressPResponse" } ] }, @@ -4141,6 +4671,22 @@ "integer": 3 } ] + }, + { + "name": "JobMasterMetaCommand", + "enum_fields": [ + { + "name": "MetaCommand_Unknown" + }, + { + "name": "MetaCommand_Nothing", + "integer": 1 + }, + { + "name": "MetaCommand_Register", + "integer": 2 + } + ] } ], "messages": [ @@ -4413,6 +4959,11 @@ "id": 7, "name": "unfinishedTasks", "type": "int32" + }, + { + "id": 8, + "name": "version", + "type": "grpc.BuildVersion" } ] }, @@ -4694,6 +5245,55 @@ } ] }, + { + "name": "JobMasterStatus", + "fields": [ + { + "id": 1, + "name": "state", + "type": "string" + }, + { + "id": 2, + "name": "masterAddress", + "type": "grpc.NetAddress" + }, + { + "id": 3, + "name": "startTime", + "type": "int64" + }, + { + "id": 4, + "name": "version", + "type": "grpc.BuildVersion" + } + ] + }, + { + "name": "GetAllMasterStatusPOptions" + }, + { + "name": "GetAllMasterStatusPRequest", + "fields": [ + { + "id": 1, + "name": "options", + "type": "GetAllMasterStatusPOptions" + } + ] + }, + { + "name": "GetAllMasterStatusPResponse", + "fields": [ + { + "id": 1, + "name": "jobMasterStatus", + "type": "JobMasterStatus", + "is_repeated": true + } + ] + }, { "name": "SubmitOptions" }, @@ -4863,27 +5463,126 @@ "name": "RegisterJobWorkerPOptions" }, { - "name": "RegisterJobWorkerPRequest", + "name": "RegisterJobWorkerPRequest", + "fields": [ + { + "id": 1, + "name": "workerNetAddress", + "type": "grpc.WorkerNetAddress" + }, + { + "id": 2, + "name": "options", + "type": "RegisterJobWorkerPOptions" + }, + { + "id": 3, + "name": "version", + "type": "grpc.BuildVersion" + } + ] + }, + { + "name": "RegisterJobWorkerPResponse", + "fields": [ + { + "id": 1, + "name": "id", + "type": "int64" + } + ] + }, + { + "name": "GetJobMasterIdPOptions" + }, + { + "name": "GetJobMasterIdPRequest", + "fields": [ + { + "id": 1, + "name": "masterAddress", + "type": "grpc.NetAddress" + }, + { + "id": 2, + "name": "options", + "type": "GetJobMasterIdPOptions" + } + ] + }, + { + "name": "GetJobMasterIdPResponse", + "fields": [ + { + "id": 1, + "name": "masterId", + "type": "int64" + } + ] + }, + { + "name": "RegisterJobMasterPOptions", + "fields": [ + { + "id": 2, + "name": "startTimeMs", + "type": "int64" + }, + { + "id": 3, + "name": "losePrimacyTimeMs", + "type": "int64" + }, + { + "id": 4, + "name": "version", + "type": "grpc.BuildVersion" + } + ] + }, + { + "name": "RegisterJobMasterPRequest", + "fields": [ + { + "id": 1, + "name": "jobMasterId", + "type": "int64" + }, + { + "id": 2, + "name": "options", + "type": "RegisterJobMasterPOptions" + } + ] + }, + { + "name": "RegisterJobMasterPResponse" + }, + { + "name": "JobMasterHeartbeatPOptions" + }, + { + "name": "JobMasterHeartbeatPRequest", "fields": [ { "id": 1, - "name": "workerNetAddress", - "type": "grpc.WorkerNetAddress" + "name": "masterId", + "type": "int64" }, { "id": 2, "name": "options", - "type": "RegisterJobWorkerPOptions" + "type": "JobMasterHeartbeatPOptions" } ] }, { - "name": "RegisterJobWorkerPResponse", + "name": "JobMasterHeartbeatPResponse", "fields": [ { "id": 1, - "name": "id", - "type": "int64" + "name": "command", + "type": "JobMasterMetaCommand" } ] } @@ -4927,6 +5626,11 @@ "in_type": "GetAllWorkerHealthPRequest", "out_type": "GetAllWorkerHealthPResponse" }, + { + "name": "GetAllMasterStatus", + "in_type": "GetAllMasterStatusPRequest", + "out_type": "GetAllMasterStatusPResponse" + }, { "name": "Submit", "in_type": "SubmitRequest", @@ -4958,6 +5662,26 @@ "out_type": "RegisterJobWorkerPResponse" } ] + }, + { + "name": "JobMasterMasterService", + "rpcs": [ + { + "name": "GetMasterId", + "in_type": "GetJobMasterIdPRequest", + "out_type": "GetJobMasterIdPResponse" + }, + { + "name": "RegisterMaster", + "in_type": "RegisterJobMasterPRequest", + "out_type": "RegisterJobMasterPResponse" + }, + { + "name": "MasterHeartbeat", + "in_type": "JobMasterHeartbeatPRequest", + "out_type": "JobMasterHeartbeatPResponse" + } + ] } ], "imports": [ @@ -5440,6 +6164,10 @@ { "name": "RAFT_ADDRESSES", "integer": 12 + }, + { + "name": "MASTER_VERSION", + "integer": 13 } ] }, @@ -5706,6 +6434,32 @@ "name": "raftAddress", "type": "string", "is_repeated": true + }, + { + "id": 14, + "name": "masterVersions", + "type": "MasterVersion", + "is_repeated": true + } + ] + }, + { + "name": "MasterVersion", + "fields": [ + { + "id": 1, + "name": "addresses", + "type": "grpc.NetAddress" + }, + { + "id": 2, + "name": "version", + "type": "string" + }, + { + "id": 3, + "name": "state", + "type": "string" } ] }, @@ -5828,6 +6582,60 @@ } ] }, + { + "name": "ProxyStatus", + "fields": [ + { + "id": 1, + "name": "address", + "type": "grpc.NetAddress" + }, + { + "id": 2, + "name": "state", + "type": "string" + }, + { + "id": 3, + "name": "startTime", + "type": "int64" + }, + { + "id": 4, + "name": "lastHeartbeatTime", + "type": "int64" + }, + { + "id": 5, + "name": "version", + "type": "grpc.BuildVersion" + } + ] + }, + { + "name": "ListProxyStatusPRequest", + "fields": [ + { + "id": 1, + "name": "options", + "type": "ListProxyStatusPOptions" + } + ] + }, + { + "name": "ListProxyStatusPOptions" + }, + { + "name": "ListProxyStatusPResponse", + "fields": [ + { + "id": 1, + "name": "proxyStatuses", + "type": "ProxyStatus", + "is_repeated": true + } + ] + }, { "name": "SetPathConfigurationPOptions" }, @@ -5940,6 +6748,26 @@ "name": "configs", "type": "grpc.ConfigProperty", "is_repeated": true + }, + { + "id": 2, + "name": "startTimeMs", + "type": "int64" + }, + { + "id": 3, + "name": "losePrimacyTimeMs", + "type": "int64" + }, + { + "id": 4, + "name": "version", + "type": "string" + }, + { + "id": 5, + "name": "revision", + "type": "string" } ] }, @@ -5962,7 +6790,19 @@ "name": "RegisterMasterPResponse" }, { - "name": "MasterHeartbeatPOptions" + "name": "MasterHeartbeatPOptions", + "fields": [ + { + "id": 1, + "name": "lastCheckpointTime", + "type": "int64" + }, + { + "id": 2, + "name": "journalEntriesSinceCheckpoint", + "type": "int64" + } + ] }, { "name": "MasterHeartbeatPRequest", @@ -6014,6 +6854,39 @@ } } ] + }, + { + "name": "ProxyHeartbeatPOptions", + "fields": [ + { + "id": 1, + "name": "proxyAddress", + "type": "grpc.NetAddress" + }, + { + "id": 2, + "name": "startTime", + "type": "int64" + }, + { + "id": 3, + "name": "version", + "type": "grpc.BuildVersion" + } + ] + }, + { + "name": "ProxyHeartbeatPRequest", + "fields": [ + { + "id": 1, + "name": "options", + "type": "ProxyHeartbeatPOptions" + } + ] + }, + { + "name": "ProxyHeartbeatPResponse" } ], "services": [ @@ -6044,6 +6917,11 @@ "name": "Checkpoint", "in_type": "CheckpointPOptions", "out_type": "CheckpointPResponse" + }, + { + "name": "ListProxyStatus", + "in_type": "ListProxyStatusPRequest", + "out_type": "ListProxyStatusPResponse" } ] }, @@ -6096,6 +6974,16 @@ "out_type": "MasterHeartbeatPResponse" } ] + }, + { + "name": "MetaMasterProxyService", + "rpcs": [ + { + "name": "ProxyHeartbeat", + "in_type": "ProxyHeartbeatPRequest", + "out_type": "ProxyHeartbeatPResponse" + } + ] } ], "imports": [ @@ -6265,12 +7153,24 @@ { "id": 1, "name": "snapshotInfoRequest", - "type": "GetSnapshotInfoRequest" + "type": "GetSnapshotInfoRequest", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] }, { "id": 2, "name": "snapshotRequest", - "type": "GetSnapshotRequest" + "type": "GetSnapshotRequest", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] }, { "id": 3, @@ -6285,7 +7185,19 @@ { "id": 1, "name": "snapshotInfoResponse", - "type": "GetSnapshotInfoResponse" + "type": "GetSnapshotInfoResponse", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + } + ], + "options": [ + { + "name": "deprecated", + "value": "true" } ] }, @@ -6305,7 +7217,19 @@ { "id": 1, "name": "snapshotInfo", - "type": "SnapshotMetadata" + "type": "SnapshotMetadata", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + } + ], + "options": [ + { + "name": "deprecated", + "value": "true" } ] }, @@ -6315,12 +7239,30 @@ { "id": 1, "name": "latest", - "type": "SnapshotMetadata" + "type": "SnapshotMetadata", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + } + ], + "options": [ + { + "name": "deprecated", + "value": "true" } ] }, { - "name": "GetSnapshotRequest" + "name": "GetSnapshotRequest", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] }, { "name": "SnapshotMetadata", @@ -6334,6 +7276,11 @@ "id": 2, "name": "snapshotIndex", "type": "int64" + }, + { + "id": 3, + "name": "exists", + "type": "bool" } ] }, @@ -6358,12 +7305,24 @@ { "id": 4, "name": "offset", - "type": "int64" + "type": "int64", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] }, { "id": 5, "name": "eof", - "type": "bool" + "type": "bool", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] } ] }, @@ -6373,7 +7332,19 @@ { "id": 1, "name": "data", - "type": "SnapshotData" + "type": "SnapshotData", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + } + ], + "options": [ + { + "name": "deprecated", + "value": "true" } ] }, @@ -6383,7 +7354,19 @@ { "id": 1, "name": "offsetReceived", - "type": "int64" + "type": "int64", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + } + ], + "options": [ + { + "name": "deprecated", + "value": "true" } ] }, @@ -6393,7 +7376,19 @@ { "id": 1, "name": "offsetReceived", - "type": "int64" + "type": "int64", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + } + ], + "options": [ + { + "name": "deprecated", + "value": "true" } ] }, @@ -6403,9 +7398,24 @@ { "id": 1, "name": "data", - "type": "SnapshotData" + "type": "SnapshotData", + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + } + ], + "options": [ + { + "name": "deprecated", + "value": "true" } ] + }, + { + "name": "LatestSnapshotInfoPRequest" } ], "services": [ @@ -6417,13 +7427,36 @@ "in_type": "UploadSnapshotPRequest", "out_type": "UploadSnapshotPResponse", "in_streamed": true, - "out_streamed": true + "out_streamed": true, + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] }, { "name": "DownloadSnapshot", "in_type": "DownloadSnapshotPRequest", "out_type": "DownloadSnapshotPResponse", "in_streamed": true, + "out_streamed": true, + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + }, + { + "name": "RequestLatestSnapshotInfo", + "in_type": "LatestSnapshotInfoPRequest", + "out_type": "SnapshotMetadata" + }, + { + "name": "RequestLatestSnapshotData", + "in_type": "SnapshotMetadata", + "out_type": "SnapshotData", "out_streamed": true } ] @@ -6718,31 +7751,6 @@ } ] } - ], - "imports": [ - { - "path": "grpc/common.proto" - }, - { - "path": "grpc/table/table_master.proto" - } - ], - "package": { - "name": "alluxio.grpc.table.layout" - }, - "options": [ - { - "name": "java_multiple_files", - "value": "true" - }, - { - "name": "java_package", - "value": "alluxio.grpc.table.layout.hive" - }, - { - "name": "java_outer_classname", - "value": "HiveLayoutProto" - } ] } }, @@ -7915,9 +8923,6 @@ } ], "imports": [ - { - "path": "grpc/common.proto" - }, { "path": "grpc/job_master.proto" } @@ -7983,6 +8988,10 @@ "name": "META_MASTER_MASTER_SERVICE", "integer": 8 }, + { + "name": "META_MASTER_PROXY_SERVICE", + "integer": 18 + }, { "name": "METRICS_MASTER_CLIENT_SERVICE", "integer": 9 @@ -7995,6 +9004,10 @@ "name": "JOB_MASTER_WORKER_SERVICE", "integer": 11 }, + { + "name": "JOB_MASTER_MASTER_SERVICE", + "integer": 19 + }, { "name": "JOURNAL_MASTER_CLIENT_SERVICE", "integer": 13 @@ -8032,6 +9045,11 @@ "id": 1, "name": "serviceType", "type": "ServiceType" + }, + { + "id": 2, + "name": "allowedOnStandbyMasters", + "type": "bool" } ] }, @@ -8120,7 +9138,100 @@ { "protopath": "proto:/:dataserver:/:protocol.proto", "def": { + "enums": [ + { + "name": "RequestType", + "enum_fields": [ + { + "name": "ALLUXIO_BLOCK" + }, + { + "name": "UFS_FILE", + "integer": 1 + } + ] + }, + { + "name": "ReadResponse.Type", + "enum_fields": [ + { + "name": "UFS_READ_HEARTBEAT", + "integer": 1 + } + ] + } + ], "messages": [ + { + "name": "ReadRequest", + "fields": [ + { + "id": 1, + "name": "block_id", + "type": "int64" + }, + { + "id": 2, + "name": "offset", + "type": "int64" + }, + { + "id": 3, + "name": "length", + "type": "int64" + }, + { + "id": 4, + "name": "cancel", + "type": "bool" + }, + { + "id": 7, + "name": "promote", + "type": "bool" + }, + { + "id": 5, + "name": "chunk_size", + "type": "int64" + }, + { + "id": 6, + "name": "open_ufs_block_options", + "type": "OpenUfsBlockOptions" + } + ] + }, + { + "name": "AsyncCacheRequest", + "fields": [ + { + "id": 1, + "name": "block_id", + "type": "int64" + }, + { + "id": 2, + "name": "source_host", + "type": "string" + }, + { + "id": 3, + "name": "source_port", + "type": "int32" + }, + { + "id": 4, + "name": "open_ufs_block_options", + "type": "OpenUfsBlockOptions" + }, + { + "id": 5, + "name": "length", + "type": "int64" + } + ] + }, { "name": "OpenUfsBlockOptions", "fields": [ @@ -8155,84 +9266,222 @@ "type": "bool" }, { - "id": 7, - "name": "user", + "id": 7, + "name": "user", + "type": "string" + }, + { + "id": 8, + "name": "block_in_ufs_tier", + "type": "bool" + } + ] + }, + { + "name": "WriteRequest", + "fields": [ + { + "id": 1, + "name": "type", + "type": "RequestType" + }, + { + "id": 2, + "name": "id", + "type": "int64" + }, + { + "id": 3, + "name": "offset", + "type": "int64" + }, + { + "id": 4, + "name": "tier", + "type": "int32" + }, + { + "id": 5, + "name": "eof", + "type": "bool" + }, + { + "id": 6, + "name": "cancel", + "type": "bool" + }, + { + "id": 7, + "name": "create_ufs_file_options", + "type": "CreateUfsFileOptions" + } + ] + }, + { + "name": "CreateUfsFileOptions", + "fields": [ + { + "id": 1, + "name": "ufs_path", + "type": "string" + }, + { + "id": 2, + "name": "owner", + "type": "string" + }, + { + "id": 3, + "name": "group", + "type": "string" + }, + { + "id": 4, + "name": "mode", + "type": "int32" + }, + { + "id": 5, + "name": "mount_id", + "type": "int64" + }, + { + "id": 6, + "name": "acl", + "type": "alluxio.proto.shared.AccessControlList" + } + ] + }, + { + "name": "CreateUfsBlockOptions", + "fields": [ + { + "id": 1, + "name": "bytes_in_block_store", + "type": "int64" + }, + { + "id": 2, + "name": "mount_id", + "type": "int64" + }, + { + "id": 3, + "name": "fallback", + "type": "bool" + } + ] + }, + { + "name": "ReadResponse", + "fields": [ + { + "id": 1, + "name": "type", + "type": "Type" + } + ] + }, + { + "name": "Heartbeat" + }, + { + "name": "Response", + "fields": [ + { + "id": 1, + "name": "status", + "type": "status.PStatus" + }, + { + "id": 2, + "name": "message", "type": "string" - }, - { - "id": 8, - "name": "block_in_ufs_tier", - "type": "bool" } ] }, { - "name": "CreateUfsFileOptions", + "name": "LocalBlockOpenRequest", "fields": [ { "id": 1, - "name": "ufs_path", - "type": "string" + "name": "block_id", + "type": "int64" }, { "id": 2, - "name": "owner", - "type": "string" - }, + "name": "promote", + "type": "bool" + } + ] + }, + { + "name": "LocalBlockOpenResponse", + "fields": [ { - "id": 3, - "name": "group", + "id": 1, + "name": "path", "type": "string" - }, - { - "id": 4, - "name": "mode", - "type": "int32" - }, + } + ] + }, + { + "name": "LocalBlockCloseRequest", + "fields": [ { - "id": 5, - "name": "mount_id", + "id": 1, + "name": "block_id", "type": "int64" - }, - { - "id": 6, - "name": "acl", - "type": "alluxio.proto.shared.AccessControlList" } ] }, { - "name": "CreateUfsBlockOptions", + "name": "LocalBlockCreateRequest", "fields": [ { "id": 1, - "name": "bytes_in_block_store", + "name": "block_id", "type": "int64" }, { - "id": 2, - "name": "mount_id", + "id": 3, + "name": "tier", + "type": "int32" + }, + { + "id": 4, + "name": "space_to_reserve", "type": "int64" }, { - "id": 3, - "name": "fallback", + "id": 5, + "name": "only_reserve_space", "type": "bool" } ] }, { - "name": "Response", + "name": "LocalBlockCreateResponse", "fields": [ { "id": 1, - "name": "status", - "type": "status.PStatus" + "name": "path", + "type": "string" + } + ] + }, + { + "name": "LocalBlockCompleteRequest", + "fields": [ + { + "id": 1, + "name": "block_id", + "type": "int64" }, { "id": 2, - "name": "message", - "type": "string" + "name": "cancel", + "type": "bool" } ] } @@ -8261,7 +9510,7 @@ "name": "OK" }, { - "name": "CANCELED", + "name": "CANCELLED", "integer": 1 }, { @@ -8358,6 +9607,11 @@ "id": 2, "name": "length", "type": "int64" + }, + { + "id": 3, + "name": "block_location", + "type": "grpc.BlockLocation" } ] }, @@ -8372,6 +9626,11 @@ ] } ], + "imports": [ + { + "path": "grpc/common.proto" + } + ], "package": { "name": "alluxio.proto.journal" } @@ -8411,6 +9670,10 @@ { "name": "FREE", "integer": 1 + }, + { + "name": "DELETE_ALLUXIO", + "integer": 2 } ] }, @@ -9368,10 +10631,160 @@ ], "imports": [ { - "path": "grpc/common.proto" + "path": "proto/shared/acl.proto" + } + ], + "package": { + "name": "alluxio.proto.journal" + } + } + }, + { + "protopath": "proto:/:journal:/:job.proto", + "def": { + "enums": [ + { + "name": "PJobState", + "enum_fields": [ + { + "name": "CREATED", + "integer": 1 + }, + { + "name": "STOPPED", + "integer": 2 + }, + { + "name": "SUCCEEDED", + "integer": 3 + }, + { + "name": "FAILED", + "integer": 4 + } + ] + } + ], + "messages": [ + { + "name": "LoadJobEntry", + "fields": [ + { + "id": 1, + "name": "load_path", + "type": "string" + }, + { + "id": 2, + "name": "state", + "type": "PJobState" + }, + { + "id": 3, + "name": "bandwidth", + "type": "int64" + }, + { + "id": 4, + "name": "verify", + "type": "bool" + }, + { + "id": 5, + "name": "user", + "type": "string" + }, + { + "id": 6, + "name": "partialListing", + "type": "bool" + }, + { + "id": 7, + "name": "job_id", + "type": "string" + }, + { + "id": 8, + "name": "end_time", + "type": "int64" + } + ] }, { - "path": "proto/shared/acl.proto" + "name": "CopyJobEntry", + "fields": [ + { + "id": 1, + "name": "src", + "type": "string" + }, + { + "id": 2, + "name": "dst", + "type": "string" + }, + { + "id": 3, + "name": "src_ufs_address", + "type": "string" + }, + { + "id": 4, + "name": "dst_ufs_address", + "type": "string" + }, + { + "id": 5, + "name": "state", + "type": "PJobState" + }, + { + "id": 6, + "name": "bandwidth", + "type": "int64" + }, + { + "id": 7, + "name": "verify", + "type": "bool" + }, + { + "id": 8, + "name": "user", + "type": "string" + }, + { + "id": 9, + "name": "partialListing", + "type": "bool" + }, + { + "id": 10, + "name": "job_id", + "type": "string" + }, + { + "id": 11, + "name": "end_time", + "type": "int64" + }, + { + "id": 12, + "name": "overwrite", + "type": "bool" + }, + { + "id": 13, + "name": "writeType", + "type": "grpc.file.WritePType" + } + ] + } + ], + "imports": [ + { + "path": "grpc/file_system_master.proto" } ], "package": { @@ -9591,6 +11004,16 @@ "name": "update_inode_file", "type": "UpdateInodeFileEntry" }, + { + "id": 53, + "name": "load_job", + "type": "LoadJobEntry" + }, + { + "id": 54, + "name": "copy_job", + "type": "CopyJobEntry" + }, { "id": 39, "name": "journal_entries", @@ -9612,6 +11035,9 @@ }, { "path": "proto/journal/table.proto" + }, + { + "path": "proto/journal/job.proto" } ], "package": { @@ -10014,6 +11440,36 @@ } } }, + { + "protopath": "proto:/:meta:/:dora_meta.proto", + "def": { + "messages": [ + { + "name": "FileStatus", + "fields": [ + { + "id": 1, + "name": "fileInfo", + "type": "alluxio.grpc.file.FileInfo" + }, + { + "id": 2, + "name": "ts", + "type": "int64" + } + ] + } + ], + "imports": [ + { + "path": "grpc/file_system_master.proto" + } + ], + "package": { + "name": "alluxio.proto.meta" + } + } + }, { "protopath": "proto:/:meta:/:inode_meta.proto", "def": { diff --git a/core/transport/src/main/proto/proto/client/cache.proto b/common/transport/src/main/proto/proto/client/cache.proto similarity index 100% rename from core/transport/src/main/proto/proto/client/cache.proto rename to common/transport/src/main/proto/proto/client/cache.proto diff --git a/common/transport/src/main/proto/proto/dataserver/protocol.proto b/common/transport/src/main/proto/proto/dataserver/protocol.proto new file mode 100644 index 000000000000..aefbe70660ee --- /dev/null +++ b/common/transport/src/main/proto/proto/dataserver/protocol.proto @@ -0,0 +1,148 @@ +syntax = "proto2"; + +package alluxio.proto.dataserver; + +import "proto/dataserver/status.proto"; +import "proto/shared/acl.proto"; + +// The read/write request type. It can either be an Alluxio block operation or a UFS file operation. +// next available id: 2 +enum RequestType { + ALLUXIO_BLOCK = 0; + UFS_FILE = 1; +} + +// The read request. +// next available id: 8 +message ReadRequest { + optional int64 block_id = 1; + optional int64 offset = 2; + optional int64 length = 3; + // If set, this request is to cancel the reading request for the id. + optional bool cancel = 4; + // Whether the block should be promoted before reading + optional bool promote = 7; + + // If set, the server should send packets in the specified packet size. + optional int64 chunk_size = 5; + + // This is only set for UFS block read. + optional OpenUfsBlockOptions open_ufs_block_options = 6; +} + +// Options for caching a block asynchronously +// next available id: 6 +message AsyncCacheRequest { + optional int64 block_id = 1; + // TODO(calvin): source host and port should be replace with WorkerNetAddress + optional string source_host = 2; + optional int32 source_port = 3; + optional OpenUfsBlockOptions open_ufs_block_options = 4; + optional int64 length = 5; +} + +// Options to open a UFS block. +// next available id: 7 +message OpenUfsBlockOptions { + optional string ufs_path = 1; + // The offset of the block in within the file. + optional int64 offset_in_file = 2; + // The block size. + optional int64 block_size = 3; + optional int32 maxUfsReadConcurrency = 4; + optional int64 mountId = 5; + // If set, do not try to cache the block locally when reading the data from the UFS. + optional bool no_cache = 6; + // The client does not need to set this. This is set by the worker. + optional string user = 7; +} + +// The write request. +// next available id: 8 +message WriteRequest { + optional RequestType type = 1; + // The block ID or UFS file ID. + optional int64 id = 2; + optional int64 offset = 3; + + // This is only applicable for block write. + optional int32 tier = 4; + + optional bool eof = 5; + optional bool cancel = 6; + + // This is only applicable for ufs writes. + optional CreateUfsFileOptions create_ufs_file_options = 7; +} + +// Options to create a UFS file. +// next available: 6 +message CreateUfsFileOptions { + optional string ufs_path = 1; + optional string owner = 2; + optional string group = 3; + optional int32 mode = 4; + optional int64 mount_id = 5; + optional alluxio.proto.shared.AccessControlList acl = 6; +} + +// The read response. +// next available id: 2 +message ReadResponse { + enum Type { + // A heatbeat message indicates that the server is still actively acquiring access to a UFS file. + // This is to avoid timing out in the client. + UFS_READ_HEARTBEAT = 1; + } + optional Type type = 1; +} + +// A heartbeat +message Heartbeat { + // Empty message +} + +// The response. +// next available id: 3 +message Response { + optional status.PStatus status = 1; + optional string message = 2; +} + +// Netty RPCs. Every RPC needs to define a request type and optionally a response type (default to Response). + +// next available id: 3 +message LocalBlockOpenRequest { + optional int64 block_id = 1; + optional bool promote = 2; +} + +// next available id: 2 +message LocalBlockOpenResponse { + optional string path = 1; +} + +// next available id: 2 +message LocalBlockCloseRequest { + optional int64 block_id = 1; +} + +// next available id: 6 +message LocalBlockCreateRequest { + optional int64 block_id = 1; + optional int32 tier = 3; + optional int64 space_to_reserve = 4; + // If set, only reserve space for the block. + optional bool only_reserve_space = 5; +} + +// next available id: 2 +message LocalBlockCreateResponse { + optional string path = 1; +} + +// next available id: 3 +message LocalBlockCompleteRequest { + optional int64 block_id = 1; + optional bool cancel = 2; +} \ No newline at end of file diff --git a/core/transport/src/main/proto/proto/dataserver/status.proto b/common/transport/src/main/proto/proto/dataserver/status.proto similarity index 99% rename from core/transport/src/main/proto/proto/dataserver/status.proto rename to common/transport/src/main/proto/proto/dataserver/status.proto index 19b009e5d633..cf559ac6a093 100644 --- a/core/transport/src/main/proto/proto/dataserver/status.proto +++ b/common/transport/src/main/proto/proto/dataserver/status.proto @@ -8,7 +8,7 @@ enum PStatus { OK = 0; // Canceled indicates the operation was cancelled (typically by the caller). - CANCELED = 1; + CANCELLED = 1; // Unknown error. An example of where this error may be returned is // if a Status value received from another address space belongs to diff --git a/core/transport/src/main/proto/proto/journal/block.proto b/common/transport/src/main/proto/proto/journal/block.proto similarity index 83% rename from core/transport/src/main/proto/proto/journal/block.proto rename to common/transport/src/main/proto/proto/journal/block.proto index 3a605bde8a7e..0a03eca15196 100644 --- a/core/transport/src/main/proto/proto/journal/block.proto +++ b/common/transport/src/main/proto/proto/journal/block.proto @@ -2,6 +2,8 @@ syntax = "proto2"; package alluxio.proto.journal; +import "grpc/common.proto"; + // Journal entry messages for the block master. // next available id: 2 @@ -13,6 +15,7 @@ message BlockContainerIdGeneratorEntry { message BlockInfoEntry { optional int64 block_id = 1; optional int64 length = 2; + optional grpc.BlockLocation block_location = 3; } // next available id: 2 diff --git a/core/transport/src/main/proto/proto/journal/file.proto b/common/transport/src/main/proto/proto/journal/file.proto similarity index 99% rename from core/transport/src/main/proto/proto/journal/file.proto rename to common/transport/src/main/proto/proto/journal/file.proto index c59840a2d2ec..bb7e8adf7af4 100644 --- a/core/transport/src/main/proto/proto/journal/file.proto +++ b/common/transport/src/main/proto/proto/journal/file.proto @@ -2,7 +2,6 @@ syntax = "proto2"; package alluxio.proto.journal; -import "grpc/common.proto"; import "proto/shared/acl.proto"; // Journal entry messages for the file master. @@ -172,6 +171,7 @@ message InodeDirectoryIdGeneratorEntry { enum PTtlAction { DELETE = 0; FREE = 1; + DELETE_ALLUXIO = 2; } // next available id: 30 diff --git a/common/transport/src/main/proto/proto/journal/job.proto b/common/transport/src/main/proto/proto/journal/job.proto new file mode 100644 index 000000000000..027c9e4c0d74 --- /dev/null +++ b/common/transport/src/main/proto/proto/journal/job.proto @@ -0,0 +1,66 @@ +syntax = "proto2"; + +package alluxio.proto.journal; + + +// Journal entry messages for the block master. +enum PJobState { + CREATED = 1; + STOPPED = 2; + SUCCEEDED = 3; + FAILED = 4; +} + +// next available id: 9 +message LoadJobEntry { + required string load_path = 1; + required PJobState state = 2; + optional int64 bandwidth = 3; + required bool verify = 4; + optional string user = 5; + required bool partialListing = 6; + required string job_id = 7; + optional int64 end_time = 8; + optional bool load_metadata_only = 9; + optional bool skip_if_exists = 10; + optional string file_filter_regx = 11; + optional int32 replicas = 12; +} + +// next available id: 13 +message CopyJobEntry { + required string src = 1; + required string dst = 2; + required PJobState state = 3; + optional int64 bandwidth = 4; + required bool verify = 5; + optional string user = 6; + required bool partialListing = 7; + required string job_id = 8; + optional int64 end_time = 9; + optional bool overwrite = 10; + optional bool check_content = 11; + optional FileFilter filter = 12; +} + +// next available id: 13 +message MoveJobEntry { + required string src= 1; + required string dst= 2; + required PJobState state = 3; + optional int64 bandwidth = 4; + required bool verify = 5; + optional string user = 6; + required bool partialListing = 7; + required string job_id = 8; + optional int64 end_time = 9; + optional bool overwrite = 10; + optional bool check_content = 11; + optional FileFilter filter = 12; +} + +message FileFilter { + required string name = 1; + optional string pattern = 2; + required string value = 3; +} diff --git a/core/transport/src/main/proto/proto/journal/journal.proto b/common/transport/src/main/proto/proto/journal/journal.proto similarity index 82% rename from core/transport/src/main/proto/proto/journal/journal.proto rename to common/transport/src/main/proto/proto/journal/journal.proto index 31ed4bcdda90..6a968feee276 100644 --- a/core/transport/src/main/proto/proto/journal/journal.proto +++ b/common/transport/src/main/proto/proto/journal/journal.proto @@ -6,8 +6,8 @@ option java_package = "alluxio.proto.journal"; import "proto/journal/block.proto"; import "proto/journal/file.proto"; import "proto/journal/meta.proto"; -import "proto/journal/table.proto"; - +import "proto/journal/job.proto"; +import "proto/journal/policy.proto"; // Wraps around all types of Alluxio journal entries. // // NOTE: Exactly one of the *Entry entries is expected to be set. Ideally we would use the 'oneof' @@ -24,19 +24,16 @@ message JournalOpPId { optional int64 leastSignificantBits = 2; } -// next available id: 53 +// next available id: 56 message JournalEntry { // shared fields. optional int64 sequence_number = 1; optional JournalOpPId operationId = 52; // action fields. optional ActiveSyncTxIdEntry active_sync_tx_id = 34; - optional AddTableEntry add_table = 43; - optional AddTablePartitionsEntry add_table_partitions = 51; optional AddSyncPointEntry add_sync_point = 32; optional AddMountPointEntry add_mount_point = 2; optional AsyncPersistRequestEntry async_persist_request = 16; - optional AttachDbEntry attach_db = 44; optional BlockContainerIdGeneratorEntry block_container_id_generator = 3; optional BlockInfoEntry block_info = 4; optional ClusterInfoEntry cluster_info = 42; @@ -44,7 +41,6 @@ message JournalEntry { optional DeleteBlockEntry delete_block = 29; optional DeleteFileEntry delete_file = 6; optional DeleteMountPointEntry delete_mount_point = 8; - optional DetachDbEntry detach_db = 45; optional InodeDirectoryEntry inode_directory = 9; optional InodeDirectoryIdGeneratorEntry inode_directory_id_generator = 10; optional InodeFileEntry inode_file = 11; @@ -53,19 +49,19 @@ message JournalEntry { optional PathPropertiesEntry path_properties = 40; optional PersistDirectoryEntry persist_directory = 15; optional RemovePathPropertiesEntry remove_path_properties = 41; - optional RemoveTableEntry remove_table = 50; - optional RemoveTransformJobInfoEntry remove_transform_job_info = 47; optional RemoveSyncPointEntry remove_sync_point = 33; optional RenameEntry rename = 19; optional SetAclEntry set_acl = 31; optional SetAttributeEntry set_attribute = 27; - optional AddTransformJobInfoEntry add_transform_job_info = 46; - optional CompleteTransformTableEntry complete_transform_table = 48; - optional UpdateDatabaseInfoEntry update_database_info = 49; optional UpdateUfsModeEntry update_ufs_mode = 30; optional UpdateInodeEntry update_inode = 35; optional UpdateInodeDirectoryEntry update_inode_directory = 36; optional UpdateInodeFileEntry update_inode_file = 37; + optional LoadJobEntry load_job = 53; + optional CopyJobEntry copy_job = 54; + optional MoveJobEntry move_job = 55; + optional PolicyDefinitionEntry policy_definition = 56; + optional PolicyRemoveEntry policy_remove = 57; // This journal entry is a list of other entries. when a journal entry // contains other journal entries, all other optional fields must be unset. diff --git a/core/transport/src/main/proto/proto/journal/meta.proto b/common/transport/src/main/proto/proto/journal/meta.proto similarity index 100% rename from core/transport/src/main/proto/proto/journal/meta.proto rename to common/transport/src/main/proto/proto/journal/meta.proto diff --git a/common/transport/src/main/proto/proto/journal/policy.proto b/common/transport/src/main/proto/proto/journal/policy.proto new file mode 100644 index 000000000000..f51fcfbf467d --- /dev/null +++ b/common/transport/src/main/proto/proto/journal/policy.proto @@ -0,0 +1,18 @@ +syntax = "proto2"; + +package alluxio.proto.journal; + +import "grpc/policy_master.proto"; + +// next available id: 5 +message PolicyDefinitionEntry { + optional int64 id = 1; + optional string name = 2; + optional int64 created_at = 3; + optional alluxio.grpc.policy.AddPolicyPOptions options = 4; +} + +// next available id: 2 +message PolicyRemoveEntry { + optional string name = 1; +} diff --git a/core/transport/src/main/proto/proto/meta/block.proto b/common/transport/src/main/proto/proto/meta/block.proto similarity index 100% rename from core/transport/src/main/proto/proto/meta/block.proto rename to common/transport/src/main/proto/proto/meta/block.proto diff --git a/common/transport/src/main/proto/proto/meta/dora_meta.proto b/common/transport/src/main/proto/proto/meta/dora_meta.proto new file mode 100644 index 000000000000..e5e62b19722c --- /dev/null +++ b/common/transport/src/main/proto/proto/meta/dora_meta.proto @@ -0,0 +1,17 @@ +syntax = "proto2"; + +package alluxio.proto.meta; + +import "grpc/file_system_master.proto"; + +/** + * Metadata used in Dora Worker. + * + * next available id: 3 + */ +message FileStatus { + optional alluxio.grpc.file.FileInfo fileInfo = 1; + + // the timestamp in nanoseconds when this FileStatus is refreshed + optional int64 ts = 2; +} diff --git a/core/transport/src/main/proto/proto/meta/inode_meta.proto b/common/transport/src/main/proto/proto/meta/inode_meta.proto similarity index 100% rename from core/transport/src/main/proto/proto/meta/inode_meta.proto rename to common/transport/src/main/proto/proto/meta/inode_meta.proto diff --git a/core/transport/src/main/proto/proto/shared/acl.proto b/common/transport/src/main/proto/proto/shared/acl.proto similarity index 100% rename from core/transport/src/main/proto/proto/shared/acl.proto rename to common/transport/src/main/proto/proto/shared/acl.proto diff --git a/conf/alluxio-env.sh.template b/conf/alluxio-env.sh.template index 243964a4b513..c2e53d2686a9 100644 --- a/conf/alluxio-env.sh.template +++ b/conf/alluxio-env.sh.template @@ -33,13 +33,6 @@ # The directory where log files are stored. (Default: ${ALLUXIO_HOME}/logs). # ALLUXIO_LOGS_DIR -# The hostname for log server. If set, remote logging is enabled. -# ALLUXIO_LOGSERVER_HOSTNAME -# If remote logging is enabled, the port for log server. (Default: 45600) -# ALLUXIO_LOGSERVER_PORT -# If remote logging is enabled, the directory where log server log files are stored. (Default: ${ALLUXIO_HOME}/logs) -# ALLUXIO_LOGSERVER_LOGS_DIR - # Config properties set for Alluxio master, worker and shell. (Default: "") # E.g. "-Dalluxio.master.rpc.port=39999" # ALLUXIO_JAVA_OPTS @@ -64,10 +57,6 @@ # E.g. "-Xms2048M -Xmx2048M" to limit the heap size of proxy. # ALLUXIO_PROXY_JAVA_OPTS -# Config properties set for Alluxio log server daemon. (Default: "") -# E.g. "-Xms2048M -Xmx2048M" to limit the heap size of log server. -# ALLUXIO_LOGSERVER_JAVA_OPTS - # Config properties set for Alluxio shell. (Default: "") # E.g. "-Dalluxio.user.file.writetype.default=CACHE_THROUGH" # ALLUXIO_USER_JAVA_OPTS @@ -80,10 +69,6 @@ # E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60001" # ALLUXIO_MASTER_ATTACH_OPTS -# Configuring remote debugging for Alluxio secondary master process. (Default: "") -# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60010" -# ALLUXIO_SECONDARY_MASTER_ATTACH_OPTS - # Configuring remote debugging for Alluxio job master process. (Default: "") # E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60002" # ALLUXIO_JOB_MASTER_ATTACH_OPTS @@ -100,10 +85,6 @@ # E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60005" # ALLUXIO_PROXY_ATTACH_OPTS -# Configuring remote debugging for Alluxio log server process. (Default: "") -# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60006" -# ALLUXIO_LOGSERVER_ATTACH_OPTS - # Configuring remote debugging for Alluxio fuse process. (Default: "") # E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60009" # ALLUXIO_FUSE_ATTACH_OPTS diff --git a/conf/alluxio-site-properties.dora-local-minimum.template b/conf/alluxio-site-properties.dora-local-minimum.template new file mode 100644 index 000000000000..e288360d8dc6 --- /dev/null +++ b/conf/alluxio-site-properties.dora-local-minimum.template @@ -0,0 +1,19 @@ +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +# This template contains a set of minimum configurations to run a dora cluster locally. + +alluxio.dora.client.ufs.fallback.enabled=false +alluxio.master.hostname=localhost +alluxio.master.journal.type=NOOP +alluxio.dora.client.ufs.root=/tmp/underFSStorage +alluxio.master.mount.table.root.ufs=/tmp/underFSStorage +alluxio.worker.http.server.enabled=false \ No newline at end of file diff --git a/conf/core-site.xml.template b/conf/core-site.xml.template index 11bb8d6b2414..7515b50d214e 100644 --- a/conf/core-site.xml.template +++ b/conf/core-site.xml.template @@ -12,116 +12,7 @@ --> - fs.swift.impl - org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem - - - - - fs.swift.service.dal05.auth.url - https://dal05.objectstorage.softlayer.net/auth/v1.0 - - - fs.swift.service.dal05.http.port - 8080 - - - fs.swift.service.dal05.public - true - - - fs.swift.service.dal05.location-aware - false - - - fs.swift.service.ibm.dal05.endpoint.prefix - endpoints - - - fs.swift.service.dal05.apikey - API_KEY - - - fs.swift.service.dal05.username - ACCOUNT:USER NAME - - - fs.swift.service.dal05.use.get.auth - true - - - - - fs.swift.service.swift1.location-aware - false - - - fs.swift.service.swift1.auth.url - http://127.0.0.1:5000/v2.0/tokens - - - fs.swift.service.swift1.http.port - 8080 - - - fs.swift.service.swift1.region - RegionOne - - - fs.swift.service.swift1.public - true - - - fs.swift.service.swift1.auth.endpoint.prefix - endpoints - - - fs.swift.service.swift1.tenant - TENANT - - - fs.swift.service.swift1.password - PASSWORD - - - fs.swift.service.swift1.username - USER NAME - - - - - fs.swift.service.swift2.auth.url - http://127.0.0.1:8080/auth/v1.0 - - - fs.swift.service.swift2.http.port - 8080 - - - fs.swift.service.swift2.public - true - - - fs.swift.service.swift2.location-aware - false - - - fs.swift.service.swift2.endpoint.prefix - endpoints - - - fs.swift.service.swift2.apikey - testing - - - fs.swift.service.swift2.username - test:tester - - - fs.swift.service.swift2.use.get.auth - true + * + * diff --git a/conf/etcd/etcd.conf.template b/conf/etcd/etcd.conf.template new file mode 100644 index 000000000000..1b125d74fae2 --- /dev/null +++ b/conf/etcd/etcd.conf.template @@ -0,0 +1,69 @@ +# This is the configuration file to start a etcd instance +# e.g. /usr/local/bin/etcd --config-file /etc/etcd/etcd.conf +# *******README****** +# To make etcd a linux service: +# After installation of etcd, make sure etcd and etcdctl +# are available in /usr/local/bin +# To make etcd a linux service: +# Copy alluxio/conf/etcd/etcd.service.template to /etc/systemd/system/etcd.service +# Copy alluxio/conf/etcd/etcd.conf.template to /etc/etcd/etcd.conf +# For each etcd instance, change the config params in etcd.conf +# accordingly. +# And do: +# #systemctl daemon-reload +# Then etcd could be registered as a linux service +# e.g. +# Check status +# #service etcd status +# Start etcd +# #service etcd start +# Stop etcd +# #service etcd stop + + +# Human-readable name for this member. +#name: 'etcd1' + +# Path to the data directory. +data-dir: /etcd-data-dir/data + +# Path to the dedicated wal directory. +wal-dir: /etcd-data-dir/wal + + +# List of comma separated URLs to listen on for peer traffic. +#give ip/hostname of this etcd instance +listen-peer-urls: http://:2380 + +# List of comma separated URLs to listen on for client traffic. +#give ip/hostname of this etcd instance +listen-client-urls: http://:2379,http://127.0.0.1:2379 + +# List of this member's peer URLs to advertise to the rest of the cluster. +# The URLs needed to be a comma-separated list. +#give ip/hostname of this etcd instance for remote etcd members communication +initial-advertise-peer-urls: http://:2380 + +# List of this member's client URLs to advertise to the public. +# The URLs needed to be a comma-separated list. +#give ip/hostname of this etcd instance for etcd client communication +advertise-client-urls: http://:2379 + +# Initial cluster configuration for bootstrapping. +#give all ip/hostnames of members of initial etcd cluster +initial-cluster: etcd0=http://:2380,etcd1=http://:2380,etcd2=http://:2380 + +# Initial cluster token for the etcd cluster during bootstrap. +#initial-cluster-token: 'etcd-cluster-1' + +# Initial cluster state ('new' or 'existing'). +initial-cluster-state: 'new' + +# Enable debug-level logging for etcd. +#log-level: debug + +#logger: zap + +# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd. +# log-outputs: [stderr] + diff --git a/conf/etcd/etcd.service.template b/conf/etcd/etcd.service.template new file mode 100644 index 000000000000..70a51c67475c --- /dev/null +++ b/conf/etcd/etcd.service.template @@ -0,0 +1,11 @@ +[Unit] +Description=Etcd Service + +[Service] +ExecStart=/usr/local/bin/etcd --config-file /etc/etcd/etcd.conf +KillSignal=SIGTERM +StandardOutput=append:/var/log/etcd.log +StandardError=append:/var/log/etcd.err + +[Install] +WantedBy=default.target diff --git a/conf/log4j.properties b/conf/log4j.properties index ff3714669160..68118926085e 100644 --- a/conf/log4j.properties +++ b/conf/log4j.properties @@ -13,9 +13,6 @@ log4j.rootLogger=INFO, ${alluxio.logger.type}, ${alluxio.remote.logger.type} -log4j.category.alluxio.logserver=INFO, ${alluxio.logserver.logger.type} -log4j.additivity.alluxio.logserver=false - log4j.logger.AUDIT_LOG=INFO, ${alluxio.master.audit.logger.type} log4j.logger.JOB_MASTER_AUDIT_LOG=INFO, ${alluxio.job.master.audit.logger.type} log4j.logger.PROXY_AUDIT_LOG=INFO, ${alluxio.proxy.audit.logger.type} @@ -31,7 +28,7 @@ log4j.appender.=org.apache.log4j.varia.NullAppender log4j.appender.Console=org.apache.log4j.ConsoleAppender log4j.appender.Console.Target=System.out log4j.appender.Console.layout=org.apache.log4j.PatternLayout -log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # The ParquetWriter logs for every row group which is not noisy for large row group size, # but very noisy for small row group size. @@ -44,7 +41,7 @@ log4j.appender.JOB_MASTER_LOGGER.File=${alluxio.logs.dir}/job_master.log log4j.appender.JOB_MASTER_LOGGER.MaxFileSize=10MB log4j.appender.JOB_MASTER_LOGGER.MaxBackupIndex=100 log4j.appender.JOB_MASTER_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.JOB_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.JOB_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Appender for Job Workers log4j.appender.JOB_WORKER_LOGGER=org.apache.log4j.RollingFileAppender @@ -52,7 +49,7 @@ log4j.appender.JOB_WORKER_LOGGER.File=${alluxio.logs.dir}/job_worker.log log4j.appender.JOB_WORKER_LOGGER.MaxFileSize=10MB log4j.appender.JOB_WORKER_LOGGER.MaxBackupIndex=100 log4j.appender.JOB_WORKER_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.JOB_WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.JOB_WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Appender for Master log4j.appender.MASTER_LOGGER=org.apache.log4j.RollingFileAppender @@ -60,15 +57,7 @@ log4j.appender.MASTER_LOGGER.File=${alluxio.logs.dir}/master.log log4j.appender.MASTER_LOGGER.MaxFileSize=10MB log4j.appender.MASTER_LOGGER.MaxBackupIndex=100 log4j.appender.MASTER_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n - -# Appender for Master -log4j.appender.SECONDARY_MASTER_LOGGER=org.apache.log4j.RollingFileAppender -log4j.appender.SECONDARY_MASTER_LOGGER.File=${alluxio.logs.dir}/secondary_master.log -log4j.appender.SECONDARY_MASTER_LOGGER.MaxFileSize=10MB -log4j.appender.SECONDARY_MASTER_LOGGER.MaxBackupIndex=100 -log4j.appender.SECONDARY_MASTER_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.SECONDARY_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Appender for Master audit log4j.appender.MASTER_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender @@ -76,7 +65,7 @@ log4j.appender.MASTER_AUDIT_LOGGER.File=${alluxio.logs.dir}/master_audit.log log4j.appender.MASTER_AUDIT_LOGGER.MaxFileSize=10MB log4j.appender.MASTER_AUDIT_LOGGER.MaxBackupIndex=100 log4j.appender.MASTER_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Appender for Job Master audit log4j.appender.JOB_MASTER_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender @@ -84,7 +73,7 @@ log4j.appender.JOB_MASTER_AUDIT_LOGGER.File=${alluxio.logs.dir}/job_master_audit log4j.appender.JOB_MASTER_AUDIT_LOGGER.MaxFileSize=10MB log4j.appender.JOB_MASTER_AUDIT_LOGGER.MaxBackupIndex=100 log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Appender for Proxy log4j.appender.PROXY_LOGGER=org.apache.log4j.RollingFileAppender @@ -92,7 +81,7 @@ log4j.appender.PROXY_LOGGER.File=${alluxio.logs.dir}/proxy.log log4j.appender.PROXY_LOGGER.MaxFileSize=10MB log4j.appender.PROXY_LOGGER.MaxBackupIndex=100 log4j.appender.PROXY_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.PROXY_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.PROXY_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Appender for Proxy audit log4j.appender.PROXY_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender @@ -100,7 +89,7 @@ log4j.appender.PROXY_AUDIT_LOGGER.File=${alluxio.logs.dir}/proxy_audit.log log4j.appender.PROXY_AUDIT_LOGGER.MaxFileSize=10MB log4j.appender.PROXY_AUDIT_LOGGER.MaxBackupIndex=100 log4j.appender.PROXY_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.PROXY_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n +log4j.appender.PROXY_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2}[%t](%F:%M:%L) - %m%n # Appender for Workers log4j.appender.WORKER_LOGGER=org.apache.log4j.RollingFileAppender @@ -108,78 +97,7 @@ log4j.appender.WORKER_LOGGER.File=${alluxio.logs.dir}/worker.log log4j.appender.WORKER_LOGGER.MaxFileSize=10MB log4j.appender.WORKER_LOGGER.MaxBackupIndex=100 log4j.appender.WORKER_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n - -# Remote appender for Job Master -log4j.appender.REMOTE_JOB_MASTER_LOGGER=org.apache.log4j.net.SocketAppender -log4j.appender.REMOTE_JOB_MASTER_LOGGER.Port=${alluxio.logserver.port} -log4j.appender.REMOTE_JOB_MASTER_LOGGER.RemoteHost=${alluxio.logserver.hostname} -log4j.appender.REMOTE_JOB_MASTER_LOGGER.ReconnectionDelay=10000 -log4j.appender.REMOTE_JOB_MASTER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter -log4j.appender.REMOTE_JOB_MASTER_LOGGER.filter.ID.ProcessType=JOB_MASTER -log4j.appender.REMOTE_JOB_MASTER_LOGGER.Threshold=WARN - -# Remote appender for Job Workers -log4j.appender.REMOTE_JOB_WORKER_LOGGER=org.apache.log4j.net.SocketAppender -log4j.appender.REMOTE_JOB_WORKER_LOGGER.Port=${alluxio.logserver.port} -log4j.appender.REMOTE_JOB_WORKER_LOGGER.RemoteHost=${alluxio.logserver.hostname} -log4j.appender.REMOTE_JOB_WORKER_LOGGER.ReconnectionDelay=10000 -log4j.appender.REMOTE_JOB_WORKER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter -log4j.appender.REMOTE_JOB_WORKER_LOGGER.filter.ID.ProcessType=JOB_WORKER -log4j.appender.REMOTE_JOB_WORKER_LOGGER.Threshold=WARN - -# Remote appender for Master -log4j.appender.REMOTE_MASTER_LOGGER=org.apache.log4j.net.SocketAppender -log4j.appender.REMOTE_MASTER_LOGGER.Port=${alluxio.logserver.port} -log4j.appender.REMOTE_MASTER_LOGGER.RemoteHost=${alluxio.logserver.hostname} -log4j.appender.REMOTE_MASTER_LOGGER.ReconnectionDelay=10000 -log4j.appender.REMOTE_MASTER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter -log4j.appender.REMOTE_MASTER_LOGGER.filter.ID.ProcessType=MASTER -log4j.appender.REMOTE_MASTER_LOGGER.Threshold=WARN - -# Remote appender for Secondary Master -log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER=org.apache.log4j.net.SocketAppender -log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.Port=${alluxio.logserver.port} -log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.RemoteHost=${alluxio.logserver.hostname} -log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.ReconnectionDelay=10000 -log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter -log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.filter.ID.ProcessType=SECONDARY_MASTER -log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.Threshold=WARN - -# Remote appender for Proxy -log4j.appender.REMOTE_PROXY_LOGGER=org.apache.log4j.net.SocketAppender -log4j.appender.REMOTE_PROXY_LOGGER.Port=${alluxio.logserver.port} -log4j.appender.REMOTE_PROXY_LOGGER.RemoteHost=${alluxio.logserver.hostname} -log4j.appender.REMOTE_PROXY_LOGGER.ReconnectionDelay=10000 -log4j.appender.REMOTE_PROXY_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter -log4j.appender.REMOTE_PROXY_LOGGER.filter.ID.ProcessType=PROXY -log4j.appender.REMOTE_PROXY_LOGGER.Threshold=WARN - -# Remote appender for Workers -log4j.appender.REMOTE_WORKER_LOGGER=org.apache.log4j.net.SocketAppender -log4j.appender.REMOTE_WORKER_LOGGER.Port=${alluxio.logserver.port} -log4j.appender.REMOTE_WORKER_LOGGER.RemoteHost=${alluxio.logserver.hostname} -log4j.appender.REMOTE_WORKER_LOGGER.ReconnectionDelay=10000 -log4j.appender.REMOTE_WORKER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter -log4j.appender.REMOTE_WORKER_LOGGER.filter.ID.ProcessType=WORKER -log4j.appender.REMOTE_WORKER_LOGGER.Threshold=WARN - -# (Local) appender for log server itself -log4j.appender.LOGSERVER_LOGGER=org.apache.log4j.RollingFileAppender -log4j.appender.LOGSERVER_LOGGER.File=${alluxio.logs.dir}/logserver.log -log4j.appender.LOGSERVER_LOGGER.MaxFileSize=10MB -log4j.appender.LOGSERVER_LOGGER.MaxBackupIndex=100 -log4j.appender.LOGSERVER_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.LOGSERVER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n - -# (Local) appender for log server to log on behalf of log clients -# No need to configure file path because log server will dynamically -# figure out for each appender. -log4j.appender.LOGSERVER_CLIENT_LOGGER=org.apache.log4j.RollingFileAppender -log4j.appender.LOGSERVER_CLIENT_LOGGER.MaxFileSize=10MB -log4j.appender.LOGSERVER_CLIENT_LOGGER.MaxBackupIndex=100 -log4j.appender.LOGSERVER_CLIENT_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.LOGSERVER_CLIENT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Appender for User log4j.appender.USER_LOGGER=org.apache.log4j.RollingFileAppender @@ -187,7 +105,7 @@ log4j.appender.USER_LOGGER.File=${alluxio.user.logs.dir}/user_${user.name}.log log4j.appender.USER_LOGGER.MaxFileSize=10MB log4j.appender.USER_LOGGER.MaxBackupIndex=10 log4j.appender.USER_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Appender for Fuse log4j.appender.FUSE_LOGGER=org.apache.log4j.RollingFileAppender @@ -195,7 +113,7 @@ log4j.appender.FUSE_LOGGER.File=${alluxio.logs.dir}/fuse.log log4j.appender.FUSE_LOGGER.MaxFileSize=100MB log4j.appender.FUSE_LOGGER.MaxBackupIndex=10 log4j.appender.FUSE_LOGGER.layout=org.apache.log4j.PatternLayout -log4j.appender.FUSE_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n +log4j.appender.FUSE_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n # Disable noisy DEBUG logs log4j.logger.com.amazonaws.util.EC2MetadataUtils=OFF diff --git a/core/client/fs/pom.xml b/core/client/fs/pom.xml deleted file mode 100644 index bf78e6e2eff6..000000000000 --- a/core/client/fs/pom.xml +++ /dev/null @@ -1,119 +0,0 @@ - - - 4.0.0 - - org.alluxio - alluxio-core-client - 2.10.0-SNAPSHOT - - alluxio-core-client-fs - jar - Alluxio Core - Client - File System - File System Client of Alluxio Core - - - - - ${project.parent.parent.parent.basedir}/build - false - - - - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - commons-codec - commons-codec - - - commons-io - commons-io - - - io.dropwizard.metrics - metrics-core - - - io.grpc - grpc-api - - - io.grpc - grpc-core - - - io.grpc - grpc-stub - - - io.netty - netty-all - - - org.rocksdb - rocksdbjni - - - - - org.alluxio - alluxio-core-common - ${project.version} - - - org.alluxio - alluxio-core-transport - ${project.version} - - - - - org.alluxio - alluxio-core-common - ${project.version} - test-jar - test - - - org.alluxio - alluxio-underfs-local - ${project.version} - test - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - - diff --git a/core/client/fs/src/main/java/alluxio/client/block/BlockMasterClient.java b/core/client/fs/src/main/java/alluxio/client/block/BlockMasterClient.java deleted file mode 100644 index bcc5864d71bc..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/BlockMasterClient.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block; - -import alluxio.Client; -import alluxio.client.block.options.GetWorkerReportOptions; -import alluxio.grpc.WorkerLostStorageInfo; -import alluxio.master.MasterClientContext; -import alluxio.wire.BlockInfo; -import alluxio.wire.BlockMasterInfo; -import alluxio.wire.BlockMasterInfo.BlockMasterInfoField; -import alluxio.wire.WorkerInfo; - -import java.io.IOException; -import java.util.List; -import java.util.Set; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A client to use for interacting with a block master. - */ -@ThreadSafe -public interface BlockMasterClient extends Client { - - /** - * Factory for {@link BlockMasterClient}. - */ - class Factory { - - private Factory() {} // prevent instantiation - - /** - * Factory method for {@link BlockMasterClient}. - * - * @param conf master client configuration - * @return a new {@link BlockMasterClient} instance - */ - public static BlockMasterClient create(MasterClientContext conf) { - return new RetryHandlingBlockMasterClient(conf); - } - } - - /** - * Gets the worker information of live workers(support older version Alluxio server). - * - * @return a list of worker information - */ - List getWorkerInfoList() throws IOException; - - /** - * Remove the metadata of a decommissioned worker. - * - * @param workerName contains a string, representing the workerName - */ - void removeDecommissionedWorker(String workerName) throws IOException; - - /** - * Gets the worker information of selected workers and selected fields for report CLI. - * - * @param options the client defined worker and field ranges - * @return a list of worker information - */ - List getWorkerReport(final GetWorkerReportOptions options) - throws IOException; - - /** - * @return a list of worker lost storage information - */ - List getWorkerLostStorage() throws IOException; - - /** - * Returns the {@link BlockInfo} for a block id. - * - * @param blockId the block id to get the BlockInfo for - * @return the {@link BlockInfo} - */ - BlockInfo getBlockInfo(final long blockId) throws IOException; - - /** - * @param fields optional list of fields to query; if null all fields will be queried - * @return the {@link BlockMasterInfo} block master information - */ - BlockMasterInfo getBlockMasterInfo(final Set fields) throws IOException; - - /** - * Gets the total Alluxio capacity in bytes, on all the tiers of all the workers. - * - * @return total capacity in bytes - */ - long getCapacityBytes() throws IOException; - - /** - * Gets the total amount of used space in bytes, on all the tiers of all the workers. - * - * @return amount of used space in bytes - */ - long getUsedBytes() throws IOException; -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/BlockStoreClient.java b/core/client/fs/src/main/java/alluxio/client/block/BlockStoreClient.java deleted file mode 100644 index 337cc64e1ccf..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/BlockStoreClient.java +++ /dev/null @@ -1,388 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block; - -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; - -import alluxio.client.WriteType; -import alluxio.client.block.policy.BlockLocationPolicy; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.client.block.stream.BlockInStream; -import alluxio.client.block.stream.BlockInStream.BlockInStreamSource; -import alluxio.client.block.stream.BlockOutStream; -import alluxio.client.block.stream.DataWriter; -import alluxio.client.block.util.BlockLocationUtils; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.URIStatus; -import alluxio.client.file.options.InStreamOptions; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.collections.Pair; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.PreconditionMessage; -import alluxio.exception.status.UnavailableException; -import alluxio.network.TieredIdentityFactory; -import alluxio.resource.CloseableResource; -import alluxio.wire.BlockInfo; -import alluxio.wire.BlockLocation; -import alluxio.wire.TieredIdentity; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Alluxio Block Store client. This is an internal client for all block level operations in Alluxio. - * An instance of this class can be obtained via {@link BlockStoreClient} constructors. - */ -@ThreadSafe -public final class BlockStoreClient { - private static final Logger LOG = LoggerFactory.getLogger(BlockStoreClient.class); - - private final FileSystemContext mContext; - private final TieredIdentity mTieredIdentity; - - /** - * Creates an Alluxio block store with default local hostname. - * - * @param context the file system context - * @return the {@link BlockStoreClient} created - */ - public static BlockStoreClient create(FileSystemContext context) { - return new BlockStoreClient(context, - TieredIdentityFactory.localIdentity(context.getClusterConf())); - } - - /** - * Creates an Alluxio block store. - * - * @param context the file system context - * @param tieredIdentity the tiered identity - */ - @VisibleForTesting - BlockStoreClient(FileSystemContext context, TieredIdentity tieredIdentity) { - mContext = context; - mTieredIdentity = tieredIdentity; - } - - /** - * Gets the block info of a block, if it exists. - * - * @param blockId the blockId to obtain information about - * @return a {@link BlockInfo} containing the metadata of the block - */ - public BlockInfo getInfo(long blockId) throws IOException { - try (CloseableResource masterClientResource = - mContext.acquireBlockMasterClientResource()) { - return masterClientResource.get().getBlockInfo(blockId); - } - } - - /** - * Gets a stream to read the data of a block. This method is primarily responsible for - * determining the data source and type of data source. The latest BlockInfo will be fetched - * from the master to ensure the locations are up-to-date. - * - * @param blockId the id of the block to read - * @param options the options associated with the read request - * @return a stream which reads from the beginning of the block - */ - public BlockInStream getInStream(long blockId, InStreamOptions options) throws IOException { - return getInStream(blockId, options, ImmutableMap.of()); - } - - /** - * Gets a stream to read the data of a block. This method is primarily responsible for - * determining the data source and type of data source. The latest BlockInfo will be fetched - * from the master to ensure the locations are up-to-date. It takes a map of failed workers and - * their most recently failed time and tries to update it when BlockInStream created failed, - * attempting to avoid reading from a recently failed worker. - * - * @param blockId the id of the block to read - * @param options the options associated with the read request - * @param failedWorkers the map of workers addresses to most recent failure time - * @return a stream which reads from the beginning of the block - */ - public BlockInStream getInStream(long blockId, InStreamOptions options, - Map failedWorkers) throws IOException { - // Get the latest block info from master - BlockInfo info = getInfo(blockId); - return getInStream(info, options, failedWorkers); - } - - /** - * {@link #getInStream(long, InStreamOptions, Map)}. - * - * @param info the block info - * @param options the options associated with the read request - * @param failedWorkers the map of workers addresses to most recent failure time - * @return a stream which reads from the beginning of the block - */ - public BlockInStream getInStream(BlockInfo info, InStreamOptions options, - Map failedWorkers) throws IOException { - Pair dataSourceAndType = getDataSourceAndType(info, - options.getStatus(), options.getUfsReadLocationPolicy(), failedWorkers); - WorkerNetAddress dataSource = dataSourceAndType.getFirst(); - BlockInStreamSource dataSourceType = dataSourceAndType.getSecond(); - try { - return BlockInStream.create(mContext, info, dataSource, dataSourceType, options); - } catch (UnavailableException e) { - //When BlockInStream created failed, it will update the passed-in failedWorkers - //to attempt to avoid reading from this failed worker in next try. - LOG.info("Added {} to failedWorkers for {}", dataSource, e.toString()); - failedWorkers.put(dataSource, System.currentTimeMillis()); - throw e; - } - } - - /** - * Gets the data source and type of data source of a block. This method is primarily responsible - * for determining the data source and type of data source. It takes a map of failed workers and - * their most recently failed time and tries to update it when BlockInStream created failed, - * attempting to avoid reading from a recently failed worker. - * - * @param info the info of the block to read - * @param status the URIStatus associated with the read request - * @param policy the policy determining the Alluxio worker location - * @param failedWorkers the map of workers addresses to most recent failure time - * @return the data source and type of data source of the block - */ - public Pair getDataSourceAndType(BlockInfo info, - URIStatus status, BlockLocationPolicy policy, Map failedWorkers) - throws IOException { - List locations = info.getLocations(); - List blockWorkerInfo = Collections.emptyList(); - // Initial target workers to read the block given the block locations. - Set workerPool; - // Note that, it is possible that the blocks have been written as UFS blocks - if (status.isPersisted() - || status.getPersistenceState().equals("TO_BE_PERSISTED")) { - blockWorkerInfo = mContext.getCachedWorkers(); - if (blockWorkerInfo.isEmpty()) { - throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()); - } - workerPool = blockWorkerInfo.stream().map(BlockWorkerInfo::getNetAddress).collect(toSet()); - } else { - if (locations.isEmpty()) { - blockWorkerInfo = mContext.getCachedWorkers(); - if (blockWorkerInfo.isEmpty()) { - throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()); - } - throw new UnavailableException(MessageFormat - .format("Block {0} is unavailable in both Alluxio and UFS.", info.getBlockId())); - } - workerPool = locations.stream().map(BlockLocation::getWorkerAddress).collect(toSet()); - } - // Workers to read the block, after considering failed workers. - Set workers = handleFailedWorkers(workerPool, failedWorkers); - // TODO(calvin, jianjian): Consider containing these two variables in one object - BlockInStreamSource dataSourceType = null; - WorkerNetAddress dataSource = null; - locations = locations.stream() - .filter(location -> workers.contains(location.getWorkerAddress())).collect(toList()); - // First try to read data from Alluxio - if (!locations.isEmpty()) { - // TODO(calvin): Get location via a policy - List tieredLocations = - locations.stream().map(BlockLocation::getWorkerAddress) - .collect(toList()); - Collections.shuffle(tieredLocations); - Optional> nearest = - BlockLocationUtils.nearest(mTieredIdentity, tieredLocations, mContext.getClusterConf()); - if (nearest.isPresent()) { - dataSource = nearest.get().getFirst(); - dataSourceType = nearest.get().getSecond() ? mContext.hasProcessLocalWorker() - ? BlockInStreamSource.PROCESS_LOCAL : BlockInStreamSource.NODE_LOCAL - : BlockInStreamSource.REMOTE; - } - } - // Can't get data from Alluxio, get it from the UFS instead - if (dataSource == null) { - dataSourceType = BlockInStreamSource.UFS; - Preconditions.checkNotNull(policy, "The UFS read location policy is not specified"); - blockWorkerInfo = blockWorkerInfo.stream() - .filter(workerInfo -> workers.contains(workerInfo.getNetAddress())).collect(toList()); - GetWorkerOptions getWorkerOptions = GetWorkerOptions.defaults() - .setBlockInfo(new BlockInfo() - .setBlockId(info.getBlockId()) - .setLength(info.getLength()) - .setLocations(locations)) - .setBlockWorkerInfos(blockWorkerInfo); - dataSource = policy.getWorker(getWorkerOptions).orElseThrow( - () -> new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()) - ); - if (mContext.hasProcessLocalWorker() - && dataSource.equals(mContext.getNodeLocalWorker())) { - dataSourceType = BlockInStreamSource.PROCESS_LOCAL; - LOG.debug("Create BlockInStream to read data from UFS through process local worker {}", - dataSource); - } else { - LOG.debug("Create BlockInStream to read data from UFS through worker {} " - + "(client embedded in local worker process: {}," - + "client co-located with worker in different processes: {}, " - + "local worker address: {})", - dataSource, mContext.hasProcessLocalWorker(), mContext.hasNodeLocalWorker(), - mContext.hasNodeLocalWorker() ? mContext.getNodeLocalWorker() : "N/A"); - } - } - return new Pair<>(dataSource, dataSourceType); - } - - private Set handleFailedWorkers(Set workers, - Map failedWorkers) { - if (workers.isEmpty()) { - return Collections.emptySet(); - } - Set nonFailed = - workers.stream().filter(worker -> !failedWorkers.containsKey(worker)).collect(toSet()); - if (nonFailed.isEmpty()) { - return Collections.singleton(workers.stream() - .min(Comparator.comparingLong(failedWorkers::get)).get()); - } - return nonFailed; - } - - /** - * Gets a stream to write data to a block. The stream can only be backed by Alluxio storage. - * - * @param blockId the block to write - * @param blockSize the standard block size to write - * @param address the address of the worker to write the block to, fails if the worker cannot - * serve the request - * @param options the output stream options - * @return an {@link BlockOutStream} which can be used to write data to the block in a streaming - * fashion - */ - public BlockOutStream getOutStream(long blockId, long blockSize, WorkerNetAddress address, - OutStreamOptions options) throws IOException { - // No specified location to write to. - Preconditions.checkNotNull(address, "address"); - LOG.debug("Create BlockOutStream for {} of block size {} at address {}, using options: {}", - blockId, blockSize, address, options); - DataWriter dataWriter = - DataWriter.Factory.create(mContext, blockId, blockSize, address, options); - return new BlockOutStream(dataWriter, blockSize, address); - } - - /** - * Gets a stream to write data to a block based on the options. The stream can only be backed by - * Alluxio storage. - * - * @param blockId the block to write - * @param blockSize the standard block size to write - * @param options the output stream option - * @return a {@link BlockOutStream} which can be used to write data to the block in a streaming - * fashion - */ - public BlockOutStream getOutStream(long blockId, long blockSize, OutStreamOptions options) - throws IOException { - WorkerNetAddress address; - BlockLocationPolicy locationPolicy = Preconditions.checkNotNull(options.getLocationPolicy(), - PreconditionMessage.BLOCK_WRITE_LOCATION_POLICY_UNSPECIFIED); - GetWorkerOptions workerOptions = GetWorkerOptions.defaults() - .setBlockInfo(new BlockInfo().setBlockId(blockId).setLength(blockSize)) - .setBlockWorkerInfos(new ArrayList<>(mContext.getCachedWorkers())); - - // The number of initial copies depends on the write type: if ASYNC_THROUGH, it is the property - // "alluxio.user.file.replication.durable" before data has been persisted; otherwise - // "alluxio.user.file.replication.min" - int initialReplicas = (options.getWriteType() == WriteType.ASYNC_THROUGH - && options.getReplicationDurable() > options.getReplicationMin()) - ? options.getReplicationDurable() : options.getReplicationMin(); - if (initialReplicas <= 1) { - address = locationPolicy.getWorker(workerOptions).orElseThrow( - () -> { - try { - if (mContext.getCachedWorkers().isEmpty()) { - return new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()); - } - } catch (IOException e) { - return e; - } - return new UnavailableException( - ExceptionMessage.NO_SPACE_FOR_BLOCK_ON_WORKER.getMessage(blockSize)); - } - ); - // TODO(ggezer): Retry on another worker if this has no storage. - return getOutStream(blockId, blockSize, address, options); - } - - // Group different block workers by their hostnames - Map> blockWorkersByHost = new HashMap<>(); - for (BlockWorkerInfo blockWorker : workerOptions.getBlockWorkerInfos()) { - String hostName = blockWorker.getNetAddress().getHost(); - if (blockWorkersByHost.containsKey(hostName)) { - blockWorkersByHost.get(hostName).add(blockWorker); - } else { - blockWorkersByHost.put(hostName, com.google.common.collect.Sets.newHashSet(blockWorker)); - } - } - - // Select N workers on different hosts where N is the value of initialReplicas for this block - List workerAddressList = new ArrayList<>(); - List updatedInfos = Lists.newArrayList(workerOptions.getBlockWorkerInfos()); - for (int i = 0; i < initialReplicas; i++) { - locationPolicy.getWorker(workerOptions).ifPresent(workerAddress -> { - workerAddressList.add(workerAddress); - updatedInfos.removeAll(blockWorkersByHost.get(workerAddress.getHost())); - workerOptions.setBlockWorkerInfos(updatedInfos); - }); - } - if (workerAddressList.size() < initialReplicas) { - throw new alluxio.exception.status.ResourceExhaustedException(String.format( - "Not enough workers for replications, %d workers selected but %d required", - workerAddressList.size(), initialReplicas)); - } - return BlockOutStream - .createReplicatedBlockOutStream(mContext, blockId, blockSize, workerAddressList, options); - } - - /** - * Gets the total capacity of Alluxio's BlockStore. - * - * @return the capacity in bytes - */ - public long getCapacityBytes() throws IOException { - try (CloseableResource blockMasterClientResource = - mContext.acquireBlockMasterClientResource()) { - return blockMasterClientResource.get().getCapacityBytes(); - } - } - - /** - * Gets the used bytes of Alluxio's BlockStore. - * - * @return the used bytes of Alluxio's BlockStore - */ - public long getUsedBytes() throws IOException { - try (CloseableResource blockMasterClientResource = - mContext.acquireBlockMasterClientResource()) { - return blockMasterClientResource.get().getUsedBytes(); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/BlockWorkerInfo.java b/core/client/fs/src/main/java/alluxio/client/block/BlockWorkerInfo.java deleted file mode 100644 index a60cb2a8e00e..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/BlockWorkerInfo.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block; - -import alluxio.annotation.PublicApi; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Information of an active block worker. - */ -@PublicApi -@ThreadSafe -public final class BlockWorkerInfo { - private final WorkerNetAddress mNetAddress; - private final long mCapacityBytes; - private final long mUsedBytes; - - /** - * Constructs the block worker information. - * - * @param netAddress the address of the worker - * @param capacityBytes the capacity of the worker in bytes - * @param usedBytes the used bytes of the worker - */ - public BlockWorkerInfo(WorkerNetAddress netAddress, long capacityBytes, long usedBytes) { - mNetAddress = Preconditions.checkNotNull(netAddress, "netAddress"); - mCapacityBytes = capacityBytes; - mUsedBytes = usedBytes; - } - - /** - * @return the address of the worker - */ - public WorkerNetAddress getNetAddress() { - return mNetAddress; - } - - /** - * @return the capacity of the worker in bytes - */ - public long getCapacityBytes() { - return mCapacityBytes; - } - - /** - * @return the used bytes of the worker - */ - public long getUsedBytes() { - return mUsedBytes; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("netAddress", mNetAddress) - .add("capacityBytes", mCapacityBytes) - .add("usedBytes", mUsedBytes) - .toString(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/RetryHandlingBlockMasterClient.java b/core/client/fs/src/main/java/alluxio/client/block/RetryHandlingBlockMasterClient.java deleted file mode 100644 index 2824c36dcb63..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/RetryHandlingBlockMasterClient.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block; - -import alluxio.AbstractMasterClient; -import alluxio.Constants; -import alluxio.client.block.options.GetWorkerReportOptions; -import alluxio.grpc.BlockMasterClientServiceGrpc; -import alluxio.grpc.GetBlockInfoPRequest; -import alluxio.grpc.GetBlockMasterInfoPOptions; -import alluxio.grpc.GetCapacityBytesPOptions; -import alluxio.grpc.GetUsedBytesPOptions; -import alluxio.grpc.GetWorkerInfoListPOptions; -import alluxio.grpc.GetWorkerLostStoragePOptions; -import alluxio.grpc.GrpcUtils; -import alluxio.grpc.RemoveDecommissionedWorkerPOptions; -import alluxio.grpc.ServiceType; -import alluxio.grpc.WorkerLostStorageInfo; -import alluxio.master.MasterClientContext; -import alluxio.wire.BlockInfo; -import alluxio.wire.BlockMasterInfo; -import alluxio.wire.BlockMasterInfo.BlockMasterInfoField; -import alluxio.wire.WorkerInfo; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A wrapper for the gRPC client to interact with the block master, used by alluxio clients. - */ -@ThreadSafe -public final class RetryHandlingBlockMasterClient extends AbstractMasterClient - implements BlockMasterClient { - private static final Logger RPC_LOG = LoggerFactory.getLogger(BlockMasterClient.class); - private BlockMasterClientServiceGrpc.BlockMasterClientServiceBlockingStub mClient = null; - - /** - * Creates a new block master client. - * - * @param conf master client configuration - */ - public RetryHandlingBlockMasterClient(MasterClientContext conf) { - super(conf); - } - - @Override - protected ServiceType getRemoteServiceType() { - return ServiceType.BLOCK_MASTER_CLIENT_SERVICE; - } - - @Override - protected String getServiceName() { - return Constants.BLOCK_MASTER_CLIENT_SERVICE_NAME; - } - - @Override - protected long getServiceVersion() { - return Constants.BLOCK_MASTER_CLIENT_SERVICE_VERSION; - } - - @Override - protected void afterConnect() { - mClient = BlockMasterClientServiceGrpc.newBlockingStub(mChannel); - } - - @Override - public List getWorkerInfoList() throws IOException { - return retryRPC(() -> { - List result = new ArrayList<>(); - for (alluxio.grpc.WorkerInfo workerInfo : mClient - .getWorkerInfoList(GetWorkerInfoListPOptions.getDefaultInstance()) - .getWorkerInfosList()) { - result.add(GrpcUtils.fromProto(workerInfo)); - } - return result; - }, RPC_LOG, "GetWorkerInfoList", ""); - } - - @Override - public void removeDecommissionedWorker(String workerName) throws IOException { - retryRPC(() -> mClient.removeDecommissionedWorker(RemoveDecommissionedWorkerPOptions - .newBuilder().setWorkerName(workerName).build()), - RPC_LOG, "RemoveDecommissionedWorker", ""); - } - - @Override - public List getWorkerReport(final GetWorkerReportOptions options) - throws IOException { - return retryRPC(() -> { - List result = new ArrayList<>(); - for (alluxio.grpc.WorkerInfo workerInfo : mClient.getWorkerReport(options.toProto()) - .getWorkerInfosList()) { - result.add(GrpcUtils.fromProto(workerInfo)); - } - return result; - }, RPC_LOG, "GetWorkerReport", "options=%s", options); - } - - @Override - public List getWorkerLostStorage() throws IOException { - return retryRPC(() -> mClient - .getWorkerLostStorage(GetWorkerLostStoragePOptions.getDefaultInstance()) - .getWorkerLostStorageInfoList(), - RPC_LOG, "GetWorkerLostStorage", ""); - } - - @Override - public BlockInfo getBlockInfo(final long blockId) throws IOException { - return retryRPC(() -> GrpcUtils.fromProto( - mClient.getBlockInfo(GetBlockInfoPRequest.newBuilder().setBlockId(blockId).build()) - .getBlockInfo()), RPC_LOG, "GetBlockInfo", "blockId=%d", blockId); - } - - @Override - public BlockMasterInfo getBlockMasterInfo(final Set fields) - throws IOException { - return retryRPC(() -> BlockMasterInfo - .fromProto(mClient.getBlockMasterInfo(GetBlockMasterInfoPOptions.newBuilder() - .addAllFilters( - fields.stream().map(BlockMasterInfoField::toProto).collect(Collectors.toList())) - .build()).getBlockMasterInfo()), RPC_LOG, "GetBlockMasterInfo", "fields=%s", fields); - } - - @Override - public long getCapacityBytes() throws IOException { - return retryRPC(() -> mClient - .getCapacityBytes(GetCapacityBytesPOptions.getDefaultInstance()).getBytes(), - RPC_LOG, "GetCapacityBytes", ""); - } - - @Override - public long getUsedBytes() throws IOException { - return retryRPC( - () -> mClient.getUsedBytes(GetUsedBytesPOptions.getDefaultInstance()).getBytes(), - RPC_LOG, "GetUsedBytes", ""); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/BlockLocationPolicy.java b/core/client/fs/src/main/java/alluxio/client/block/policy/BlockLocationPolicy.java deleted file mode 100644 index 28ab8655c2f4..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/BlockLocationPolicy.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.annotation.PublicApi; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.util.CommonUtils; -import alluxio.wire.WorkerNetAddress; - -import java.util.Optional; - -/** - *

- * Interface for determining the Alluxio worker location to serve a block write or UFS block read. - *

- * - *

- * {@link alluxio.client.file.FileInStream} uses this to determine where to read a UFS block. - *

- * - *

- * A policy must have an empty constructor to be used as default policy. - *

- */ -@PublicApi -public interface BlockLocationPolicy { - - /** - * The factory for the {@link BlockLocationPolicy}. - */ - class Factory { - private Factory() {} // prevent instantiation - - /** - * Factory for creating {@link BlockLocationPolicy}. - * - * @param conf Alluxio configuration - * @return a new instance of {@link BlockLocationPolicy} - */ - public static BlockLocationPolicy create(Class blockLocationPolicyClass, - AlluxioConfiguration conf) { - try { - Class clazz = blockLocationPolicyClass - .asSubclass(BlockLocationPolicy.class); - return CommonUtils.createNewClassInstance(clazz, new Class[] {AlluxioConfiguration.class}, - new Object[] {conf}); - } catch (ClassCastException e) { - throw new RuntimeException(e); - } - } - } - - /** - * Gets the worker's network address for serving operations requested for the block. - * - * @param options the options to get a block worker network address for a block - * @return the address of the worker to write to, or empty if no worker can be selected - */ - Optional getWorker(GetWorkerOptions options); -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/CapacityBaseRandomPolicy.java b/core/client/fs/src/main/java/alluxio/client/block/policy/CapacityBaseRandomPolicy.java deleted file mode 100644 index 11c1c0dc2ce9..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/CapacityBaseRandomPolicy.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.wire.WorkerNetAddress; - -import java.util.Optional; -import java.util.TreeMap; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicLong; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Randomly distribute workload based on the worker capacities so bigger workers get more requests. - * The randomness is based on the capacity instead of availability because in the long run, - * all workers will be filled up and have availability close to 0. - * We do not want the policy to degenerate to all workers having the same chance. - */ -@ThreadSafe -public class CapacityBaseRandomPolicy implements BlockLocationPolicy { - - /** - * Constructs a new {@link CapacityBaseRandomPolicy} - * needed for instantiation in {@link BlockLocationPolicy.Factory}. - * - * @param ignoredConf is unused - */ - public CapacityBaseRandomPolicy(AlluxioConfiguration ignoredConf) { - } - - @Override - public Optional getWorker(GetWorkerOptions options) { - Iterable blockWorkerInfos = options.getBlockWorkerInfos(); - // All the capacities will form a ring of continuous intervals - // And we throw a die in the ring and decide which worker to pick - // For example if worker1 has capacity 10, worker2 has 20, worker3 has 40, - // the ring will look like [0, 10), [10, 30), [30, 70). - // A key in the map is the LHS of a range. - // So the map will look like {0 -> w1, 10 -> w2, 30 -> w3}. - TreeMap rangeStartMap = new TreeMap<>(); - AtomicLong totalCapacity = new AtomicLong(0L); - blockWorkerInfos.forEach(workerInfo -> { - if (workerInfo.getCapacityBytes() > 0) { - long capacityRangeStart = totalCapacity.getAndAdd(workerInfo.getCapacityBytes()); - rangeStartMap.put(capacityRangeStart, workerInfo); - } - }); - if (totalCapacity.get() == 0L) { - return Optional.empty(); - } - long randomLong = randomInCapacity(totalCapacity.get()); - return Optional.of(rangeStartMap.floorEntry(randomLong).getValue().getNetAddress()); - } - - protected long randomInCapacity(long totalCapacity) { - return ThreadLocalRandom.current().nextLong(totalCapacity); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/DeterministicHashPolicy.java b/core/client/fs/src/main/java/alluxio/client/block/policy/DeterministicHashPolicy.java deleted file mode 100644 index 1ec409a7f19f..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/DeterministicHashPolicy.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.Random; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This policy maps the blockId to several deterministic Alluxio workers. The number of workers a - * block can be mapped to can be passed through the constructor. The default is 1. It skips the - * workers that do not have enough capacity to hold the block. - * - * This policy is useful for limiting the amount of replication that occurs when reading blocks from - * the UFS with high concurrency. With 30 workers and 100 remote clients reading the same block - * concurrently, the replication level for the block would get close to 30 as each worker reads - * and caches the block for one or more clients. If the clients use DeterministicHashPolicy with - * 3 shards, the 100 clients will split their reads between just 3 workers, so that the replication - * level for the block will be only 3 when the data is first loaded. - * - * Note that the hash function relies on the number of workers in the cluster, so if the number of - * workers changes, the workers chosen by the policy for a given block will likely change. - */ -@NotThreadSafe -public final class DeterministicHashPolicy implements BlockLocationPolicy { - /** The default number of shards to serve a block. */ - private final int mShards; - private final Random mRandom = new Random(); - private final HashFunction mHashFunc = Hashing.md5(); - - /** - * Constructs a new {@link DeterministicHashPolicy} - * needed for instantiation in {@link BlockLocationPolicy.Factory}. - * - * @param conf Alluxio configuration - */ - public DeterministicHashPolicy(AlluxioConfiguration conf) { - int numShards = - conf.getInt(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY_DETERMINISTIC_HASH_SHARDS); - Preconditions.checkArgument(numShards >= 1); - mShards = numShards; - } - - @Override - public Optional getWorker(GetWorkerOptions options) { - List workerInfos = Lists.newArrayList(options.getBlockWorkerInfos()); - workerInfos.sort((o1, o2) -> - o1.getNetAddress().toString().compareToIgnoreCase(o2.getNetAddress().toString())); - HashMap blockWorkerInfoMap = new HashMap<>(); - for (BlockWorkerInfo workerInfo : options.getBlockWorkerInfos()) { - blockWorkerInfoMap.put(workerInfo.getNetAddress(), workerInfo); - } - - List workers = new ArrayList<>(); - // Try the next one if the worker mapped from the blockId doesn't work until all the workers - // are examined. - int hv = - Math.abs(mHashFunc.newHasher().putLong(options.getBlockInfo().getBlockId()).hash().asInt()); - int index = hv % workerInfos.size(); - for (BlockWorkerInfo ignored : workerInfos) { - WorkerNetAddress candidate = workerInfos.get(index).getNetAddress(); - BlockWorkerInfo workerInfo = blockWorkerInfoMap.get(candidate); - if (workerInfo != null - && workerInfo.getCapacityBytes() >= options.getBlockInfo().getLength()) { - workers.add(candidate); - if (workers.size() >= mShards) { - break; - } - } - index = (index + 1) % workerInfos.size(); - } - return workers.isEmpty() ? Optional.empty() : - Optional.of(workers.get(mRandom.nextInt(workers.size()))); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof DeterministicHashPolicy)) { - return false; - } - DeterministicHashPolicy that = (DeterministicHashPolicy) o; - return Objects.equal(mShards, that.mShards); - } - - @Override - public int hashCode() { - return Objects.hashCode(mShards); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("shards", mShards).toString(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/LocalFirstAvoidEvictionPolicy.java b/core/client/fs/src/main/java/alluxio/client/block/policy/LocalFirstAvoidEvictionPolicy.java deleted file mode 100644 index 1abb0f5d0f6b..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/LocalFirstAvoidEvictionPolicy.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.wire.TieredIdentity; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.collect.Lists; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A policy that returns the local worker first, and if the local worker doesn't - * exist or doesn't have enough availability, will select the nearest worker from the active - * workers list with sufficient availability. - * - * The definition of 'nearest worker' is based on {@link alluxio.wire.TieredIdentity}. - * @see alluxio.util.TieredIdentityUtils#nearest - * - * The calculation of which worker gets selected is done for each block write. - * - * The {@link alluxio.conf.PropertyKey.USER_BLOCK_AVOID_EVICTION_POLICY_RESERVED_BYTES} - * (alluxio.user.block.avoid.eviction.policy.reserved.size.bytes) - * is used as buffer space on each worker when calculating available space - * to store each block. - */ -@ThreadSafe -public final class LocalFirstAvoidEvictionPolicy implements BlockLocationPolicy { - private final LocalFirstPolicy mPolicy; - private final long mBlockCapacityReserved; - - /** - * Constructs a new {@link LocalFirstAvoidEvictionPolicy}. - * - * @param conf Alluxio configuration - */ - public LocalFirstAvoidEvictionPolicy(AlluxioConfiguration conf) { - this(conf.getBytes(alluxio.conf.PropertyKey.USER_BLOCK_AVOID_EVICTION_POLICY_RESERVED_BYTES), - conf); - } - - LocalFirstAvoidEvictionPolicy(long blockCapacityReserved, - AlluxioConfiguration conf) { - mPolicy = LocalFirstPolicy.create(conf); - mBlockCapacityReserved = blockCapacityReserved; - } - - @VisibleForTesting - LocalFirstAvoidEvictionPolicy(long blockCapacityReserved, TieredIdentity identity, - AlluxioConfiguration conf) { - mPolicy = new LocalFirstPolicy(identity, conf); - mBlockCapacityReserved = blockCapacityReserved; - } - - @Override - public Optional getWorker(GetWorkerOptions options) { - List allWorkers = Lists.newArrayList(options.getBlockWorkerInfos()); - // Prefer workers with enough availability. - List workers = allWorkers.stream() - .filter(worker -> getAvailableBytes(worker) >= options.getBlockInfo().getLength()) - .collect(Collectors.toList()); - if (workers.isEmpty()) { - workers = allWorkers; - } - GetWorkerOptions filteredWorkers = GetWorkerOptions.defaults() - .setBlockInfo(options.getBlockInfo()) - .setBlockWorkerInfos(workers); - return mPolicy.getWorker(filteredWorkers); - } - - /** - * Calculate the available bytes for a worker with the added buffer of - * {@link alluxio.conf.PropertyKey.USER_BLOCK_AVOID_EVICTION_POLICY_RESERVED_BYTES} - * (alluxio.user.block.avoid.eviction.policy.reserved.size.bytes) - * - * Since the information of BlockWorkerInfo is updated after a file - * completes a write, mCapacityBytes minus mUsedBytes may not be the true available bytes. - * - * @param workerInfo BlockWorkerInfo of the worker - * @return the available bytes of the worker - */ - private long getAvailableBytes(BlockWorkerInfo workerInfo) { - long capacityBytes = workerInfo.getCapacityBytes(); - long usedBytes = workerInfo.getUsedBytes(); - return capacityBytes - usedBytes - mBlockCapacityReserved; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof LocalFirstAvoidEvictionPolicy)) { - return false; - } - LocalFirstAvoidEvictionPolicy that = (LocalFirstAvoidEvictionPolicy) o; - return Objects.equal(mPolicy, that.mPolicy) - && Objects.equal(mBlockCapacityReserved, that.mBlockCapacityReserved); - } - - @Override - public int hashCode() { - return Objects.hashCode(mPolicy); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("policy", mPolicy) - .add("blockCapacityReservered", mBlockCapacityReserved) - .toString(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/LocalFirstPolicy.java b/core/client/fs/src/main/java/alluxio/client/block/policy/LocalFirstPolicy.java deleted file mode 100644 index 10d0aadfae2c..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/LocalFirstPolicy.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.client.block.util.BlockLocationUtils; -import alluxio.collections.Pair; -import alluxio.conf.AlluxioConfiguration; -import alluxio.network.TieredIdentityFactory; -import alluxio.wire.TieredIdentity; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; - -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A policy that returns the local worker first, and if the local worker doesn't - * exist or doesn't have enough capacity, will select the nearest worker from the active - * workers list with sufficient capacity. - * - * The definition of 'nearest worker' is based on {@link alluxio.wire.TieredIdentity}. - * @see alluxio.util.TieredIdentityUtils#nearest - * - * The calculation of which worker gets selected is done for each block write. - */ -@ThreadSafe -public final class LocalFirstPolicy implements BlockLocationPolicy { - private final TieredIdentity mTieredIdentity; - private final AlluxioConfiguration mConf; - - /** - * Constructs a new {@link LocalFirstPolicy}. - * - * @param conf Alluxio configuration - */ - public LocalFirstPolicy(AlluxioConfiguration conf) { - mTieredIdentity = TieredIdentityFactory.localIdentity(conf); - mConf = conf; - } - - static LocalFirstPolicy create(AlluxioConfiguration conf) { - return new LocalFirstPolicy(conf); - } - - @VisibleForTesting - LocalFirstPolicy(TieredIdentity identity, AlluxioConfiguration conf) { - mTieredIdentity = identity; - mConf = conf; - } - - @Override - public Optional getWorker(GetWorkerOptions options) { - List shuffledWorkers = Lists.newArrayList(options.getBlockWorkerInfos()); - Collections.shuffle(shuffledWorkers); - // Workers must have enough capacity to hold the block. - List candidateWorkers = shuffledWorkers.stream() - .filter(worker -> worker.getCapacityBytes() >= options.getBlockInfo().getLength()) - .collect(Collectors.toList()); - - // Try finding the nearest worker. - List addresses = candidateWorkers.stream() - .map(BlockWorkerInfo::getNetAddress) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - Optional> nearest = - BlockLocationUtils.nearest(mTieredIdentity, addresses, mConf); - return nearest.map(Pair::getFirst); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof LocalFirstPolicy)) { - return false; - } - LocalFirstPolicy that = (LocalFirstPolicy) o; - return Objects.equals(mTieredIdentity, that.mTieredIdentity); - } - - @Override - public int hashCode() { - return Objects.hash(mTieredIdentity); - } - - @Override - public String toString() { - return com.google.common.base.MoreObjects.toStringHelper(this) - .add("tieredIdentity", mTieredIdentity) - .toString(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/MostAvailableFirstPolicy.java b/core/client/fs/src/main/java/alluxio/client/block/policy/MostAvailableFirstPolicy.java deleted file mode 100644 index e0a604884ebf..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/MostAvailableFirstPolicy.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.base.MoreObjects; - -import java.util.Optional; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A policy that returns the worker with the most available bytes. - */ -@ThreadSafe -public final class MostAvailableFirstPolicy implements BlockLocationPolicy { - - /** - * Constructs a new {@link MostAvailableFirstPolicy} - * needed for instantiation in {@link BlockLocationPolicy.Factory}. - * - * @param ignoredConf is unused - */ - public MostAvailableFirstPolicy(AlluxioConfiguration ignoredConf) {} - - /** - * The policy returns null if no worker is qualified. - */ - @Override - public Optional getWorker(GetWorkerOptions options) { - long mostAvailableBytes = -1; - WorkerNetAddress result = null; - for (BlockWorkerInfo workerInfo : options.getBlockWorkerInfos()) { - if (workerInfo.getCapacityBytes() - workerInfo.getUsedBytes() > mostAvailableBytes) { - mostAvailableBytes = workerInfo.getCapacityBytes() - workerInfo.getUsedBytes(); - result = workerInfo.getNetAddress(); - } - } - return Optional.ofNullable(result); - } - - @Override - public boolean equals(Object o) { - return o instanceof MostAvailableFirstPolicy; - } - - @Override - public int hashCode() { - return 0; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).toString(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/RoundRobinPolicy.java b/core/client/fs/src/main/java/alluxio/client/block/policy/RoundRobinPolicy.java deleted file mode 100644 index c7f8935f28e8..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/RoundRobinPolicy.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.collect.Lists; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A policy that chooses the worker for the next block in a round-robin manner and skips workers - * that do not have enough space. - */ -@NotThreadSafe -public final class RoundRobinPolicy implements BlockLocationPolicy { - private List mWorkerInfoList; - private int mIndex; - private boolean mInitialized = false; - /** This caches the {@link WorkerNetAddress} for the block IDs.*/ - private final HashMap mBlockLocationCache = new HashMap<>(); - - /** - * Constructs a new {@link RoundRobinPolicy}. - * - * @param ignoredConf unused, but needed for instantiation in {@link BlockLocationPolicy.Factory} - */ - public RoundRobinPolicy(AlluxioConfiguration ignoredConf) {} - - /** - * The policy uses the first fetch of worker info list as the base, and visits each of them in a - * round-robin manner in the subsequent calls. The policy doesn't assume the list of worker info - * in the subsequent calls has the same order from the first, and it will skip the workers that - * are no longer active. - * - * Returns null if no worker can be found. - * - * @param options options - * @return the address of the worker to write to - */ - @Override - public Optional getWorker(GetWorkerOptions options) { - Set eligibleAddresses = new HashSet<>(); - for (BlockWorkerInfo info : options.getBlockWorkerInfos()) { - eligibleAddresses.add(info.getNetAddress()); - } - - WorkerNetAddress address = mBlockLocationCache.get(options.getBlockInfo().getBlockId()); - if (address != null && eligibleAddresses.contains(address)) { - return Optional.of(address); - } else { - address = null; - } - - if (!mInitialized) { - mWorkerInfoList = Lists.newArrayList(options.getBlockWorkerInfos()); - Collections.shuffle(mWorkerInfoList); - mIndex = 0; - mInitialized = true; - } - - // at most try all the workers - for (int i = 0; i < mWorkerInfoList.size(); i++) { - WorkerNetAddress candidate = mWorkerInfoList.get(mIndex).getNetAddress(); - BlockWorkerInfo workerInfo = findBlockWorkerInfo(options.getBlockWorkerInfos(), candidate); - mIndex = (mIndex + 1) % mWorkerInfoList.size(); - if (workerInfo != null - && workerInfo.getCapacityBytes() >= options.getBlockInfo().getLength() - && eligibleAddresses.contains(candidate)) { - address = candidate; - break; - } - } - mBlockLocationCache.put(options.getBlockInfo().getBlockId(), address); - return Optional.ofNullable(address); - } - - /** - * @param workerInfoList the list of worker info - * @param address the address to look for - * @return the worker info in the list that matches the host name, null if not found - */ - private BlockWorkerInfo findBlockWorkerInfo(Iterable workerInfoList, - WorkerNetAddress address) { - for (BlockWorkerInfo info : workerInfoList) { - if (info.getNetAddress().equals(address)) { - return info; - } - } - return null; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof RoundRobinPolicy)) { - return false; - } - RoundRobinPolicy that = (RoundRobinPolicy) o; - return Objects.equal(mWorkerInfoList, that.mWorkerInfoList) - && Objects.equal(mIndex, that.mIndex) - && Objects.equal(mInitialized, that.mInitialized) - && Objects.equal(mBlockLocationCache, that.mBlockLocationCache); - } - - @Override - public int hashCode() { - return Objects.hashCode(mWorkerInfoList, mIndex, mInitialized, mBlockLocationCache); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("workerInfoList", mWorkerInfoList) - .add("index", mIndex) - .add("initialized", mInitialized) - .add("blockLocationCache", mBlockLocationCache) - .toString(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/SpecificHostPolicy.java b/core/client/fs/src/main/java/alluxio/client/block/policy/SpecificHostPolicy.java deleted file mode 100644 index befb3248142b..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/SpecificHostPolicy.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; - -import java.util.Optional; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Always returns a worker with the hostname specified by - * {@link PropertyKey#WORKER_HOSTNAME} (alluxio.worker.hostname). - */ -@ThreadSafe -public final class SpecificHostPolicy implements BlockLocationPolicy { - private final String mHostname; - - /** - * Constructs a new {@link SpecificHostPolicy} - * needed for instantiation in {@link BlockLocationPolicy.Factory}. - * - * @param conf Alluxio configuration - */ - public SpecificHostPolicy(AlluxioConfiguration conf) { - this(conf.getString(PropertyKey.WORKER_HOSTNAME)); - } - - /** - * Constructs the policy with the hostname. - * - * @param hostname the name of the host - */ - public SpecificHostPolicy(String hostname) { - mHostname = Preconditions.checkNotNull(hostname, "hostname"); - } - - /** - * Returns null if no active worker matches the hostname - * provided in WORKER_HOSTNAME (alluxio.worker.hostname). - */ - @Override - public Optional getWorker(GetWorkerOptions options) { - // find the first worker matching the host name - for (BlockWorkerInfo info : options.getBlockWorkerInfos()) { - if (info.getNetAddress().getHost().equals(mHostname)) { - return Optional.of(info.getNetAddress()); - } - } - return Optional.empty(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof SpecificHostPolicy)) { - return false; - } - SpecificHostPolicy that = (SpecificHostPolicy) o; - return Objects.equal(mHostname, that.mHostname); - } - - @Override - public int hashCode() { - return Objects.hashCode(mHostname); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("hostname", mHostname) - .toString(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/policy/options/GetWorkerOptions.java b/core/client/fs/src/main/java/alluxio/client/block/policy/options/GetWorkerOptions.java deleted file mode 100644 index 6f73e3093b43..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/policy/options/GetWorkerOptions.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy.options; - -import alluxio.annotation.PublicApi; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.wire.BlockInfo; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; - -import java.util.List; - -/** - * Method options for - * {@link alluxio.client.block.policy.BlockLocationPolicy#getWorker(GetWorkerOptions)}. - */ -@PublicApi -public final class GetWorkerOptions { - private List mBlockWorkerInfos; - private BlockInfo mBlockInfo; - - /** - * @return the default {@link GetWorkerOptions} - */ - public static GetWorkerOptions defaults() { - return new GetWorkerOptions(); - } - - /** - * Creates a new instance with defaults. - */ - private GetWorkerOptions() { - mBlockInfo = new BlockInfo(); - } - - /** - * @return the list of block worker infos - */ - public BlockInfo getBlockInfo() { - return mBlockInfo; - } - - /** - * @return the list of block worker infos - */ - public Iterable getBlockWorkerInfos() { - return mBlockWorkerInfos; - } - - /** - * @param blockInfo the block information - * @return the updated options - */ - public GetWorkerOptions setBlockInfo(BlockInfo blockInfo) { - mBlockInfo = blockInfo; - return this; - } - - /** - * @param blockWorkerInfos the block worker infos - * @return the updated options - */ - public GetWorkerOptions setBlockWorkerInfos( - List blockWorkerInfos) { - mBlockWorkerInfos = blockWorkerInfos; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof GetWorkerOptions)) { - return false; - } - GetWorkerOptions that = (GetWorkerOptions) o; - return Objects.equal(mBlockWorkerInfos, that.mBlockWorkerInfos) - && Objects.equal(mBlockInfo, that.mBlockInfo); - } - - @Override - public int hashCode() { - return Objects.hashCode(mBlockWorkerInfos, mBlockInfo); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("blockInfo", mBlockInfo) - .add("blockWorkerInfos", mBlockWorkerInfos) - .toString(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/BlockInStream.java b/core/client/fs/src/main/java/alluxio/client/block/stream/BlockInStream.java deleted file mode 100644 index 042662935f28..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/BlockInStream.java +++ /dev/null @@ -1,540 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.Seekable; -import alluxio.client.BoundedStream; -import alluxio.client.PositionedReadable; -import alluxio.client.ReadType; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.options.InStreamOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.PreconditionMessage; -import alluxio.exception.status.NotFoundException; -import alluxio.exception.status.OutOfRangeException; -import alluxio.grpc.ReadRequest; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.proto.dataserver.Protocol; -import alluxio.util.io.BufferUtils; -import alluxio.util.network.NettyUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.Objects; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Provides an {@link InputStream} implementation that is based on {@link DataReader}s to - * stream data chunk by chunk. - */ -@NotThreadSafe -public class BlockInStream extends InputStream implements BoundedStream, Seekable, - PositionedReadable { - private static final Logger LOG = LoggerFactory.getLogger(BlockInStream.class); - - /** the source tracking where the block is from. */ - public enum BlockInStreamSource { - PROCESS_LOCAL, // The block is from a worker in the same process - NODE_LOCAL, // The block is from a separate worker process on the same node - REMOTE, // The block is from a remote worker - UFS // The block is in UFS - } - - private final WorkerNetAddress mAddress; - private final BlockInStreamSource mInStreamSource; - /** The id of the block or UFS file to which this InStream provides access. */ - private final long mId; - /** The size in bytes of the block. */ - private final long mLength; - private final byte[] mSingleByte = new byte[1]; - - /** Current position of the stream, relative to the start of the block. */ - private long mPos = 0; - /** The current data chunk. */ - private DataBuffer mCurrentChunk; - - private DataReader mDataReader; - private final DataReader.Factory mDataReaderFactory; - - private boolean mClosed = false; - private boolean mEOF = false; - - /** - * Creates a {@link BlockInStream}. - * - * One of several read behaviors: - * - * 1. Domain socket - if the data source is the local worker and the local worker has a domain - * socket server - * 2. Short-Circuit - if the data source is the local worker - * 3. Local Loopback Read - if the data source is the local worker and short circuit is disabled - * 4. Read from remote worker - if the data source is a remote worker - * 5. UFS Read from worker - if the data source is UFS, read from the UFS policy's designated - * worker (ufs -> local or remote worker -> client) - * - * @param context the file system context - * @param info the block info - * @param dataSource the Alluxio worker which should read the data - * @param dataSourceType the source location of the block - * @param options the InStream options - * @return the {@link BlockInStream} object - */ - public static BlockInStream create(FileSystemContext context, BlockInfo info, - WorkerNetAddress dataSource, BlockInStreamSource dataSourceType, InStreamOptions options) - throws IOException { - long blockId = info.getBlockId(); - long blockSize = info.getLength(); - - if (dataSourceType == BlockInStreamSource.PROCESS_LOCAL) { - // Interaction between the current client and the worker it embedded to should - // go through worker internal communication directly without RPC involves - LOG.debug("Creating worker process local input stream for block {} @ {}", - blockId, dataSource); - return createProcessLocalBlockInStream(context, dataSource, blockId, blockSize, options); - } - - AlluxioConfiguration alluxioConf = context.getClusterConf(); - boolean shortCircuit = alluxioConf.getBoolean(PropertyKey.USER_SHORT_CIRCUIT_ENABLED); - boolean shortCircuitPreferred = - alluxioConf.getBoolean(PropertyKey.USER_SHORT_CIRCUIT_PREFERRED); - boolean sourceSupportsDomainSocket = NettyUtils.isDomainSocketSupported(dataSource); - boolean sourceIsLocal = dataSourceType == BlockInStreamSource.NODE_LOCAL; - - // Short circuit is enabled when - // 1. data source is local node - // 2. alluxio.user.short.circuit.enabled is true - // 3. the worker's domain socket is not configured - // OR alluxio.user.short.circuit.preferred is true - if (sourceIsLocal && shortCircuit && (shortCircuitPreferred || !sourceSupportsDomainSocket)) { - LOG.debug("Creating short circuit input stream for block {} @ {}", blockId, dataSource); - try { - return createLocalBlockInStream(context, dataSource, blockId, blockSize, options); - } catch (NotFoundException e) { - // Failed to do short circuit read because the block is not available in Alluxio. - // We will try to read via gRPC. So this exception is ignored. - LOG.warn("Failed to create short circuit input stream for block {} @ {}. Falling back to " - + "network transfer", blockId, dataSource); - } - } - - // gRPC - LOG.debug("Creating gRPC input stream for block {} @ {} from client {} reading through {} (" - + "data locates in the local worker {}, shortCircuitEnabled {}, " - + "shortCircuitPreferred {}, sourceSupportDomainSocket {})", - blockId, dataSource, NetworkAddressUtils.getClientHostName(alluxioConf), dataSource, - sourceIsLocal, shortCircuit, shortCircuitPreferred, sourceSupportsDomainSocket); - return createGrpcBlockInStream(context, dataSource, dataSourceType, blockId, - blockSize, options); - } - - /** - * Creates a {@link BlockInStream} to read from the worker process-local to this client - * directly without RPC involves, if the block does not exist in this worker, will read from - * the UFS storage via this worker. - * - * @param context the file system context - * @param address the network address of the gRPC data server to read from - * @param blockId the block ID - * @param length the block length - * @param options the in stream options - * @return the {@link BlockInStream} created - */ - private static BlockInStream createProcessLocalBlockInStream(FileSystemContext context, - WorkerNetAddress address, long blockId, long length, InStreamOptions options) { - AlluxioConfiguration conf = context.getClusterConf(); - long chunkSize = conf.getBytes( - PropertyKey.USER_LOCAL_READER_CHUNK_SIZE_BYTES); - return new BlockInStream(new BlockWorkerDataReader.Factory( - context.getProcessLocalWorker().orElseThrow(NullPointerException::new), - blockId, chunkSize, options), - address, BlockInStreamSource.PROCESS_LOCAL, blockId, length); - } - - /** - * Creates a {@link BlockInStream} to read from a local file. - * - * @param context the file system context - * @param address the network address of the gRPC data server to read from - * @param blockId the block ID - * @param length the block length - * @param options the in stream options - * @return the {@link BlockInStream} created - */ - private static BlockInStream createLocalBlockInStream(FileSystemContext context, - WorkerNetAddress address, long blockId, long length, InStreamOptions options) - throws IOException { - AlluxioConfiguration conf = context.getClusterConf(); - long chunkSize = conf.getBytes( - PropertyKey.USER_LOCAL_READER_CHUNK_SIZE_BYTES); - return new BlockInStream( - new LocalFileDataReader.Factory(context, address, blockId, chunkSize, options), - address, BlockInStreamSource.NODE_LOCAL, blockId, length); - } - - /** - * Creates a {@link BlockInStream} to read from a gRPC data server. - * - * @param context the file system context - * @param address the address of the gRPC data server - * @param blockSource the source location of the block - * @param blockSize the block size - * @param blockId the block id - * @return the {@link BlockInStream} created - */ - private static BlockInStream createGrpcBlockInStream(FileSystemContext context, - WorkerNetAddress address, BlockInStreamSource blockSource, - long blockId, long blockSize, InStreamOptions options) { - AlluxioConfiguration conf = context.getClusterConf(); - long chunkSize = conf.getBytes( - PropertyKey.USER_STREAMING_READER_CHUNK_SIZE_BYTES); - // Construct the partial read request - ReadRequest.Builder builder = ReadRequest.newBuilder() - .setBlockId(blockId) - .setPromote(ReadType.fromProto(options.getOptions().getReadType()).isPromote()) - .setOpenUfsBlockOptions(options.getOpenUfsBlockOptions(blockId)) // Add UFS fallback options - .setPositionShort(options.getPositionShort()) - .setChunkSize(chunkSize); - DataReader.Factory factory; - if (context.getClusterConf().getBoolean(PropertyKey.FUSE_SHARED_CACHING_READER_ENABLED) - && blockSize > chunkSize * 4) { - // Heuristic to resolve issues/12146, guarded by alluxio.fuse.shared.caching.reader.enabled - // GrpcDataReader instances are shared across FileInStreams to mitigate seek cost - factory = new SharedGrpcDataReader.Factory(context, address, builder, blockSize); - } else { - factory = new GrpcDataReader.Factory(context, address, builder); - } - return new BlockInStream(factory, address, blockSource, blockId, blockSize); - } - - /** - * Creates a {@link BlockInStream} to read from a specific remote server. Should only be used - * in cases where the data source and method of reading is known, i.e. worker - worker - * communication. - * - * @param context the file system context - * @param blockId the block id - * @param address the address of the gRPC data server - * @param blockSource the source location of the block - * @param blockSize the size of the block - * @param ufsOptions the ufs read options - * @return the {@link BlockInStream} created - */ - public static BlockInStream createRemoteBlockInStream(FileSystemContext context, long blockId, - WorkerNetAddress address, BlockInStreamSource blockSource, long blockSize, - Protocol.OpenUfsBlockOptions ufsOptions) { - AlluxioConfiguration conf = context.getClusterConf(); - long chunkSize = conf.getBytes( - PropertyKey.USER_STREAMING_READER_CHUNK_SIZE_BYTES); - ReadRequest readRequest = ReadRequest.newBuilder().setBlockId(blockId) - .setOpenUfsBlockOptions(ufsOptions).setChunkSize(chunkSize).buildPartial(); - DataReader.Factory factory = new GrpcDataReader.Factory(context, address, - readRequest.toBuilder()); - return new BlockInStream(factory, address, blockSource, blockId, blockSize); - } - - /** - * Creates an instance of {@link BlockInStream}. - * - * @param dataReaderFactory the data reader factory - * @param address the address of the gRPC data server - * @param blockSource the source location of the block - * @param id the ID (either block ID or UFS file ID) - * @param length the length - */ - @VisibleForTesting - protected BlockInStream(DataReader.Factory dataReaderFactory, - WorkerNetAddress address, BlockInStreamSource blockSource, long id, long length) { - mDataReaderFactory = dataReaderFactory; - mAddress = address; - mInStreamSource = blockSource; - mId = id; - mLength = length; - } - - @Override - public long getPos() { - return mPos; - } - - @Override - public int read() throws IOException { - int bytesRead = read(mSingleByte); - if (bytesRead == -1) { - return -1; - } - Preconditions.checkState(bytesRead == 1); - return BufferUtils.byteToInt(mSingleByte[0]); - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - Objects.requireNonNull(b, "Read buffer cannot be null"); - return read(ByteBuffer.wrap(b), off, len); - } - - /** - * Reads up to len bytes of data from the input stream into the byte buffer. - * - * @param byteBuffer the buffer into which the data is read - * @param off the start offset in the buffer at which the data is written - * @param len the maximum number of bytes to read - * @return the total number of bytes read into the buffer, or -1 if there is no more data because - * the end of the stream has been reached - */ - public int read(ByteBuffer byteBuffer, int off, int len) throws IOException { - Preconditions.checkArgument(off >= 0 && len >= 0 && len + off <= byteBuffer.capacity(), - PreconditionMessage.ERR_BUFFER_STATE.toString(), byteBuffer.capacity(), off, len); - checkIfClosed(); - if (len == 0) { - - return 0; - } - if (mPos == mLength) { - return -1; - } - readChunk(); - if (mCurrentChunk == null) { - mEOF = true; - } - if (mEOF) { - closeDataReader(); - if (mPos < mLength) { - throw new OutOfRangeException(String.format("Block %s is expected to be %s bytes, " - + "but only %s bytes are available in the UFS. " - + "Please retry the read and on the next access, " - + "Alluxio will sync with the UFS and fetch the updated file content.", - mId, mLength, mPos)); - } - return -1; - } - int toRead = Math.min(len, mCurrentChunk.readableBytes()); - byteBuffer.position(off).limit(off + toRead); - mCurrentChunk.readBytes(byteBuffer); - mPos += toRead; - if (mPos == mLength) { - // a performance improvement introduced by https://github.com/Alluxio/alluxio/issues/14020 - closeDataReader(); - } - return toRead; - } - - @Override - public int positionedRead(long pos, byte[] b, int off, int len) throws IOException { - if (len == 0) { - return 0; - } - if (pos < 0 || pos >= mLength) { - return -1; - } - - int lenCopy = len; - try (DataReader reader = mDataReaderFactory.create(pos, len)) { - // We try to read len bytes instead of returning after reading one chunk because - // it is not free to create/close a DataReader. - while (len > 0) { - DataBuffer dataBuffer = null; - try { - dataBuffer = reader.readChunk(); - if (dataBuffer == null) { - break; - } - Preconditions.checkState(dataBuffer.readableBytes() <= len); - int toRead = dataBuffer.readableBytes(); - dataBuffer.readBytes(b, off, toRead); - len -= toRead; - off += toRead; - } finally { - if (dataBuffer != null) { - dataBuffer.release(); - } - } - } - } - if (lenCopy == len) { - return -1; - } - return lenCopy - len; - } - - @Override - public long remaining() { - return mEOF ? 0 : mLength - mPos; - } - - @Override - public void seek(long pos) throws IOException { - checkIfClosed(); - Preconditions.checkArgument(pos >= 0, PreconditionMessage.ERR_SEEK_NEGATIVE.toString(), pos); - Preconditions.checkArgument(pos <= mLength, - "Seek position past the end of the read region (block or file).", mId); - if (pos == mPos) { - return; - } - // When alluxio.fuse.shared.caching.reader.enabled is on (to resolve issues/12146), - // use the heuristic to improve seek performance with fewer data reader close. - if (mDataReader instanceof SharedGrpcDataReader) { - seekForSharedGrpcDataReader(pos); - return; - } - if (pos < mPos) { - mEOF = false; - } - closeDataReader(); - mPos = pos; - } - - private void seekForSharedGrpcDataReader(long pos) throws IOException { - if (pos < mPos) { - mEOF = false; - // because the reader is shared, let's not close it but simply seek - ((SharedGrpcDataReader) mDataReader).seek(pos); - if (mCurrentChunk != null) { - mCurrentChunk.release(); - mCurrentChunk = null; - } - } else { - // TODO(lu) combine the original seek logic and the following general improvements - // that are helpful in both fuse and non-fuse scenarios - // Try to read data already received but haven't processed - long curPos = mPos; - while (mCurrentChunk != null && curPos < pos) { - long nextPos = curPos + mCurrentChunk.readableBytes(); - if (nextPos <= pos) { - curPos = nextPos; - mCurrentChunk.release(); - mCurrentChunk = mDataReader.readChunk(); - } else { - // TODO(chaowang) introduce seek in DataBuffer - int toRead = (int) (pos - curPos); - final byte[] b = new byte[toRead]; - mCurrentChunk.readBytes(b, 0, toRead); - curPos = pos; - } - } - - if (curPos < pos) { - // Not enough data in queue, close the data reader - closeDataReader(); - } - } - mPos = pos; - } - - @Override - public long skip(long n) throws IOException { - checkIfClosed(); - if (n <= 0) { - return 0; - } - - long toSkip = Math.min(remaining(), n); - mPos += toSkip; - - closeDataReader(); - return toSkip; - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - try { - closeDataReader(); - } finally { - mDataReaderFactory.close(); - } - mClosed = true; - } - - /** - * @return the underlying data reader factory - */ - @VisibleForTesting - public DataReader.Factory getDataReaderFactory() { - return mDataReaderFactory; - } - - /** - * Reads a new chunk from the channel if all of the current chunk is read. - */ - private void readChunk() throws IOException { - if (mDataReader == null) { - mDataReader = mDataReaderFactory.create(mPos, mLength - mPos); - } - - if (mCurrentChunk != null && mCurrentChunk.readableBytes() == 0) { - mCurrentChunk.release(); - mCurrentChunk = null; - } - if (mCurrentChunk == null) { - mCurrentChunk = mDataReader.readChunk(); - } - } - - /** - * Close the current data reader. - */ - private void closeDataReader() throws IOException { - if (mCurrentChunk != null) { - mCurrentChunk.release(); - mCurrentChunk = null; - } - if (mDataReader != null) { - mDataReader.close(); - } - mDataReader = null; - } - - /** - * Convenience method to ensure the stream is not closed. - */ - private void checkIfClosed() { - Preconditions.checkState(!mClosed, "Cannot do operations on a closed BlockInStream"); - } - - /** - * @return the address of the data server - */ - public WorkerNetAddress getAddress() { - return mAddress; - } - - /** - * @return the source of the block location - */ - public BlockInStreamSource getSource() { - return mInStreamSource; - } - - /** - * @return the block ID - */ - public long getId() { - return mId; - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/BlockOutStream.java b/core/client/fs/src/main/java/alluxio/client/block/stream/BlockOutStream.java deleted file mode 100644 index ed07ea9cde90..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/BlockOutStream.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.client.BoundedStream; -import alluxio.client.Cancelable; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; -import io.netty.buffer.Unpooled; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Provides an {@link OutputStream} implementation that is based on {@link DataWriter} which - * streams data chunk by chunk. - */ -@NotThreadSafe -public class BlockOutStream extends OutputStream implements BoundedStream, Cancelable { - - private final Closer mCloser; - /** Length of the stream. If unknown, set to Long.MAX_VALUE. */ - private final long mLength; - private final WorkerNetAddress mAddress; - private ByteBuf mCurrentChunk = null; - - private final List mDataWriters; - private boolean mClosed; - - /** - * Constructs a new {@link BlockOutStream} with only one {@link DataWriter}. - * - * @param dataWriter the data writer - * @param length the length of the stream - * @param address the Alluxio worker address - */ - public BlockOutStream(DataWriter dataWriter, long length, WorkerNetAddress address) { - mCloser = Closer.create(); - mLength = length; - Objects.requireNonNull(address); - mAddress = address; - mDataWriters = new ArrayList<>(1); - Objects.requireNonNull(dataWriter); - mDataWriters.add(dataWriter); - mCloser.register(dataWriter); - mClosed = false; - } - - /** - * @return the remaining size of the block - */ - @Override - public long remaining() { - long pos = Long.MAX_VALUE; - for (DataWriter dataWriter : mDataWriters) { - pos = Math.min(pos, dataWriter.pos()); - } - return mLength - pos - (mCurrentChunk != null ? mCurrentChunk.readableBytes() : 0); - } - - /** - * Creates a new remote block output stream. - * - * @param context the file system context - * @param blockId the block id - * @param blockSize the block size - * @param workerNetAddresses the worker network addresses - * @param options the options - * @return the {@link BlockOutStream} instance created - */ - public static BlockOutStream createReplicatedBlockOutStream(FileSystemContext context, - long blockId, long blockSize, java.util.List workerNetAddresses, - OutStreamOptions options) throws IOException { - List dataWriters = new ArrayList<>(); - for (WorkerNetAddress address: workerNetAddresses) { - DataWriter dataWriter = - DataWriter.Factory.create(context, blockId, blockSize, address, options); - dataWriters.add(dataWriter); - } - return new BlockOutStream(dataWriters, blockSize, workerNetAddresses); - } - - /** - * Constructs a new {@link BlockOutStream} with only one {@link DataWriter}. - * - * @param dataWriters the data writer - * @param length the length of the stream - * @param workerNetAddresses the worker network addresses - */ - protected BlockOutStream(List dataWriters, long length, - java.util.List workerNetAddresses) { - mCloser = Closer.create(); - mLength = length; - mAddress = workerNetAddresses.get(0); - mDataWriters = dataWriters; - for (DataWriter dataWriter : dataWriters) { - mCloser.register(dataWriter); - } - mClosed = false; - } - - @Override - public void write(int b) throws IOException { - Preconditions.checkState(remaining() > 0, "Cannot write past end of block"); - updateCurrentChunk(false); - mCurrentChunk.writeByte(b); - } - - @Override - public void write(byte[] b) throws IOException { - write(b, 0, b.length); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - writeInternal(Unpooled.wrappedBuffer(b), off, len); - } - - /** - * Writes the data in the specified byte buf to this output stream. - * - * @param buf the buffer - * @throws IOException exception - */ - public void write(io.netty.buffer.ByteBuf buf) throws IOException { - write(buf, 0, buf.readableBytes()); - } - - /** - * Writes len bytes from the specified byte buf starting at offset off to this output stream. - * - * @param buf the buffer - * @param off the offset - * @param len the length - */ - public void write(io.netty.buffer.ByteBuf buf, int off, int len) throws IOException { - writeInternal(buf, off, len); - } - - private void writeInternal(ByteBuf b, int off, int len) throws IOException { - if (len == 0) { - return; - } - - while (len > 0) { - updateCurrentChunk(false); - int toWrite = Math.min(len, mCurrentChunk.writableBytes()); - mCurrentChunk.writeBytes(b, off, toWrite); - off += toWrite; - len -= toWrite; - } - updateCurrentChunk(false); - } - - @Override - public void flush() throws IOException { - if (mClosed) { - return; - } - updateCurrentChunk(true); - for (DataWriter dataWriter : mDataWriters) { - dataWriter.flush(); - } - } - - @Override - public void cancel() throws IOException { - if (mClosed) { - return; - } - releaseCurrentChunk(); - - List exceptions = new LinkedList<>(); - for (DataWriter dataWriter : mDataWriters) { - try { - dataWriter.cancel(); - } catch (IOException e) { - exceptions.add(e); - } - } - if (exceptions.size() > 0) { - IOException ex = new IOException("Failed to cancel all block write attempts"); - exceptions.forEach(ex::addSuppressed); - throw ex; - } - - close(); - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - try { - updateCurrentChunk(true); - } catch (Throwable t) { - throw mCloser.rethrow(t); - } finally { - mClosed = true; - mCloser.close(); - } - } - - /** - * @return the worker address for this stream - */ - public WorkerNetAddress getAddress() { - return mAddress; - } - - /** - * Updates the current chunk. - * - * @param lastChunk if the current packet is the last packet - */ - private void updateCurrentChunk(boolean lastChunk) throws IOException { - // Early return for the most common case. - if (mCurrentChunk != null && mCurrentChunk.writableBytes() > 0 && !lastChunk) { - return; - } - - if (mCurrentChunk == null) { - if (!lastChunk) { - mCurrentChunk = allocateBuffer(); - } - return; - } - - if (mCurrentChunk.writableBytes() == 0 || lastChunk) { - try { - if (mCurrentChunk.readableBytes() > 0) { - for (DataWriter dataWriter : mDataWriters) { - mCurrentChunk.retain(); - dataWriter.writeChunk(mCurrentChunk.duplicate()); - } - } else { - Preconditions.checkState(lastChunk); - } - } finally { - // If the packet has bytes to read, we increment its refcount explicitly for every packet - // writer. So we need to release here. If the packet has no bytes to read, then it has - // to be the last packet. It needs to be released as well. - mCurrentChunk.release(); - mCurrentChunk = null; - } - } - if (!lastChunk) { - mCurrentChunk = allocateBuffer(); - } - } - - /** - * Releases the current packet. - */ - private void releaseCurrentChunk() { - if (mCurrentChunk != null) { - mCurrentChunk.release(); - mCurrentChunk = null; - } - } - - /** - * @return a newly allocated byte buffer of the user defined default size - */ - private ByteBuf allocateBuffer() { - return PooledByteBufAllocator.DEFAULT.buffer(mDataWriters.get(0).chunkSize()); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/BlockWorkerClient.java b/core/client/fs/src/main/java/alluxio/client/block/stream/BlockWorkerClient.java deleted file mode 100644 index f079da633db9..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/BlockWorkerClient.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.conf.AlluxioConfiguration; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.ClearMetricsRequest; -import alluxio.grpc.ClearMetricsResponse; -import alluxio.grpc.CreateLocalBlockRequest; -import alluxio.grpc.CreateLocalBlockResponse; -import alluxio.grpc.GrpcServerAddress; -import alluxio.grpc.LoadRequest; -import alluxio.grpc.LoadResponse; -import alluxio.grpc.MoveBlockRequest; -import alluxio.grpc.MoveBlockResponse; -import alluxio.grpc.OpenLocalBlockRequest; -import alluxio.grpc.OpenLocalBlockResponse; -import alluxio.grpc.ReadRequest; -import alluxio.grpc.ReadResponse; -import alluxio.grpc.RemoveBlockRequest; -import alluxio.grpc.RemoveBlockResponse; -import alluxio.grpc.WriteRequest; -import alluxio.grpc.WriteResponse; -import alluxio.security.user.UserState; - -import com.google.common.util.concurrent.ListenableFuture; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; - -import java.io.Closeable; -import java.io.IOException; - -/** - * gRPC client for worker communication. - */ -public interface BlockWorkerClient extends Closeable { - /** - * Factory for block worker client. - */ - class Factory { - /** - * Creates a new block worker client. - * - * @param userState the user subject - * @param address the address of the worker - * @param alluxioConf Alluxio configuration - * @return a new {@link BlockWorkerClient} - */ - public static BlockWorkerClient create(UserState userState, GrpcServerAddress address, - AlluxioConfiguration alluxioConf) - throws IOException { - return new DefaultBlockWorkerClient(userState, address, alluxioConf); - } - } - - /** - * @return whether the client is shutdown - */ - boolean isShutdown(); - - /** - * @return whether the client is healthy - */ - boolean isHealthy(); - - /** - * Writes a block to the worker asynchronously. The caller should pass in a response observer - * for receiving server responses and handling errors. - * - * @param responseObserver the stream observer for the server response - * @return the stream observer for the client request - */ - StreamObserver writeBlock(StreamObserver responseObserver); - - /** - * Reads a block from the worker. When client is done with the file, it should close the stream - * using the gRPC context. - * - * @param responseObserver the stream observer for the server response - * @return the stream observer for the client request - */ - StreamObserver readBlock(StreamObserver responseObserver); - - /** - * Creates a local block on the worker. This is a two stage operations: - * 1. Client sends a create request through the request stream. Server will respond with the name - * of the file to write to. - * 2. When client is done with the file, it should signal complete or cancel on the request stream - * based on the intent. The server will signal complete on the response stream once the - * operation is done. - * - * @param responseObserver the stream observer for the server response - * @return the stream observer for the client request - */ - StreamObserver createLocalBlock( - StreamObserver responseObserver); - - /** - * Opens a local block. This is a two stage operations: - * 1. Client sends an open request through the request stream. Server will respond with the name - * of the file to read from. - * 2. When client is done with the file, it should close the stream. - * - * @param responseObserver the stream observer for the server response - * @return the stream observer for the client request - */ - StreamObserver openLocalBlock( - StreamObserver responseObserver); - - /** - * Removes a block from worker. - * @param request the remove block request - * @return the response from server - * @throws StatusRuntimeException if any error occurs - */ - RemoveBlockResponse removeBlock(RemoveBlockRequest request); - - /** - * Move a block from worker. - * @param request the remove block request - * @return the response from server - * @throws StatusRuntimeException if any error occurs - */ - MoveBlockResponse moveBlock(MoveBlockRequest request); - - /** - * Clear the worker metrics. - * - * @param request the request to clear metrics - * @return the response from server - */ - ClearMetricsResponse clearMetrics(ClearMetricsRequest request); - - /** - * Caches a block. - * - * @param request the cache request - * @throws StatusRuntimeException if any error occurs - */ - void cache(CacheRequest request); - - /** - * Free this worker. - */ - void freeWorker(); - - /** - * load blocks into alluxio. - * - * @param request the cache request - * @return listenable future of LoadResponse - * @throws StatusRuntimeException if any error occurs - */ - ListenableFuture load(LoadRequest request); -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/DataWriter.java b/core/client/fs/src/main/java/alluxio/client/block/stream/DataWriter.java deleted file mode 100644 index 90541ebf6e2f..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/DataWriter.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.client.Cancelable; -import alluxio.client.WriteType; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.RequestType; -import alluxio.util.CommonUtils; -import alluxio.util.network.NettyUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.WorkerNetAddress; - -import io.netty.buffer.ByteBuf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import javax.annotation.concurrent.ThreadSafe; - -/** - * The interface to write data. - */ -public interface DataWriter extends Closeable, Cancelable { - - /** - * Factory for {@link DataWriter}. - */ - @ThreadSafe - class Factory { - public static final Logger LOG = LoggerFactory.getLogger(DataWriter.Factory.class); - - private Factory() {} // prevent instantiation - - /** - * @param context the file system context - * @param blockId the block ID - * @param blockSize the block size in bytes - * @param address the Alluxio worker address - * @param options the out stream options - * @return the {@link DataWriter} instance - */ - public static DataWriter create(FileSystemContext context, long blockId, long blockSize, - WorkerNetAddress address, OutStreamOptions options) throws IOException { - AlluxioConfiguration alluxioConf = context.getClusterConf(); - boolean shortCircuit = alluxioConf.getBoolean(PropertyKey.USER_SHORT_CIRCUIT_ENABLED); - boolean shortCircuitPreferred = - alluxioConf.getBoolean(PropertyKey.USER_SHORT_CIRCUIT_PREFERRED); - boolean ufsFallbackEnabled = options.getWriteType() == WriteType.ASYNC_THROUGH - && alluxioConf.getBoolean(PropertyKey.USER_FILE_UFS_TIER_ENABLED); - boolean workerIsLocal = CommonUtils.isLocalHost(address, alluxioConf); - - if (workerIsLocal && context.hasProcessLocalWorker() && !ufsFallbackEnabled) { - LOG.debug("Creating worker process local output stream for block {} @ {}", - blockId, address); - return BlockWorkerDataWriter.create(context, blockId, blockSize, options); - } - LOG.debug("Doesn't create worker process local output stream for block {} @ {} " - + "(data locates in local worker: {}, client locates in local worker process: {}, " - + "ufs fallback enabled: {})", blockId, address, - workerIsLocal, context.hasProcessLocalWorker(), ufsFallbackEnabled); - - boolean domainSocketSupported = NettyUtils.isDomainSocketSupported(address); - if (workerIsLocal && shortCircuit - && (shortCircuitPreferred || !domainSocketSupported)) { - if (ufsFallbackEnabled) { - LOG.info("Creating UFS-fallback short circuit output stream for block {} @ {}", blockId, - address); - return UfsFallbackLocalFileDataWriter.create( - context, address, blockId, blockSize, options); - } - LOG.debug("Creating short circuit output stream for block {} @ {}", blockId, address); - return LocalFileDataWriter.create(context, address, blockId, blockSize, options); - } else { - LOG.debug("Creating gRPC output stream for block {} @ {} from client {} " - + "(data locates in local worker: {}, shortCircuitEnabled: {}, " - + "shortCircuitPreferred: {}, domainSocketSupported: {})", - blockId, address, NetworkAddressUtils.getClientHostName(alluxioConf), - workerIsLocal, shortCircuit, shortCircuitPreferred, domainSocketSupported); - return GrpcDataWriter - .create(context, address, blockId, blockSize, RequestType.ALLUXIO_BLOCK, - options); - } - } - } - - /** - * Writes a chunk. This method takes the ownership of this chunk even if it fails to write - * the chunk. - * - * @param chunk the chunk - */ - void writeChunk(ByteBuf chunk) throws IOException; - - /** - * Flushes all the pending chunks. - */ - void flush() throws IOException; - - /** - * @return the chunk size in bytes used - */ - int chunkSize(); - - /** - * @return the current pos which is the same as the totally number of bytes written so far - */ - long pos(); -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/DefaultBlockWorkerClient.java b/core/client/fs/src/main/java/alluxio/client/block/stream/DefaultBlockWorkerClient.java deleted file mode 100644 index 95d5c16d255a..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/DefaultBlockWorkerClient.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.UnauthenticatedException; -import alluxio.grpc.BlockWorkerGrpc; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.ClearMetricsRequest; -import alluxio.grpc.ClearMetricsResponse; -import alluxio.grpc.CreateLocalBlockRequest; -import alluxio.grpc.CreateLocalBlockResponse; -import alluxio.grpc.DataMessageMarshaller; -import alluxio.grpc.DataMessageMarshallerProvider; -import alluxio.grpc.FreeWorkerRequest; -import alluxio.grpc.GrpcChannel; -import alluxio.grpc.GrpcChannelBuilder; -import alluxio.grpc.GrpcNetworkGroup; -import alluxio.grpc.GrpcSerializationUtils; -import alluxio.grpc.GrpcServerAddress; -import alluxio.grpc.LoadRequest; -import alluxio.grpc.LoadResponse; -import alluxio.grpc.MoveBlockRequest; -import alluxio.grpc.MoveBlockResponse; -import alluxio.grpc.OpenLocalBlockRequest; -import alluxio.grpc.OpenLocalBlockResponse; -import alluxio.grpc.ReadRequest; -import alluxio.grpc.ReadResponse; -import alluxio.grpc.RemoveBlockRequest; -import alluxio.grpc.RemoveBlockResponse; -import alluxio.grpc.WriteRequest; -import alluxio.grpc.WriteResponse; -import alluxio.resource.AlluxioResourceLeakDetectorFactory; -import alluxio.retry.RetryPolicy; -import alluxio.retry.RetryUtils; -import alluxio.security.user.UserState; - -import com.google.common.io.Closer; -import com.google.common.util.concurrent.ListenableFuture; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; -import io.netty.util.ResourceLeakDetector; -import io.netty.util.ResourceLeakTracker; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; - -/** - * Default implementation of {@link BlockWorkerClient}. - */ -public class DefaultBlockWorkerClient implements BlockWorkerClient { - private static final Logger LOG = - LoggerFactory.getLogger(DefaultBlockWorkerClient.class.getName()); - - private static final ResourceLeakDetector DETECTOR = - AlluxioResourceLeakDetectorFactory.instance() - .newResourceLeakDetector(DefaultBlockWorkerClient.class); - - private GrpcChannel mStreamingChannel; - private GrpcChannel mRpcChannel; - private final GrpcServerAddress mAddress; - private final long mRpcTimeoutMs; - - private final BlockWorkerGrpc.BlockWorkerStub mStreamingAsyncStub; - private final BlockWorkerGrpc.BlockWorkerBlockingStub mRpcBlockingStub; - private final BlockWorkerGrpc.BlockWorkerFutureStub mRpcFutureStub; - - @Nullable - private final ResourceLeakTracker mTracker; - - /** - * Creates a client instance for communicating with block worker. - * - * @param userState the user state - * @param address the address of the worker - * @param alluxioConf Alluxio configuration - */ - public DefaultBlockWorkerClient(UserState userState, GrpcServerAddress address, - AlluxioConfiguration alluxioConf) throws IOException { - RetryPolicy retryPolicy = RetryUtils.defaultClientRetry(); - UnauthenticatedException lastException = null; - // TODO(feng): unify worker client with AbstractClient - while (retryPolicy.attempt()) { - try { - // Disables channel pooling for data streaming to achieve better throughput. - // Channel is still reused due to client pooling. - mStreamingChannel = GrpcChannelBuilder.newBuilder(address, alluxioConf) - .setSubject(userState.getSubject()) - .setNetworkGroup(GrpcNetworkGroup.STREAMING) - .build(); - mStreamingChannel.intercept(new StreamSerializationClientInterceptor()); - // Uses default pooling strategy for RPC calls for better scalability. - mRpcChannel = GrpcChannelBuilder.newBuilder(address, alluxioConf) - .setSubject(userState.getSubject()) - .setNetworkGroup(GrpcNetworkGroup.RPC) - .build(); - lastException = null; - break; - } catch (StatusRuntimeException e) { - close(); - throw AlluxioStatusException.fromStatusRuntimeException(e); - } catch (UnauthenticatedException e) { - close(); - userState.relogin(); - lastException = e; - } - } - if (lastException != null) { - throw lastException; - } - mStreamingAsyncStub = BlockWorkerGrpc.newStub(mStreamingChannel); - mRpcBlockingStub = BlockWorkerGrpc.newBlockingStub(mRpcChannel); - mRpcFutureStub = BlockWorkerGrpc.newFutureStub(mRpcChannel); - mAddress = address; - mRpcTimeoutMs = alluxioConf.getMs(PropertyKey.USER_RPC_RETRY_MAX_DURATION); - mTracker = DETECTOR.track(this); - } - - @Override - public boolean isShutdown() { - return mStreamingChannel.isShutdown() || mRpcChannel.isShutdown(); - } - - @Override - public boolean isHealthy() { - return !isShutdown() && mStreamingChannel.isHealthy() && mRpcChannel.isHealthy(); - } - - @Override - public void close() throws IOException { - try (Closer closer = Closer.create()) { - closer.register(() -> { - if (mStreamingChannel != null) { - mStreamingChannel.shutdown(); - } - }); - closer.register(() -> { - if (mRpcChannel != null) { - mRpcChannel.shutdown(); - } - }); - closer.register(() -> { - if (mTracker != null) { - mTracker.close(this); - } - }); - } - } - - @Override - public StreamObserver writeBlock(StreamObserver responseObserver) { - if (responseObserver instanceof DataMessageMarshallerProvider) { - DataMessageMarshaller marshaller = - ((DataMessageMarshallerProvider) responseObserver) - .getRequestMarshaller().orElseThrow(NullPointerException::new); - return mStreamingAsyncStub - .withOption(GrpcSerializationUtils.OVERRIDDEN_METHOD_DESCRIPTOR, - BlockWorkerGrpc.getWriteBlockMethod().toBuilder() - .setRequestMarshaller(marshaller) - .build()) - .writeBlock(responseObserver); - } else { - return mStreamingAsyncStub.writeBlock(responseObserver); - } - } - - @Override - public StreamObserver readBlock(StreamObserver responseObserver) { - if (responseObserver instanceof DataMessageMarshallerProvider) { - DataMessageMarshaller marshaller = - ((DataMessageMarshallerProvider) responseObserver) - .getResponseMarshaller().orElseThrow(NullPointerException::new); - return mStreamingAsyncStub - .withOption(GrpcSerializationUtils.OVERRIDDEN_METHOD_DESCRIPTOR, - BlockWorkerGrpc.getReadBlockMethod().toBuilder() - .setResponseMarshaller(marshaller) - .build()) - .readBlock(responseObserver); - } else { - return mStreamingAsyncStub.readBlock(responseObserver); - } - } - - @Override - public StreamObserver createLocalBlock( - StreamObserver responseObserver) { - return mStreamingAsyncStub.createLocalBlock(responseObserver); - } - - @Override - public StreamObserver openLocalBlock( - StreamObserver responseObserver) { - return mStreamingAsyncStub.openLocalBlock(responseObserver); - } - - @Override - public RemoveBlockResponse removeBlock(final RemoveBlockRequest request) { - return mRpcBlockingStub.withDeadlineAfter(mRpcTimeoutMs, TimeUnit.MILLISECONDS) - .removeBlock(request); - } - - @Override - public MoveBlockResponse moveBlock(MoveBlockRequest request) { - return mRpcBlockingStub.withDeadlineAfter(mRpcTimeoutMs, TimeUnit.MILLISECONDS) - .moveBlock(request); - } - - @Override - public ClearMetricsResponse clearMetrics(ClearMetricsRequest request) { - return mRpcBlockingStub.withDeadlineAfter(mRpcTimeoutMs, TimeUnit.MILLISECONDS) - .clearMetrics(request); - } - - @Override - public void cache(CacheRequest request) { - boolean async = request.getAsync(); - try { - mRpcBlockingStub.withDeadlineAfter(mRpcTimeoutMs, TimeUnit.MILLISECONDS).cache(request); - } catch (Exception e) { - if (!async) { - throw e; - } - LOG.warn("Error sending async cache request {} to worker {}.", request, mAddress, e); - } - } - - @Override - public void freeWorker() { - mRpcBlockingStub.withDeadlineAfter(mRpcTimeoutMs, TimeUnit.MILLISECONDS) - .freeWorker(FreeWorkerRequest.getDefaultInstance()); - } - - @Override - public ListenableFuture load(LoadRequest request) { - return mRpcFutureStub.load(request); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/LocalFileDataReader.java b/core/client/fs/src/main/java/alluxio/client/block/stream/LocalFileDataReader.java deleted file mode 100644 index e8d611b83eaa..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/LocalFileDataReader.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.client.ReadType; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.options.InStreamOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.NotFoundException; -import alluxio.grpc.OpenLocalBlockRequest; -import alluxio.grpc.OpenLocalBlockResponse; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.network.protocol.databuffer.NioDataBuffer; -import alluxio.resource.CloseableResource; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.io.LocalFileBlockReader; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Paths; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A data reader that simply reads packets from a local file. - */ -@NotThreadSafe -public final class LocalFileDataReader implements DataReader { - /** The file reader to read a local block. */ - private final LocalFileBlockReader mReader; - private final long mEnd; - private final long mChunkSize; - private long mPos; - private boolean mClosed; - - /** - * Creates an instance of {@link LocalFileDataReader}. - * - * @param reader the file reader to the block path - * @param offset the offset - * @param len the length to read - * @param chunkSize the chunk size - */ - private LocalFileDataReader(LocalFileBlockReader reader, long offset, long len, long chunkSize) { - mReader = reader; - Preconditions.checkArgument(chunkSize > 0); - mPos = offset; - mEnd = Math.min(mReader.getLength(), offset + len); - mChunkSize = chunkSize; - } - - @Override - public DataBuffer readChunk() throws IOException { - if (mPos >= mEnd) { - return null; - } - ByteBuffer buffer = mReader.read(mPos, Math.min(mChunkSize, mEnd - mPos)); - DataBuffer dataBuffer = new NioDataBuffer(buffer, buffer.remaining()); - mPos += dataBuffer.getLength(); - MetricsSystem.counter(MetricKey.CLIENT_BYTES_READ_LOCAL.getName()).inc(dataBuffer.getLength()); - MetricsSystem.meter(MetricKey.CLIENT_BYTES_READ_LOCAL_THROUGHPUT.getName()) - .mark(dataBuffer.getLength()); - return dataBuffer; - } - - @Override - public long pos() { - return mPos; - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - mClosed = true; - mReader.decreaseUsageCount(); - } - - /** - * Factory class to create {@link LocalFileDataReader}s. - */ - @NotThreadSafe - public static class Factory implements DataReader.Factory { - private final CloseableResource mBlockWorker; - private final String mPath; - private final long mLocalReaderChunkSize; - private final GrpcBlockingStream mStream; - - private LocalFileBlockReader mReader; - private final long mDataTimeoutMs; - private boolean mClosed; - - /** - * Creates an instance of {@link Factory}. - * - * @param context the file system context - * @param address the worker address - * @param blockId the block ID - * @param localReaderChunkSize chunk size in bytes for local reads - * @param options the InStream options - */ - public Factory(FileSystemContext context, WorkerNetAddress address, long blockId, - long localReaderChunkSize, InStreamOptions options) throws IOException { - AlluxioConfiguration conf = context.getClusterConf(); - mLocalReaderChunkSize = localReaderChunkSize; - mDataTimeoutMs = conf.getMs(PropertyKey.USER_STREAMING_DATA_READ_TIMEOUT); - if (conf.getBoolean(PropertyKey.USER_DIRECT_MEMORY_IO_ENABLED)) { - mBlockWorker = null; - mStream = null; - PropertyKey tierDirPathConf = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(0); - String storageDir = conf.getString(tierDirPathConf).split(",")[0]; - String workerDir = conf.getString(PropertyKey.WORKER_DATA_FOLDER); - mPath = Paths.get(storageDir, workerDir, Long.toString(blockId)).toString(); - return; - } - - boolean isPromote = ReadType.fromProto(options.getOptions().getReadType()).isPromote(); - OpenLocalBlockRequest request = OpenLocalBlockRequest.newBuilder() - .setBlockId(blockId).setPromote(isPromote).build(); - - mBlockWorker = context.acquireBlockWorkerClient(address); - try { - mStream = new GrpcBlockingStream<>(mBlockWorker.get()::openLocalBlock, - conf.getInt(PropertyKey.USER_STREAMING_READER_BUFFER_SIZE_MESSAGES), - MoreObjects.toStringHelper(LocalFileDataReader.class) - .add("request", request) - .add("address", address) - .toString()); - mStream.send(request, mDataTimeoutMs); - OpenLocalBlockResponse response = mStream.receive(mDataTimeoutMs); - Preconditions.checkState(response.hasPath()); - mPath = response.getPath(); - } catch (Exception e) { - mBlockWorker.close(); - throw e; - } - - if (!Files.exists(Paths.get(mPath))) { - mStream.close(); - mBlockWorker.close(); - throw new NotFoundException(String.format( - "LocalFileDataReader can not find the path:%s", mPath)); - } - } - - @Override - public DataReader create(long offset, long len) throws IOException { - if (mReader == null) { - mReader = new LocalFileBlockReader(mPath); - } - Preconditions.checkState(mReader.getUsageCount() == 0); - mReader.increaseUsageCount(); - return new LocalFileDataReader(mReader, offset, len, mLocalReaderChunkSize); - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - try { - if (mReader != null) { - mReader.close(); - } - if (mStream != null) { - mStream.close(); - mStream.waitForComplete(mDataTimeoutMs); - } - } finally { - mClosed = true; - if (mBlockWorker != null) { - mBlockWorker.close(); - } - } - } - } -} - diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/LocalFileDataWriter.java b/core/client/fs/src/main/java/alluxio/client/block/stream/LocalFileDataWriter.java deleted file mode 100644 index b3cd3778350c..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/LocalFileDataWriter.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.client.WriteType; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.CreateLocalBlockRequest; -import alluxio.grpc.CreateLocalBlockResponse; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.resource.CloseableResource; -import alluxio.util.CommonUtils; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.io.LocalFileBlockWriter; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import io.netty.buffer.ByteBuf; - -import java.io.IOException; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A local data writer that simply writes packets to a local file. - */ -@NotThreadSafe -public final class LocalFileDataWriter implements DataWriter { - - private final long mFileBufferBytes; - private final long mDataTimeoutMs; - private final LocalFileBlockWriter mWriter; - private final long mChunkSize; - private final CreateLocalBlockRequest mCreateRequest; - private final Closer mCloser; - private final GrpcBlockingStream mStream; - - /** The position to write the next byte at. */ - private long mPos; - /** The number of bytes reserved on the block worker to hold the block. */ - private long mPosReserved; - - /** - * Creates an instance of {@link LocalFileDataWriter}. This requires the block to be locked - * beforehand. - * - * @param context the file system context - * @param address the worker network address - * @param blockId the block ID - * @param blockSize the block size in bytes - * @param options the output stream options - * @return the {@link LocalFileDataWriter} created - */ - public static LocalFileDataWriter create(final FileSystemContext context, - final WorkerNetAddress address, - long blockId, long blockSize, OutStreamOptions options) throws IOException { - AlluxioConfiguration conf = context.getClusterConf(); - long chunkSize = conf.getBytes(PropertyKey.USER_LOCAL_WRITER_CHUNK_SIZE_BYTES); - - Closer closer = Closer.create(); - try { - CloseableResource blockWorker = - context.acquireBlockWorkerClient(address); - closer.register(blockWorker); - int writerBufferSizeMessages = - conf.getInt(PropertyKey.USER_STREAMING_WRITER_BUFFER_SIZE_MESSAGES); - long fileBufferBytes = conf.getBytes(PropertyKey.USER_FILE_BUFFER_BYTES); - long dataTimeout = conf.getMs(PropertyKey.USER_STREAMING_DATA_WRITE_TIMEOUT); - // in cases where we know precise block size, make more accurate reservation. - long reservedBytes = Math.min(blockSize, conf.getBytes(PropertyKey.USER_FILE_RESERVED_BYTES)); - - CreateLocalBlockRequest.Builder builder = - CreateLocalBlockRequest.newBuilder().setBlockId(blockId).setTier(options.getWriteTier()) - .setSpaceToReserve(reservedBytes).setMediumType(options.getMediumType()) - .setPinOnCreate(options.getWriteType() == WriteType.ASYNC_THROUGH); - if (options.getWriteType() == WriteType.ASYNC_THROUGH - && conf.getBoolean(PropertyKey.USER_FILE_UFS_TIER_ENABLED)) { - builder.setCleanupOnFailure(false); - } - CreateLocalBlockRequest createRequest = builder.build(); - - GrpcBlockingStream stream = - new GrpcBlockingStream<>(blockWorker.get()::createLocalBlock, writerBufferSizeMessages, - MoreObjects.toStringHelper(LocalFileDataWriter.class) - .add("request", createRequest) - .add("address", address) - .toString()); - stream.send(createRequest, dataTimeout); - CreateLocalBlockResponse response = stream.receive(dataTimeout); - Preconditions.checkState(response != null && response.hasPath()); - LocalFileBlockWriter writer = - closer.register(new LocalFileBlockWriter(response.getPath())); - return new LocalFileDataWriter(chunkSize, writer, createRequest, stream, closer, - fileBufferBytes, dataTimeout); - } catch (Exception e) { - throw CommonUtils.closeAndRethrow(closer, e); - } - } - - @Override - public long pos() { - return mPos; - } - - @Override - public int chunkSize() { - return (int) mChunkSize; - } - - @Override - public void writeChunk(final ByteBuf buf) throws IOException { - try { - Preconditions.checkState(!mStream.isCanceled() && !mStream.isClosed(), - "DataWriter is closed while writing chunks."); - int sz = buf.readableBytes(); - ensureReserved(mPos + sz); - mPos += sz; - Preconditions.checkState(mWriter.append(buf) == sz); - MetricsSystem.counter(MetricKey.CLIENT_BYTES_WRITTEN_LOCAL.getName()).inc(sz); - MetricsSystem.meter(MetricKey.CLIENT_BYTES_WRITTEN_LOCAL_THROUGHPUT.getName()).mark(sz); - } finally { - buf.release(); - } - } - - @Override - public void cancel() throws IOException { - mCloser.register(mStream::cancel); - mCloser.close(); - } - - @Override - public void flush() {} - - @Override - public void close() throws IOException { - mCloser.register(() -> { - mStream.close(); - mStream.waitForComplete(mDataTimeoutMs); - }); - mCloser.close(); - } - - /** - * Creates an instance of {@link LocalFileDataWriter}. - * - * @param packetSize the packet size - * @param writer the file writer - * @param createRequest the request - * @param stream the gRPC stream - * @param closer the closer - */ - private LocalFileDataWriter(long packetSize, LocalFileBlockWriter writer, - CreateLocalBlockRequest createRequest, - GrpcBlockingStream stream, - Closer closer, long fileBufferBytes, long dataTimeoutMs) { - mFileBufferBytes = fileBufferBytes; - mDataTimeoutMs = dataTimeoutMs; - mCloser = closer; - mWriter = writer; - mCreateRequest = createRequest; - mStream = stream; - mPosReserved = createRequest.getSpaceToReserve(); - mChunkSize = packetSize; - } - - /** - * Reserves enough space in the block worker. - * - * @param pos the pos of the file/block to reserve to - */ - private void ensureReserved(long pos) throws IOException { - if (pos <= mPosReserved) { - return; - } - long toReserve = Math.max(pos - mPosReserved, mFileBufferBytes); - CreateLocalBlockRequest request = - mCreateRequest.toBuilder().setSpaceToReserve(toReserve).setOnlyReserveSpace(true).build(); - mStream.send(request, mDataTimeoutMs); - CreateLocalBlockResponse response = mStream.receive(mDataTimeoutMs); - Preconditions.checkState(response != null, - String.format("Stream closed while waiting for reserve request %s", request)); - Preconditions.checkState(!response.hasPath(), - String.format("Invalid response for reserve request %s", request)); - mPosReserved += toReserve; - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/UfsFallbackLocalFileDataWriter.java b/core/client/fs/src/main/java/alluxio/client/block/stream/UfsFallbackLocalFileDataWriter.java deleted file mode 100644 index b534ea28f326..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/UfsFallbackLocalFileDataWriter.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.exception.status.ResourceExhaustedException; -import alluxio.grpc.RequestType; -import alluxio.wire.WorkerNetAddress; - -import io.netty.buffer.ByteBuf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A data writer that writes to local first and fallback to UFS block writes when the block - * storage on this local worker is full. - */ -@NotThreadSafe -public final class UfsFallbackLocalFileDataWriter implements DataWriter { - private static final Logger LOG = LoggerFactory.getLogger(UfsFallbackLocalFileDataWriter.class); - private final DataWriter mLocalFileDataWriter; - private final FileSystemContext mContext; - private final WorkerNetAddress mWorkerNetAddress; - private final long mBlockSize; - private final long mBlockId; - private final OutStreamOptions mOutStreamOptions; - private GrpcDataWriter mGrpcDataWriter; - private boolean mIsWritingToLocal; - - /** - * @param context the file system context - * @param address the worker network address - * @param blockId the block ID - * @param blockSize the block size - * @param options the output stream options - * @return the {@link UfsFallbackLocalFileDataWriter} instance created - */ - public static UfsFallbackLocalFileDataWriter create(FileSystemContext context, - WorkerNetAddress address, long blockId, long blockSize, OutStreamOptions options) - throws IOException { - try { - LocalFileDataWriter localFilePacketWriter = - LocalFileDataWriter.create(context, address, blockId, blockSize, options); - return new UfsFallbackLocalFileDataWriter(localFilePacketWriter, null, context, address, - blockId, blockSize, options); - } catch (ResourceExhaustedException e) { - LOG.warn("Fallback to create new block {} in UFS due to a failure of insufficient space on " - + "the local worker: {}", blockId, e.toString()); - } - // Failed to create the local writer due to insufficient space, fallback to gRPC data writer - // directly - GrpcDataWriter grpcDataWriter = GrpcDataWriter - .create(context, address, blockId, blockSize, RequestType.UFS_FALLBACK_BLOCK, - options); - return new UfsFallbackLocalFileDataWriter(null, grpcDataWriter, context, address, blockId, - blockSize, options); - } - - UfsFallbackLocalFileDataWriter(DataWriter localFileDataWriter, - GrpcDataWriter grpcDataWriter, FileSystemContext context, - final WorkerNetAddress address, long blockId, long blockSize, OutStreamOptions options) { - mLocalFileDataWriter = localFileDataWriter; - mGrpcDataWriter = grpcDataWriter; - mBlockId = blockId; - mContext = context; - mWorkerNetAddress = address; - mBlockSize = blockSize; - mOutStreamOptions = options; - mIsWritingToLocal = mLocalFileDataWriter != null; - } - - @Override - public void writeChunk(ByteBuf chunk) throws IOException { - if (mIsWritingToLocal) { - long pos = mLocalFileDataWriter.pos(); - try { - // chunk.refcount++ to ensure chunk not garbage-collected if writeChunk fails - chunk.retain(); - // chunk.refcount-- inside regardless of exception - mLocalFileDataWriter.writeChunk(chunk); - // chunk.refcount-- on success - chunk.release(); - return; - } catch (ResourceExhaustedException e) { - LOG.warn("Fallback to write to UFS for block {} due to a failure of insufficient space " - + "on the local worker: {}", mBlockId, e.toString()); - mIsWritingToLocal = false; - } - try { - if (pos == 0) { - // Nothing has been written to temp block, we can cancel this failed local writer and - // cleanup the temp block. - mLocalFileDataWriter.cancel(); - } else { - // Note that, we can not cancel mLocalFileDataWriter now as the cancel message may - // arrive and clean the temp block before it is written to UFS. - mLocalFileDataWriter.flush(); - } - // Close the block writer. We do not close the mLocalFileDataWriter to prevent the worker - // completes the block, commit it and remove it. - //mLocalFileDataWriter.getWriter().close(); - mGrpcDataWriter = GrpcDataWriter - .create(mContext, mWorkerNetAddress, mBlockId, mBlockSize, - RequestType.UFS_FALLBACK_BLOCK, mOutStreamOptions); - // Instruct the server to write the previously transferred data from temp block to UFS only - // when there is data already written. - if (pos > 0) { - mGrpcDataWriter.writeFallbackInitRequest(pos); - } - } catch (Exception e) { - // chunk.refcount-- on exception - chunk.release(); - throw new IOException("Failed to switch to writing block " + mBlockId + " to UFS", e); - } - } - mGrpcDataWriter.writeChunk(chunk); // refcount-- inside to release chunk - } - - @Override - public void flush() throws IOException { - if (mIsWritingToLocal) { - mLocalFileDataWriter.flush(); - } else { - mGrpcDataWriter.flush(); - } - } - - @Override - public int chunkSize() { - if (mIsWritingToLocal) { - return mLocalFileDataWriter.chunkSize(); - } else { - return mGrpcDataWriter.chunkSize(); - } - } - - @Override - public long pos() { - if (mIsWritingToLocal) { - return mLocalFileDataWriter.pos(); - } else { - return mGrpcDataWriter.pos(); - } - } - - @Override - public void cancel() throws IOException { - if (mIsWritingToLocal) { - mLocalFileDataWriter.cancel(); - } else { - // Clean up the state of previous temp block left over - if (mLocalFileDataWriter != null) { - mLocalFileDataWriter.cancel(); - } - mGrpcDataWriter.cancel(); - } - } - - @Override - public void close() throws IOException { - if (mIsWritingToLocal) { - mLocalFileDataWriter.close(); - } else { - // Clean up the state of previous temp block left over - if (mLocalFileDataWriter != null) { - mLocalFileDataWriter.cancel(); - } - mGrpcDataWriter.close(); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/stream/UnderFileSystemFileOutStream.java b/core/client/fs/src/main/java/alluxio/client/block/stream/UnderFileSystemFileOutStream.java deleted file mode 100644 index fd0066ab3786..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/stream/UnderFileSystemFileOutStream.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.grpc.RequestType; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.wire.WorkerNetAddress; - -import com.codahale.metrics.Timer; - -import java.io.IOException; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Provides a streaming API to write to a file in the under file system through an Alluxio - * worker's data server. - */ -@NotThreadSafe -public class UnderFileSystemFileOutStream extends BlockOutStream { - private static final int ID_UNUSED = -1; - - /** - * Creates an instance of {@link UnderFileSystemFileOutStream} that writes to a UFS file. - * - * @param context the file system context - * @param address the data server address - * @param options the out stream options - * @return the under file system output stream - */ - public static UnderFileSystemFileOutStream create(FileSystemContext context, - WorkerNetAddress address, OutStreamOptions options) throws IOException { - return new UnderFileSystemFileOutStream(GrpcDataWriter.create(context, address, - ID_UNUSED, Long.MAX_VALUE, RequestType.UFS_FILE, options), address); - } - - /** - * Constructs a new {@link UnderFileSystemFileOutStream} with only one {@link DataWriter}. - * - * @param dataWriter the data writer - */ - protected UnderFileSystemFileOutStream(DataWriter dataWriter, WorkerNetAddress address) { - super(dataWriter, Long.MAX_VALUE, address); - } - - @Override - public void close() throws IOException { - try (Timer.Context ctx = MetricsSystem - .uniformTimer(MetricKey.CLOSE_UFS_OUTSTREAM_LATENCY.getName()).time()) { - super.close(); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/block/util/BlockLocationUtils.java b/core/client/fs/src/main/java/alluxio/client/block/util/BlockLocationUtils.java deleted file mode 100644 index 9e252002c85d..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/block/util/BlockLocationUtils.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.util; - -import alluxio.Constants; -import alluxio.collections.Pair; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.util.TieredIdentityUtils; -import alluxio.util.network.NettyUtils; -import alluxio.wire.TieredIdentity; -import alluxio.wire.WorkerNetAddress; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Utility functions for working with block locations. - */ -@ThreadSafe -public final class BlockLocationUtils { - private static final Logger LOG = LoggerFactory.getLogger(BlockLocationUtils.class); - - /** - * @param tieredIdentity the tiered identity - * @param addresses the candidate worker addresses - * @param conf Alluxio configuration - * @return the first in the pair indicates the address closest to this one. If none of the - * identities match, the first address is returned. the second in the pair indicates - * whether or not the location is local - */ - public static Optional> nearest(TieredIdentity tieredIdentity, - List addresses, AlluxioConfiguration conf) { - if (conf.getBoolean(PropertyKey.WORKER_DATA_SERVER_DOMAIN_SOCKET_AS_UUID)) { - // Determine by inspecting the file system if worker is local - for (WorkerNetAddress addr : addresses) { - if (NettyUtils.isDomainSocketAccessible(addr, conf)) { - LOG.debug("Found local worker by file system inspection of path {}", - addr.getDomainSocketPath()); - // Returns the first local worker and does not shuffle - return Optional.of(new Pair<>(addr, true)); - } - } - } - // Find nearest tiered identity - Optional nearestIdentity = TieredIdentityUtils.nearest(tieredIdentity, - addresses.stream().map(addr -> addr.getTieredIdentity()).collect(Collectors.toList()), - conf); - if (!nearestIdentity.isPresent()) { - return Optional.empty(); - } - boolean isLocal = tieredIdentity.getTier(0).getTierName().equals(Constants.LOCALITY_NODE) - && tieredIdentity.topTiersMatch(nearestIdentity.get()); - Optional dataSource = addresses.stream() - .filter(addr -> addr.getTieredIdentity().equals(nearestIdentity.get())).findFirst(); - if (!dataSource.isPresent()) { - return Optional.empty(); - } - return Optional.of(new Pair<>(dataSource.get(), isLocal)); - } - - private BlockLocationUtils() {} // prevent instantiation -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/AlluxioFileInStream.java b/core/client/fs/src/main/java/alluxio/client/file/AlluxioFileInStream.java deleted file mode 100644 index f2a7f63eb7f6..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/AlluxioFileInStream.java +++ /dev/null @@ -1,515 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import alluxio.AlluxioURI; -import alluxio.annotation.PublicApi; -import alluxio.client.ReadType; -import alluxio.client.block.BlockStoreClient; -import alluxio.client.block.stream.BlockInStream; -import alluxio.client.block.stream.BlockWorkerClient; -import alluxio.client.file.options.InStreamOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.PreconditionMessage; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.OutOfRangeException; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.FileSystemMasterCommonPOptions; -import alluxio.grpc.ListStatusPOptions; -import alluxio.resource.CloseableResource; -import alluxio.retry.ExponentialTimeBoundedRetry; -import alluxio.retry.RetryPolicy; -import alluxio.util.CommonUtils; -import alluxio.util.FileSystemOptionsUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.BlockLocation; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; -import com.google.common.io.Closer; -import io.grpc.StatusRuntimeException; -import io.netty.util.internal.OutOfDirectMemoryError; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * An implementation of {@link FileInStream} for data stored in Alluxio. - * - * This class wraps the block in stream for each of the blocks in the file and abstracts the - * switching between streams. The backing streams can read from Alluxio space in the local machine, - * remote machines, or the under storage system. - * - * The internal bookkeeping works as follows: - * - * 1. {@link #updateStream()} is a potentially expensive operation and is responsible for - * creating new BlockInStreams and updating {@link #mBlockInStream}. After calling this method, - * {@link #mBlockInStream} is ready to serve reads from the current {@link #mPosition}. - * 2. {@link #mPosition} can become out of sync with {@link #mBlockInStream} when seek or skip is - * called. When this happens, {@link #mBlockInStream} is set to null and no effort is made to - * sync between the two until {@link #updateStream()} is called. - * 3. {@link #updateStream()} is only called when followed by a read request. Thus, if a - * {@link #mBlockInStream} is created, it is guaranteed we read at least one byte from it. - */ -@PublicApi -@NotThreadSafe -public class AlluxioFileInStream extends FileInStream { - private static final Logger LOG = LoggerFactory.getLogger(AlluxioFileInStream.class); - - private Supplier mRetryPolicySupplier; - private final URIStatus mStatus; - private final InStreamOptions mOptions; - private final BlockStoreClient mBlockStore; - private final FileSystemContext mContext; - private final boolean mPassiveCachingEnabled; - - /* Convenience values derived from mStatus, use these instead of querying mStatus. */ - /** Length of the file in bytes. */ - private final long mLength; - /** Block size in bytes. */ - private final long mBlockSize; - - /* Underlying stream and associated bookkeeping. */ - /** Current offset in the file. */ - private long mPosition; - /** Underlying block stream, null if a position change has invalidated the previous stream. */ - private BlockInStream mBlockInStream; - - /** Cached block stream for the positioned read API. */ - private BlockInStream mCachedPositionedReadStream; - - /** The last block id for which async cache was triggered. */ - private long mLastBlockIdCached; - - /** A map of worker addresses to the most recent epoch time when client fails to read from it. */ - private Map mFailedWorkers = new HashMap<>(); - - private Closer mCloser; - - protected AlluxioFileInStream(URIStatus status, InStreamOptions options, - FileSystemContext context) { - mCloser = Closer.create(); - // Acquire a resource to block FileSystemContext reinitialization, this needs to be done before - // using mContext. - // The resource will be released in close(). - mContext = context; - mCloser.register(mContext.blockReinit()); - try { - AlluxioConfiguration conf = mContext.getPathConf(new AlluxioURI(status.getPath())); - mPassiveCachingEnabled = conf.getBoolean(PropertyKey.USER_FILE_PASSIVE_CACHE_ENABLED); - final Duration blockReadRetryMaxDuration = - conf.getDuration(PropertyKey.USER_BLOCK_READ_RETRY_MAX_DURATION); - final Duration blockReadRetrySleepBase = - conf.getDuration(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MIN); - final Duration blockReadRetrySleepMax = - conf.getDuration(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MAX); - mRetryPolicySupplier = - () -> ExponentialTimeBoundedRetry.builder() - .withMaxDuration(blockReadRetryMaxDuration) - .withInitialSleep(blockReadRetrySleepBase) - .withMaxSleep(blockReadRetrySleepMax) - .withSkipInitialSleep().build(); - mStatus = status; - mOptions = options; - mBlockStore = BlockStoreClient.create(mContext); - mLength = mStatus.getLength(); - mBlockSize = mStatus.getBlockSizeBytes(); - mPosition = 0; - mBlockInStream = null; - mCachedPositionedReadStream = null; - mLastBlockIdCached = 0; - } catch (Throwable t) { - // If there is any exception, including RuntimeException such as thrown by conf.getBoolean, - // release the acquired resource, otherwise, FileSystemContext reinitialization will be - // blocked forever. - throw CommonUtils.closeAndRethrowRuntimeException(mCloser, t); - } - } - - /* Input Stream methods */ - @Override - public int read() throws IOException { - if (mPosition == mLength) { // at end of file - return -1; - } - RetryPolicy retry = mRetryPolicySupplier.get(); - IOException lastException = null; - while (retry.attempt()) { - try { - updateStream(); - int result = mBlockInStream.read(); - if (result != -1) { - mPosition++; - } - if (mBlockInStream.remaining() == 0) { - closeBlockInStream(mBlockInStream); - } - return result; - } catch (IOException e) { - lastException = e; - if (mBlockInStream != null) { - handleRetryableException(mBlockInStream, e); - mBlockInStream = null; - } - if (e instanceof OutOfRangeException) { - refreshMetadataOnMismatchedLength((OutOfRangeException) e); - } - } - } - throw lastException; - } - - @Override - public int read(ByteBuffer byteBuffer, int off, int len) throws IOException { - Preconditions.checkArgument(off >= 0 && len >= 0 && len + off <= byteBuffer.capacity(), - PreconditionMessage.ERR_BUFFER_STATE.toString(), byteBuffer.capacity(), off, len); - if (len == 0) { - return 0; - } - if (mPosition == mLength) { // at end of file - return -1; - } - - int bytesLeft = len; - int currentOffset = off; - RetryPolicy retry = mRetryPolicySupplier.get(); - IOException lastException = null; - while (bytesLeft > 0 && mPosition != mLength && retry.attempt()) { - try { - updateStream(); - int bytesRead = mBlockInStream.read(byteBuffer, currentOffset, bytesLeft); - if (bytesRead > 0) { - bytesLeft -= bytesRead; - currentOffset += bytesRead; - mPosition += bytesRead; - } - retry = mRetryPolicySupplier.get(); - lastException = null; - if (mBlockInStream.remaining() == 0) { - closeBlockInStream(mBlockInStream); - } - } catch (IOException e) { - lastException = e; - if (mBlockInStream != null) { - handleRetryableException(mBlockInStream, e); - mBlockInStream = null; - } - if (e instanceof OutOfRangeException) { - refreshMetadataOnMismatchedLength((OutOfRangeException) e); - } - } - } - if (lastException != null) { - throw lastException; - } - return len - bytesLeft; - } - - // When Alluxio detects the underlying file length has changed, - // force a sync to update the latest metadata, then abort the current stream - // The user should restart the stream and read the updated file - private void refreshMetadataOnMismatchedLength(OutOfRangeException e) { - try (CloseableResource client = - mContext.acquireMasterClientResource()) { - // Force refresh the file metadata by loadMetadata - AlluxioURI path = new AlluxioURI(mStatus.getPath()); - ListStatusPOptions refreshPathOptions = ListStatusPOptions.newBuilder() - .setCommonOptions( - FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build()) - .setLoadMetadataOnly(true) - .build(); - ListStatusPOptions mergedOptions = FileSystemOptionsUtils.listStatusDefaults( - mContext.getPathConf(path)).toBuilder().mergeFrom(refreshPathOptions).build(); - client.get().listStatus(path, mergedOptions); - LOG.info("Notified the master that {} should be sync-ed with UFS on the next access", - mStatus.getPath()); - throw new IllegalStateException(e.getMessage()); - } catch (AlluxioStatusException x) { - String msg = String.format("Failed to force a metadata sync on path %s. " - + "Please manually sync metadata by `bin/alluxio fs loadMetadata -f {path}` " - + "before you retry reading this file.", mStatus.getPath()); - throw new IllegalStateException(msg, e); - } - } - - @Override - public long skip(long n) throws IOException { - if (n <= 0) { - return 0; - } - - long toSkip = Math.min(n, mLength - mPosition); - seek(mPosition + toSkip); - return toSkip; - } - - @Override - public void close() throws IOException { - closeBlockInStream(mBlockInStream); - closeBlockInStream(mCachedPositionedReadStream); - mCloser.close(); - } - - /* Bounded Stream methods */ - @Override - public long remaining() { - return mLength - mPosition; - } - - /* Positioned Readable methods */ - @Override - public int positionedRead(long pos, byte[] b, int off, int len) throws IOException { - return positionedReadInternal(pos, b, off, len); - } - - private int positionedReadInternal(long pos, byte[] b, int off, int len) throws IOException { - if (pos < 0 || pos >= mLength) { - return -1; - } - - if (len < mContext.getPathConf(new AlluxioURI(mStatus.getPath())) - .getBytes(PropertyKey.USER_FILE_SEQUENTIAL_PREAD_THRESHOLD)) { - mOptions.setPositionShort(true); - } - int lenCopy = len; - RetryPolicy retry = mRetryPolicySupplier.get(); - IOException lastException = null; - while (len > 0 && retry.attempt()) { - if (pos >= mLength) { - break; - } - long blockId = mStatus.getBlockIds().get(Math.toIntExact(pos / mBlockSize)); - try { - // Positioned read may be called multiple times for the same block. Caching the in-stream - // allows us to avoid the block store rpc to open a new stream for each call. - if (mCachedPositionedReadStream == null) { - mCachedPositionedReadStream = mBlockStore.getInStream(blockId, mOptions, mFailedWorkers); - } else if (mCachedPositionedReadStream.getId() != blockId) { - closeBlockInStream(mCachedPositionedReadStream); - mCachedPositionedReadStream = mBlockStore.getInStream(blockId, mOptions, mFailedWorkers); - } - long offset = pos % mBlockSize; - int bytesRead = mCachedPositionedReadStream.positionedRead(offset, b, off, - (int) Math.min(mBlockSize - offset, len)); - Preconditions.checkState(bytesRead > 0, "No data is read before EOF"); - pos += bytesRead; - off += bytesRead; - len -= bytesRead; - retry = mRetryPolicySupplier.get(); - lastException = null; - BlockInStream.BlockInStreamSource source = mCachedPositionedReadStream.getSource(); - if (source != BlockInStream.BlockInStreamSource.NODE_LOCAL - && source != BlockInStream.BlockInStreamSource.PROCESS_LOCAL) { - triggerAsyncCaching(mCachedPositionedReadStream); - } - if (bytesRead == mBlockSize - offset) { - mCachedPositionedReadStream.close(); - mCachedPositionedReadStream = null; - } - } catch (IOException e) { - lastException = e; - if (mCachedPositionedReadStream != null) { - handleRetryableException(mCachedPositionedReadStream, e); - mCachedPositionedReadStream = null; - } - } - } - if (lastException != null) { - throw lastException; - } - return lenCopy - len; - } - - /* Seekable methods */ - @Override - public long getPos() { - return mPosition; - } - - @Override - public void seek(long pos) throws IOException { - if (mPosition == pos) { - return; - } - Preconditions.checkArgument(pos >= 0, PreconditionMessage.ERR_SEEK_NEGATIVE.toString(), pos); - Preconditions.checkArgument(pos <= mLength, - PreconditionMessage.ERR_SEEK_PAST_END_OF_FILE.toString(), pos); - - if (mBlockInStream == null) { // no current stream open, advance position - mPosition = pos; - return; - } - - long delta = pos - mPosition; - if (delta <= mBlockInStream.remaining() && delta >= -mBlockInStream.getPos()) { // within block - mBlockInStream.seek(mBlockInStream.getPos() + delta); - } else { // close the underlying stream as the new position is no longer in bounds - closeBlockInStream(mBlockInStream); - } - mPosition += delta; - } - - /** - * Initializes the underlying block stream if necessary. This method must be called before - * reading from mBlockInStream. - */ - private void updateStream() throws IOException { - if (mBlockInStream != null && mBlockInStream.remaining() > 0) { // can still read from stream - return; - } - - if (mBlockInStream != null && mBlockInStream.remaining() == 0) { // current stream is done - closeBlockInStream(mBlockInStream); - } - - /* Create a new stream to read from mPosition. */ - // Calculate block id. - long blockId = mStatus.getBlockIds().get(Math.toIntExact(mPosition / mBlockSize)); - BlockInfo blockInfo = mStatus.getBlockInfo(blockId); - if (blockInfo == null) { - throw new IOException("No BlockInfo for block(id=" + blockId + ") of file" - + "(id=" + mStatus.getFileId() + ", path=" + mStatus.getPath() + ")"); - } - // Create stream - boolean isBlockInfoOutdated = true; - // blockInfo is "outdated" when all the locations in that blockInfo are failed workers, - // if there is at least one location that is not a failed worker, then it's not outdated. - if (mFailedWorkers.isEmpty() || mFailedWorkers.size() < blockInfo.getLocations().size()) { - isBlockInfoOutdated = false; - } else { - for (BlockLocation location : blockInfo.getLocations()) { - if (!mFailedWorkers.containsKey(location.getWorkerAddress())) { - isBlockInfoOutdated = false; - break; - } - } - } - if (isBlockInfoOutdated) { - mBlockInStream = mBlockStore.getInStream(blockId, mOptions, mFailedWorkers); - } else { - mBlockInStream = mBlockStore.getInStream(blockInfo, mOptions, mFailedWorkers); - } - // Set the stream to the correct position. - long offset = mPosition % mBlockSize; - mBlockInStream.seek(offset); - } - - private void closeBlockInStream(BlockInStream stream) throws IOException { - if (stream != null) { - BlockInStream.BlockInStreamSource blockSource = stream.getSource(); - stream.close(); - // TODO(calvin): we should be able to do a close check instead of using null - if (stream == mBlockInStream) { // if stream is instance variable, set to null - mBlockInStream = null; - } - if (blockSource == BlockInStream.BlockInStreamSource.NODE_LOCAL - || blockSource == BlockInStream.BlockInStreamSource.PROCESS_LOCAL) { - return; - } - triggerAsyncCaching(stream); - } - } - - // Send an async cache request to a worker based on read type and passive cache options. - // Note that, this is best effort - @VisibleForTesting - boolean triggerAsyncCaching(BlockInStream stream) { - final long blockId = stream.getId(); - final BlockInfo blockInfo = mStatus.getBlockInfo(blockId); - if (blockInfo == null) { - return false; - } - try { - boolean cache = ReadType.fromProto(mOptions.getOptions().getReadType()).isCache(); - boolean overReplicated = mStatus.getReplicationMax() > 0 - && blockInfo.getLocations().size() >= mStatus.getReplicationMax(); - cache = cache && !overReplicated; - // Get relevant information from the stream. - WorkerNetAddress dataSource = stream.getAddress(); - if (cache && (mLastBlockIdCached != blockId)) { - // Construct the async cache request - long blockLength = mOptions.getBlockInfo(blockId).getLength(); - String host = dataSource.getHost(); - // issues#11172: If the worker is in a container, use the container hostname - // to establish the connection. - if (!dataSource.getContainerHost().equals("")) { - LOG.debug("Worker is in a container. Use container host {} instead of physical host {}", - dataSource.getContainerHost(), host); - host = dataSource.getContainerHost(); - } - CacheRequest request = - CacheRequest.newBuilder().setBlockId(blockId).setLength(blockLength) - .setOpenUfsBlockOptions(mOptions.getOpenUfsBlockOptions(blockId)) - .setSourceHost(host).setSourcePort(dataSource.getDataPort()) - .setAsync(true).build(); - if (mPassiveCachingEnabled && mContext.hasProcessLocalWorker()) { - mContext.getProcessLocalWorker().orElseThrow(NullPointerException::new).cache(request); - mLastBlockIdCached = blockId; - return true; - } - WorkerNetAddress worker; - if (mPassiveCachingEnabled && mContext.hasNodeLocalWorker()) { - // send request to local worker - worker = mContext.getNodeLocalWorker(); - } else { // send request to data source - worker = dataSource; - } - try (CloseableResource blockWorker = - mContext.acquireBlockWorkerClient(worker)) { - blockWorker.get().cache(request); - mLastBlockIdCached = blockId; - } - } - return true; - } catch (Exception e) { - LOG.warn("Failed to complete async cache request (best effort) for block {} of file {}: {}", - stream.getId(), mStatus.getPath(), e.toString()); - return false; - } - } - - private void handleRetryableException(BlockInStream stream, IOException e) { - WorkerNetAddress workerAddress = stream.getAddress(); - - boolean causedByClientOOM = (e.getCause() instanceof StatusRuntimeException) - && (e.getCause().getCause() instanceof OutOfDirectMemoryError); - if (causedByClientOOM) { - LOG.warn("Failed to read block {} of file {} from worker {}, will retry: {}.", - stream.getId(), mStatus.getPath(), workerAddress, e.toString()); - } else { - LOG.warn("Failed to read block {} of file {} from worker {}. " - + "This worker will be skipped for future read operations, will retry: {}.", - stream.getId(), mStatus.getPath(), workerAddress, e.toString()); - } - - try { - stream.close(); - } catch (Exception ex) { - // Do not throw doing a best effort close - LOG.warn("Failed to close input stream for block {} of file {}: {}", - stream.getId(), mStatus.getPath(), ex.toString()); - } - // TODO(lu) consider recovering failed workers - if (!causedByClientOOM) { - mFailedWorkers.put(workerAddress, System.currentTimeMillis()); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/AlluxioFileOutStream.java b/core/client/fs/src/main/java/alluxio/client/file/AlluxioFileOutStream.java deleted file mode 100644 index 5930ddef119e..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/AlluxioFileOutStream.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.client.AlluxioStorageType; -import alluxio.client.UnderStorageType; -import alluxio.client.block.BlockStoreClient; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.client.block.stream.BlockOutStream; -import alluxio.client.block.stream.UnderFileSystemFileOutStream; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.PreconditionMessage; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.CompleteFilePOptions; -import alluxio.grpc.FileSystemMasterCommonPOptions; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.resource.CloseableResource; -import alluxio.retry.ExponentialTimeBoundedRetry; -import alluxio.retry.RetryPolicy; -import alluxio.util.CommonUtils; -import alluxio.util.FileSystemOptionsUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.OperationId; -import alluxio.wire.WorkerNetAddress; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Timer; -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import javax.annotation.concurrent.NotThreadSafe; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Provides a streaming API to write a file. This class wraps the BlockOutStreams for each of the - * blocks in the file and abstracts the switching between streams. The backing streams can write to - * Alluxio space in the local machine or remote machines. If the {@link UnderStorageType} is - * {@link UnderStorageType#SYNC_PERSIST}, another stream will write the data to the under storage - * system. - */ -@NotThreadSafe -public class AlluxioFileOutStream extends FileOutStream { - private static final Logger LOG = LoggerFactory.getLogger(AlluxioFileOutStream.class); - - /** Used to manage closeable resources. */ - private final Closer mCloser; - private final long mBlockSize; - private final AlluxioStorageType mAlluxioStorageType; - private final UnderStorageType mUnderStorageType; - private final FileSystemContext mContext; - private final BlockStoreClient mBlockStore; - /** Stream to the file in the under storage, null if not writing to the under storage. */ - private final UnderFileSystemFileOutStream mUnderStorageOutputStream; - private final OutStreamOptions mOptions; - - private boolean mCanceled; - private boolean mClosed; - private boolean mShouldCacheCurrentBlock; - private BlockOutStream mCurrentBlockOutStream; - private final List mPreviousBlockOutStreams; - - protected final AlluxioURI mUri; - - /** - * Creates a new file output stream. - * - * @param path the file path - * @param options the client options - * @param context the file system context - */ - public AlluxioFileOutStream(AlluxioURI path, OutStreamOptions options, FileSystemContext context) - throws IOException { - mCloser = Closer.create(); - // Acquire a resource to block FileSystemContext reinitialization, this needs to be done before - // using mContext. - // The resource will be released in close(). - mContext = context; - mCloser.register(mContext.blockReinit()); - try { - mUri = Preconditions.checkNotNull(path, "path"); - mBlockSize = options.getBlockSizeBytes(); - mAlluxioStorageType = options.getAlluxioStorageType(); - mUnderStorageType = options.getUnderStorageType(); - mOptions = options; - mBlockStore = BlockStoreClient.create(mContext); - mPreviousBlockOutStreams = new ArrayList<>(); - mClosed = false; - mCanceled = false; - mShouldCacheCurrentBlock = mAlluxioStorageType.isStore(); - mBytesWritten = 0; - - if (!mUnderStorageType.isSyncPersist()) { - mUnderStorageOutputStream = null; - } else { // Write is through to the under storage, create mUnderStorageOutputStream. - // Create retry policy for initializing write. - AlluxioConfiguration pathConf = mContext.getPathConf(path); - RetryPolicy initRetryPolicy = ExponentialTimeBoundedRetry.builder() - .withMaxDuration(pathConf.getDuration(PropertyKey.USER_FILE_WRITE_INIT_MAX_DURATION)) - .withInitialSleep(pathConf.getDuration(PropertyKey.USER_FILE_WRITE_INIT_SLEEP_MIN)) - .withMaxSleep(pathConf.getDuration(PropertyKey.USER_FILE_WRITE_INIT_SLEEP_MAX)) - .withSkipInitialSleep().build(); - // Try find a worker from policy. - Optional workerNetAddress = Optional.empty(); - while (!workerNetAddress.isPresent() && initRetryPolicy.attempt()) { - GetWorkerOptions getWorkerOptions = GetWorkerOptions.defaults() - .setBlockWorkerInfos(mContext.getCachedWorkers()) - .setBlockInfo(new BlockInfo() - .setBlockId(-1) - .setLength(0)); // not storing data to Alluxio, so block size is 0 - workerNetAddress = options.getLocationPolicy().getWorker(getWorkerOptions); - } - if (!workerNetAddress.isPresent()) { - // Assume no worker is available because block size is 0. - throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()); - } - mUnderStorageOutputStream = mCloser - .register(UnderFileSystemFileOutStream.create(mContext, - workerNetAddress.get(), mOptions)); - } - } catch (Throwable t) { - throw CommonUtils.closeAndRethrow(mCloser, t); - } - } - - @Override - public void cancel() throws IOException { - mCanceled = true; - close(); - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - try (Timer.Context ctx = MetricsSystem - .uniformTimer(MetricKey.CLOSE_ALLUXIO_OUTSTREAM_LATENCY.getName()).time()) { - if (mCurrentBlockOutStream != null) { - mPreviousBlockOutStreams.add(mCurrentBlockOutStream); - } - - CompleteFilePOptions.Builder optionsBuilder = CompleteFilePOptions.newBuilder(); - optionsBuilder.setCommonOptions(FileSystemMasterCommonPOptions.newBuilder() - .setOperationId(new OperationId(UUID.randomUUID()).toFsProto()).buildPartial()); - if (mUnderStorageType.isSyncPersist()) { - if (mCanceled) { - mUnderStorageOutputStream.cancel(); - } else { - mUnderStorageOutputStream.close(); - optionsBuilder.setUfsLength(mBytesWritten); - } - } - - if (mAlluxioStorageType.isStore()) { - if (mCanceled) { - for (BlockOutStream bos : mPreviousBlockOutStreams) { - bos.cancel(); - } - } else { - // Note, this is a workaround to prevent commit(blockN-1) and write(blockN) - // race, in worse case, this may result in commit(blockN-1) completes earlier than - // write(blockN), and blockN evicts the committed blockN-1 and causing file lost. - if (mCurrentBlockOutStream != null) { - mCurrentBlockOutStream.close(); - } - for (BlockOutStream bos : mPreviousBlockOutStreams) { - bos.close(); - } - } - } - - // Whether to complete file with async persist request. - if (!mCanceled && mUnderStorageType.isAsyncPersist() - && mOptions.getPersistenceWaitTime() != Constants.NO_AUTO_PERSIST) { - optionsBuilder.setAsyncPersistOptions( - FileSystemOptionsUtils.scheduleAsyncPersistDefaults( - mContext.getPathConf(mUri)).toBuilder() - .setCommonOptions(mOptions.getCommonOptions()) - .setPersistenceWaitTime(mOptions.getPersistenceWaitTime())); - } - - // Complete the file if it's ready to be completed. - if (!mCanceled && (mUnderStorageType.isSyncPersist() || mAlluxioStorageType.isStore())) { - try (CloseableResource masterClient = mContext - .acquireMasterClientResource()) { - masterClient.get().completeFile(mUri, optionsBuilder.build()); - } - } - } catch (Throwable e) { // must catch Throwable - throw mCloser.rethrow(e); // IOException will be thrown as-is. - } finally { - mClosed = true; - mCloser.close(); - } - } - - @Override - public void flush() throws IOException { - // TODO(yupeng): Handle flush for Alluxio storage stream as well. - if (mUnderStorageType.isSyncPersist()) { - mUnderStorageOutputStream.flush(); - } - } - - @Override - public void write(int b) throws IOException { - writeInternal(b); - } - - @Override - public void write(byte[] b) throws IOException { - Preconditions.checkArgument(b != null, PreconditionMessage.ERR_WRITE_BUFFER_NULL); - writeInternal(b, 0, b.length); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - writeInternal(b, off, len); - } - - private void writeInternal(int b) throws IOException { - if (mShouldCacheCurrentBlock) { - try { - if (mCurrentBlockOutStream == null || mCurrentBlockOutStream.remaining() == 0) { - getNextBlock(); - } - mCurrentBlockOutStream.write(b); - } catch (IOException e) { - handleCacheWriteException(e); - } - } - - if (mUnderStorageType.isSyncPersist()) { - mUnderStorageOutputStream.write(b); - Metrics.BYTES_WRITTEN_UFS.inc(); - } - mBytesWritten++; - } - - private void writeInternal(byte[] b, int off, int len) throws IOException { - Preconditions.checkArgument(b != null, PreconditionMessage.ERR_WRITE_BUFFER_NULL); - Preconditions.checkArgument(off >= 0 && len >= 0 && len + off <= b.length, - PreconditionMessage.ERR_BUFFER_STATE.toString(), b.length, off, len); - - if (mShouldCacheCurrentBlock) { - try { - int tLen = len; - int tOff = off; - while (tLen > 0) { - if (mCurrentBlockOutStream == null || mCurrentBlockOutStream.remaining() == 0) { - getNextBlock(); - } - long currentBlockLeftBytes = mCurrentBlockOutStream.remaining(); - if (currentBlockLeftBytes >= tLen) { - mCurrentBlockOutStream.write(b, tOff, tLen); - tLen = 0; - } else { - mCurrentBlockOutStream.write(b, tOff, (int) currentBlockLeftBytes); - tOff += currentBlockLeftBytes; - tLen -= currentBlockLeftBytes; - } - } - } catch (Exception e) { - handleCacheWriteException(e); - } - } - - if (mUnderStorageType.isSyncPersist()) { - mUnderStorageOutputStream.write(b, off, len); - Metrics.BYTES_WRITTEN_UFS.inc(len); - } - mBytesWritten += len; - } - - private void getNextBlock() throws IOException { - if (mCurrentBlockOutStream != null) { - Preconditions.checkState(mCurrentBlockOutStream.remaining() <= 0, - "The current block still has space left, no need to get new block"); - mCurrentBlockOutStream.flush(); - mPreviousBlockOutStreams.add(mCurrentBlockOutStream); - } - - if (mAlluxioStorageType.isStore()) { - mCurrentBlockOutStream = - mBlockStore.getOutStream(getNextBlockId(), mBlockSize, mOptions); - mShouldCacheCurrentBlock = true; - } - } - - private long getNextBlockId() throws IOException { - try (CloseableResource masterClient = mContext - .acquireMasterClientResource()) { - return masterClient.get().getNewBlockIdForFile(mUri); - } - } - - private void handleCacheWriteException(Exception e) throws IOException { - LOG.warn("Failed to write into AlluxioStore, canceling write attempt.", e); - if (!mUnderStorageType.isSyncPersist()) { - mCanceled = true; - throw new IOException(ExceptionMessage.FAILED_CACHE.getMessage(e.getMessage()), e); - } - - if (mCurrentBlockOutStream != null) { - mShouldCacheCurrentBlock = false; - mCurrentBlockOutStream.cancel(); - } - } - - /** - * Class that contains metrics about FileOutStream. - */ - @ThreadSafe - private static final class Metrics { - // Note that only counter can be added here. - // Both meter and timer need to be used inline - // because new meter and timer will be created after {@link MetricsSystem.resetAllMetrics()} - private static final Counter BYTES_WRITTEN_UFS = - MetricsSystem.counter(MetricKey.CLIENT_BYTES_WRITTEN_UFS.getName()); - - private Metrics() {} // prevent instantiation - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/BaseFileSystem.java b/core/client/fs/src/main/java/alluxio/client/file/BaseFileSystem.java deleted file mode 100644 index 6babc837edc0..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/BaseFileSystem.java +++ /dev/null @@ -1,650 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.client.block.BlockStoreClient; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.file.FileSystemContextReinitializer.ReinitBlockerResource; -import alluxio.client.file.options.InStreamOptions; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.AlluxioException; -import alluxio.exception.DirectoryNotEmptyException; -import alluxio.exception.FileAlreadyExistsException; -import alluxio.exception.FileDoesNotExistException; -import alluxio.exception.FileIncompleteException; -import alluxio.exception.InvalidPathException; -import alluxio.exception.OpenDirectoryException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.AlreadyExistsException; -import alluxio.exception.status.FailedPreconditionException; -import alluxio.exception.status.InvalidArgumentException; -import alluxio.exception.status.NotFoundException; -import alluxio.exception.status.UnauthenticatedException; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.Bits; -import alluxio.grpc.CheckAccessPOptions; -import alluxio.grpc.CreateDirectoryPOptions; -import alluxio.grpc.CreateFilePOptions; -import alluxio.grpc.DeletePOptions; -import alluxio.grpc.ExistsPOptions; -import alluxio.grpc.FreePOptions; -import alluxio.grpc.GetStatusPOptions; -import alluxio.grpc.ListStatusPOptions; -import alluxio.grpc.ListStatusPartialPOptions; -import alluxio.grpc.LoadMetadataPType; -import alluxio.grpc.MountPOptions; -import alluxio.grpc.OpenFilePOptions; -import alluxio.grpc.RenamePOptions; -import alluxio.grpc.ScheduleAsyncPersistencePOptions; -import alluxio.grpc.SetAclAction; -import alluxio.grpc.SetAclPOptions; -import alluxio.grpc.SetAttributePOptions; -import alluxio.grpc.UnmountPOptions; -import alluxio.master.MasterInquireClient; -import alluxio.resource.CloseableResource; -import alluxio.security.authorization.AclEntry; -import alluxio.uri.Authority; -import alluxio.util.FileSystemOptionsUtils; -import alluxio.wire.BlockLocation; -import alluxio.wire.BlockLocationInfo; -import alluxio.wire.FileBlockInfo; -import alluxio.wire.MountPointInfo; -import alluxio.wire.SyncPointInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import com.google.common.net.HostAndPort; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.Consumer; -import javax.annotation.concurrent.ThreadSafe; - -/** -* Default implementation of the {@link FileSystem} interface. Developers can extend this class -* instead of implementing the interface. This implementation reads and writes data through -* {@link FileInStream} and {@link FileOutStream}. This class is thread safe. -*/ -@ThreadSafe -public class BaseFileSystem implements FileSystem { - private static final Logger LOG = LoggerFactory.getLogger(BaseFileSystem.class); - /** Used to manage closeable resources. */ - private final Closer mCloser = Closer.create(); - protected final FileSystemContext mFsContext; - protected final BlockStoreClient mBlockStore; - - protected volatile boolean mClosed = false; - - /** - * Constructs a new base file system. - * - * @param fsContext file system context - */ - public BaseFileSystem(FileSystemContext fsContext) { - mFsContext = fsContext; - mBlockStore = BlockStoreClient.create(fsContext); - mCloser.register(mFsContext); - } - - /** - * Shuts down the FileSystem. Closes all thread pools and resources used to perform operations. If - * any operations are called after closing the context the behavior is undefined. - * - * @throws IOException - */ - @Override - public synchronized void close() throws IOException { - // TODO(zac) Determine the behavior when closing the context during operations. - if (!mClosed) { - mClosed = true; - } - mCloser.close(); - } - - @Override - public boolean isClosed() { - // Doesn't require locking because mClosed is volatile and marked first upon close - return mClosed; - } - - @Override - public void checkAccess(AlluxioURI path, CheckAccessPOptions options) - throws InvalidPathException, IOException, AlluxioException { - checkUri(path); - rpc(client -> { - CheckAccessPOptions mergedOptions = FileSystemOptionsUtils - .checkAccessDefaults(mFsContext.getPathConf(path)) - .toBuilder().mergeFrom(options).build(); - client.checkAccess(path, mergedOptions); - LOG.debug("Checked access {}, options: {}", path.getPath(), mergedOptions); - return null; - }); - } - - @Override - public void createDirectory(AlluxioURI path, CreateDirectoryPOptions options) - throws FileAlreadyExistsException, InvalidPathException, IOException, AlluxioException { - checkUri(path); - rpc(client -> { - CreateDirectoryPOptions mergedOptions = FileSystemOptionsUtils.createDirectoryDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - client.createDirectory(path, mergedOptions); - LOG.debug("Created directory {}, options: {}", path.getPath(), mergedOptions); - return null; - }); - } - - @Override - public FileOutStream createFile(AlluxioURI path, CreateFilePOptions options) - throws FileAlreadyExistsException, InvalidPathException, IOException, AlluxioException { - checkUri(path); - return rpc(client -> { - CreateFilePOptions mergedOptions = FileSystemOptionsUtils.createFileDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - URIStatus status = client.createFile(path, mergedOptions); - LOG.debug("Created file {}, options: {}", path.getPath(), mergedOptions); - OutStreamOptions outStreamOptions = - new OutStreamOptions(mergedOptions, mFsContext, - mFsContext.getPathConf(path)); - outStreamOptions.setUfsPath(status.getUfsPath()); - outStreamOptions.setMountId(status.getMountId()); - outStreamOptions.setAcl(status.getAcl()); - try { - return new AlluxioFileOutStream(path, outStreamOptions, mFsContext); - } catch (Exception e) { - delete(path); - throw e; - } - }); - } - - @Override - public void delete(AlluxioURI path, DeletePOptions options) - throws DirectoryNotEmptyException, FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - rpc(client -> { - DeletePOptions mergedOptions = FileSystemOptionsUtils.deleteDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - client.delete(path, mergedOptions); - LOG.debug("Deleted {}, options: {}", path.getPath(), mergedOptions); - return null; - }); - } - - @Override - public boolean exists(AlluxioURI path, final ExistsPOptions options) - throws IOException, AlluxioException { - checkUri(path); - return rpc(client -> { - ExistsPOptions mergedOptions = FileSystemOptionsUtils.existsDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - return client.exists(path, mergedOptions); - }); - } - - @Override - public void free(AlluxioURI path, final FreePOptions options) - throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - rpc(client -> { - FreePOptions mergedOptions = FileSystemOptionsUtils.freeDefaults(mFsContext.getPathConf(path)) - .toBuilder().mergeFrom(options).build(); - client.free(path, mergedOptions); - LOG.debug("Freed {}, options: {}", path.getPath(), mergedOptions); - return null; - }); - } - - @Override - public List getBlockLocations(URIStatus status) - throws IOException, AlluxioException { - List blockLocations = new ArrayList<>(); - List blocks = status.getFileBlockInfos(); - for (FileBlockInfo fileBlockInfo : blocks) { - // add the existing in-Alluxio block locations - List locations = fileBlockInfo.getBlockInfo().getLocations() - .stream().map(BlockLocation::getWorkerAddress).collect(toList()); - if (locations.isEmpty()) { // No in-Alluxio location - if (!fileBlockInfo.getUfsLocations().isEmpty()) { - // Case 1: Fallback to use under file system locations with co-located workers. - // This maps UFS locations to a worker which is co-located. - Map finalWorkerHosts = getHostWorkerMap(); - locations = fileBlockInfo.getUfsLocations().stream().map( - location -> finalWorkerHosts.get(HostAndPort.fromString(location).getHost())) - .filter(Objects::nonNull).collect(toList()); - } - if (locations.isEmpty() && mFsContext.getPathConf(new AlluxioURI(status.getPath())) - .getBoolean(PropertyKey.USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED)) { - // Case 2: Fallback to add all workers to locations so some apps (Impala) won't panic. - locations.addAll(getHostWorkerMap().values()); - Collections.shuffle(locations); - } - } - blockLocations.add(new BlockLocationInfo(fileBlockInfo, locations)); - } - return blockLocations; - } - - private Map getHostWorkerMap() throws IOException { - List workers = mFsContext.getCachedWorkers(); - return workers.stream().collect( - toMap(worker -> worker.getNetAddress().getHost(), BlockWorkerInfo::getNetAddress, - (worker1, worker2) -> worker1)); - } - - @Override - public AlluxioConfiguration getConf() { - return mFsContext.getClusterConf(); - } - - @Override - public URIStatus getStatus(AlluxioURI path, final GetStatusPOptions options) - throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - URIStatus status = rpc(client -> { - GetStatusPOptions mergedOptions = FileSystemOptionsUtils.getStatusDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - return client.getStatus(path, mergedOptions); - }); - if (!status.isCompleted()) { - LOG.debug("File {} is not yet completed. getStatus will see incomplete metadata.", path); - } - return status; - } - - @Override - public List listStatus(AlluxioURI path, final ListStatusPOptions options) - throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - return rpc(client -> { - // TODO(calvin): Fix the exception handling in the master - ListStatusPOptions mergedOptions = FileSystemOptionsUtils.listStatusDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - return client.listStatus(path, mergedOptions); - }); - } - - @Override - public void iterateStatus(AlluxioURI path, final ListStatusPOptions options, - Consumer action) - throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - rpc(client -> { - // TODO(calvin): Fix the exception handling in the master - ListStatusPOptions mergedOptions = FileSystemOptionsUtils.listStatusDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - client.iterateStatus(path, mergedOptions, action); - return null; - }); - } - - @Override - public ListStatusPartialResult listStatusPartial( - AlluxioURI path, final ListStatusPartialPOptions options) - throws AlluxioException, IOException { - checkUri(path); - return rpc(client -> { - ListStatusPartialPOptions mergedOptions = FileSystemOptionsUtils.listStatusPartialDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - return client.listStatusPartial(path, mergedOptions); - }); - } - - @Override - public void loadMetadata(AlluxioURI path, final ListStatusPOptions options) - throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - rpc(client -> { - ListStatusPOptions mergedOptions = FileSystemOptionsUtils.listStatusDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options) - .setLoadMetadataType(LoadMetadataPType.ALWAYS).setLoadMetadataOnly(true).build(); - client.listStatus(path, mergedOptions); - return null; - }); - } - - @Override - public void mount(AlluxioURI alluxioPath, AlluxioURI ufsPath, final MountPOptions options) - throws IOException, AlluxioException { - checkUri(alluxioPath); - rpc(client -> { - MountPOptions mergedOptions = FileSystemOptionsUtils.mountDefaults( - mFsContext.getPathConf(alluxioPath)).toBuilder().mergeFrom(options).build(); - // TODO(calvin): Make this fail on the master side - client.mount(alluxioPath, ufsPath, mergedOptions); - LOG.debug("Mount {} to {}", ufsPath, alluxioPath.getPath()); - return null; - }); - } - - @Override - public void updateMount(AlluxioURI alluxioPath, final MountPOptions options) - throws IOException, AlluxioException { - checkUri(alluxioPath); - rpc(client -> { - MountPOptions mergedOptions = FileSystemOptionsUtils.mountDefaults( - mFsContext.getPathConf(alluxioPath)).toBuilder().mergeFrom(options).build(); - client.updateMount(alluxioPath, mergedOptions); - LOG.debug("UpdateMount on {}", alluxioPath.getPath()); - return null; - }); - } - - @Override - public Map getMountTable(boolean checkUfs) - throws IOException, AlluxioException { - return rpc(client -> client.getMountTable(checkUfs)); - } - - @Override - public List getSyncPathList() throws IOException, AlluxioException { - return rpc(FileSystemMasterClient::getSyncPathList); - } - - @Override - public void persist(final AlluxioURI path, final ScheduleAsyncPersistencePOptions options) - throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - rpc(client -> { - ScheduleAsyncPersistencePOptions mergedOptions = - FileSystemOptionsUtils - .scheduleAsyncPersistDefaults(mFsContext.getPathConf(path)).toBuilder() - .mergeFrom(options).build(); - client.scheduleAsyncPersist(path, mergedOptions); - LOG.debug("Scheduled persist for {}, options: {}", path.getPath(), mergedOptions); - return null; - }); - } - - @Override - public FileInStream openFile(AlluxioURI path, OpenFilePOptions options) - throws FileDoesNotExistException, OpenDirectoryException, FileIncompleteException, - IOException, AlluxioException { - checkUri(path); - AlluxioConfiguration conf = mFsContext.getPathConf(path); - URIStatus status = getStatus(path, - FileSystemOptionsUtils.getStatusDefaults(conf).toBuilder() - .setAccessMode(Bits.READ) - .setUpdateTimestamps(options.getUpdateLastAccessTime()) - .build()); - return openFile(status, options); - } - - @Override - public FileInStream openFile(URIStatus status, OpenFilePOptions options) - throws FileDoesNotExistException, OpenDirectoryException, FileIncompleteException, - IOException, AlluxioException { - AlluxioURI path = new AlluxioURI(status.getPath()); - if (status.isFolder()) { - throw new OpenDirectoryException(path); - } - if (!status.isCompleted()) { - throw new FileIncompleteException(path); - } - AlluxioConfiguration conf = mFsContext.getPathConf(path); - OpenFilePOptions mergedOptions = FileSystemOptionsUtils.openFileDefaults(conf) - .toBuilder().mergeFrom(options).build(); - InStreamOptions inStreamOptions = new InStreamOptions(status, mergedOptions, conf, mFsContext); - return new AlluxioFileInStream(status, inStreamOptions, mFsContext); - } - - @Override - public void rename(AlluxioURI src, AlluxioURI dst, RenamePOptions options) - throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(src); - checkUri(dst); - rpc(client -> { - RenamePOptions mergedOptions = FileSystemOptionsUtils - .renameDefaults(mFsContext.getPathConf(dst)) - .toBuilder().mergeFrom(options).build(); - // TODO(calvin): Update this code on the master side. - client.rename(src, dst, mergedOptions); - LOG.debug("Renamed {} to {}, options: {}", src.getPath(), dst.getPath(), mergedOptions); - return null; - }); - } - - @Override - public AlluxioURI reverseResolve(AlluxioURI ufsUri) throws IOException, AlluxioException { - return rpc(client -> { - AlluxioURI path = client.reverseResolve(ufsUri); - LOG.debug("Reverse resolved {} to {}", ufsUri, path.getPath()); - return path; - }); - } - - @Override - public void setAcl(AlluxioURI path, SetAclAction action, List entries, - SetAclPOptions options) throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - rpc(client -> { - SetAclPOptions mergedOptions = FileSystemOptionsUtils.setAclDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - client.setAcl(path, action, entries, mergedOptions); - LOG.debug("Set ACL for {}, entries: {} options: {}", path.getPath(), entries, - mergedOptions); - return null; - }); - } - - @Override - public void setAttribute(AlluxioURI path, SetAttributePOptions options) - throws FileDoesNotExistException, IOException, AlluxioException { - checkUri(path); - SetAttributePOptions mergedOptions = - FileSystemOptionsUtils.setAttributeClientDefaults(mFsContext.getPathConf(path)) - .toBuilder().mergeFrom(options).build(); - rpc(client -> { - client.setAttribute(path, mergedOptions); - LOG.debug("Set attributes for {}, options: {}", path.getPath(), options); - return null; - }); - } - - /** - * Starts the active syncing process on an Alluxio path. - * - * @param path the path to sync - */ - @Override - public void startSync(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - rpc(client -> { - client.startSync(path); - LOG.debug("Start syncing for {}", path.getPath()); - return null; - }); - } - - /** - * Stops the active syncing process on an Alluxio path. - * @param path the path to stop syncing - */ - @Override - public void stopSync(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - rpc(client -> { - client.stopSync(path); - LOG.debug("Stop syncing for {}", path.getPath()); - return null; - }); - } - - @Override - public void unmount(AlluxioURI path, UnmountPOptions options) - throws IOException, AlluxioException { - checkUri(path); - rpc(client -> { - UnmountPOptions mergedOptions = FileSystemOptionsUtils.unmountDefaults( - mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).build(); - client.unmount(path); - LOG.debug("Unmounted {}, options: {}", path.getPath(), mergedOptions); - return null; - }); - } - - @Override - public void needsSync(AlluxioURI path) - throws IOException, AlluxioException { - checkUri(path); - rpc(client -> { - client.needsSync(path); - return null; - }); - } - - /** - * Checks an {@link AlluxioURI} for scheme and authority information. Warn the user and throw an - * exception if necessary. - */ - protected void checkUri(AlluxioURI uri) { - Preconditions.checkNotNull(uri, "uri"); - if (!mFsContext.getUriValidationEnabled()) { - return; - } - - if (uri.hasScheme()) { - String warnMsg = "The URI scheme \"{}\" is ignored and not required in URIs passed to" - + " the Alluxio Filesystem client."; - switch (uri.getScheme()) { - case Constants.SCHEME: - LOG.warn(warnMsg, Constants.SCHEME); - break; - default: - throw new IllegalArgumentException( - String.format("Scheme %s:// in AlluxioURI is invalid. Schemes in filesystem" - + " operations are ignored. \"alluxio://\" or no scheme at all is valid.", - uri.getScheme())); - } - } - - if (uri.hasAuthority()) { - LOG.warn("The URI authority (hostname and port) is ignored and not required in URIs passed " - + "to the Alluxio Filesystem client."); - - AlluxioConfiguration conf = mFsContext.getClusterConf(); - boolean skipAuthorityCheck = conf.isSet(PropertyKey.USER_SKIP_AUTHORITY_CHECK) - && conf.getBoolean(PropertyKey.USER_SKIP_AUTHORITY_CHECK); - if (!skipAuthorityCheck) { - /* Even if we choose to log the warning, check if the Configuration host matches what the - * user passes. If not, throw an exception letting the user know they don't match. - */ - Authority configured = - MasterInquireClient.Factory - .create(mFsContext.getClusterConf(), - mFsContext.getClientContext().getUserState()) - .getConnectDetails().toAuthority(); - if (!configured.equals(uri.getAuthority())) { - throw new IllegalArgumentException( - String.format("The URI authority %s does not match the configured value of %s.", - uri.getAuthority(), configured)); - } - } - } - } - - @FunctionalInterface - interface RpcCallable { - R call(T t) throws IOException, AlluxioException; - } - - /** - * Sends an RPC to filesystem master. - * - * A resource is internally acquired to block FileSystemContext reinitialization before sending - * the RPC. - * - * @param fn the RPC call - * @param the type of return value for the RPC - * @return the RPC result - */ - R rpc(RpcCallable fn) - throws IOException, AlluxioException { - try (ReinitBlockerResource r = mFsContext.blockReinit(); - CloseableResource client = - mFsContext.acquireMasterClientResource()) { - // Explicitly connect to trigger loading configuration from meta master. - client.get().connect(); - return fn.call(client.get()); - } catch (NotFoundException e) { - throw new FileDoesNotExistException(e.getMessage()); - } catch (AlreadyExistsException e) { - throw new FileAlreadyExistsException(e.getMessage()); - } catch (InvalidArgumentException e) { - throw new InvalidPathException(e.getMessage()); - } catch (FailedPreconditionException e) { - // A little sketchy, but this should be the only case that throws FailedPrecondition. - throw new DirectoryNotEmptyException(e.getMessage()); - } catch (UnavailableException e) { - throw e; - } catch (UnauthenticatedException e) { - throw e; - } catch (AlluxioStatusException e) { - throw e.toAlluxioException(); - } - } - - /** - * Same as {@link BaseFileSystem} rpc except does not release the - * client resource. The caller is responsible for releasing the client - * resource. - * @param fn the RPC call - * @param the type of return value for the RPC - * @return the RPC result - */ - R rpcKeepClientResource(RpcCallable, R> fn) - throws IOException, AlluxioException { - CloseableResource client = null; - try (ReinitBlockerResource r = mFsContext.blockReinit()) { - client = mFsContext.acquireMasterClientResource(); - // Explicitly connect to trigger loading configuration from meta master. - client.get().connect(); - return fn.call(client); - } catch (NotFoundException e) { - client.close(); - throw new FileDoesNotExistException(e.getMessage()); - } catch (AlreadyExistsException e) { - client.close(); - throw new FileAlreadyExistsException(e.getMessage()); - } catch (InvalidArgumentException e) { - client.close(); - throw new InvalidPathException(e.getMessage()); - } catch (FailedPreconditionException e) { - client.close(); - // A little sketchy, but this should be the only case that throws FailedPrecondition. - throw new DirectoryNotEmptyException(e.getMessage()); - } catch (UnavailableException e) { - client.close(); - throw e; - } catch (UnauthenticatedException e) { - client.close(); - throw e; - } catch (AlluxioStatusException e) { - client.close(); - throw e.toAlluxioException(); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/ConfigHashSync.java b/core/client/fs/src/main/java/alluxio/client/file/ConfigHashSync.java deleted file mode 100644 index 144be4e7f6f1..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/ConfigHashSync.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import alluxio.client.meta.RetryHandlingMetaMasterConfigClient; -import alluxio.exception.status.UnavailableException; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.master.MasterClientContext; -import alluxio.wire.ConfigHash; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Optional; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Heartbeat task for getting the latest configuration versions from meta master, if versions - * change, then re-initialize the filesystem context. - * - * If a heartbeat fails, the internal meta master client will disconnect and try to reconnect. - * The passed in client will not be closed by this class. - */ -@ThreadSafe -public final class ConfigHashSync implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(ConfigHashSync.class); - - private final FileSystemContext mContext; - private volatile RetryHandlingMetaMasterConfigClient mClient; - - private volatile IOException mException; - - /** - * Constructs a new {@link ConfigHashSync}. - * - * @param context the filesystem context - */ - public ConfigHashSync(FileSystemContext context) { - mContext = context; - mClient = new RetryHandlingMetaMasterConfigClient(mContext.getMasterClientContext()); - } - - /** - * Resets the internal meta master client based on the new configuration. - * - * @param context the context containing the new configuration - */ - public void resetMetaMasterConfigClient(MasterClientContext context) { - mClient.close(); - mClient = new RetryHandlingMetaMasterConfigClient(context); - } - - /** - * @return empty if there is no exception during reinitialization, otherwise, return the exception - */ - public Optional getException() { - if (mException == null) { - return Optional.empty(); - } - return Optional.of(mException); - } - - @Override - public synchronized void heartbeat() { - if (!mContext.getClientContext().getClusterConf().clusterDefaultsLoaded()) { - // Wait until the initial cluster defaults are loaded. - return; - } - ConfigHash hash; - try { - hash = mClient.getConfigHash(); - } catch (IOException e) { - LOG.error("Failed to heartbeat to meta master to get configuration hash:", e); - // Disconnect to reconnect in the next heartbeat. - mClient.disconnect(); - return; - } - boolean isClusterConfUpdated = !hash.getClusterConfigHash().equals( - mContext.getClientContext().getClusterConfHash()); - boolean isPathConfUpdated = !hash.getPathConfigHash().equals( - mContext.getClientContext().getPathConfHash()); - if (isClusterConfUpdated || isPathConfUpdated) { - try { - mContext.reinit(isClusterConfUpdated, isPathConfUpdated); - mException = null; - } catch (UnavailableException e) { - LOG.error("Failed to reinitialize FileSystemContext:", e); - // Meta master might be temporarily unavailable, retry in next heartbeat. - } catch (IOException e) { - LOG.error("Failed to close FileSystemContext, interrupting the heartbeat thread", e); - mException = e; - // If the heartbeat keeps running, the context might be reinitialized successfully in the - // next heartbeat, then the resources that are not closed in the old context are leaked. - Thread.currentThread().interrupt(); - } - } - } - - @Override - public void close() { - mClient.close(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/FileSystem.java b/core/client/fs/src/main/java/alluxio/client/file/FileSystem.java deleted file mode 100644 index cbf6a9534c01..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/FileSystem.java +++ /dev/null @@ -1,740 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import alluxio.AlluxioURI; -import alluxio.ClientContext; -import alluxio.annotation.PublicApi; -import alluxio.client.file.cache.CacheManager; -import alluxio.client.file.cache.LocalCacheFileSystem; -import alluxio.client.file.options.FileSystemOptions; -import alluxio.client.file.ufs.UfsBaseFileSystem; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.conf.Source; -import alluxio.exception.AlluxioException; -import alluxio.exception.DirectoryNotEmptyException; -import alluxio.exception.FileAlreadyExistsException; -import alluxio.exception.FileDoesNotExistException; -import alluxio.exception.FileIncompleteException; -import alluxio.exception.InvalidPathException; -import alluxio.exception.OpenDirectoryException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.grpc.CheckAccessPOptions; -import alluxio.grpc.CreateDirectoryPOptions; -import alluxio.grpc.CreateFilePOptions; -import alluxio.grpc.DeletePOptions; -import alluxio.grpc.ExistsPOptions; -import alluxio.grpc.FreePOptions; -import alluxio.grpc.GetStatusPOptions; -import alluxio.grpc.ListStatusPOptions; -import alluxio.grpc.ListStatusPartialPOptions; -import alluxio.grpc.LoadMetadataPOptions; -import alluxio.grpc.LoadMetadataPType; -import alluxio.grpc.MountPOptions; -import alluxio.grpc.OpenFilePOptions; -import alluxio.grpc.RenamePOptions; -import alluxio.grpc.ScheduleAsyncPersistencePOptions; -import alluxio.grpc.SetAclAction; -import alluxio.grpc.SetAclPOptions; -import alluxio.grpc.SetAttributePOptions; -import alluxio.grpc.UnmountPOptions; -import alluxio.security.authorization.AclEntry; -import alluxio.security.user.UserState; -import alluxio.util.CommonUtils; -import alluxio.wire.BlockLocationInfo; -import alluxio.wire.MountPointInfo; -import alluxio.wire.SyncPointInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; -import javax.security.auth.Subject; - -/** - * Basic file system interface supporting metadata operations and data operations. Developers - * should not implement this class but extend the default implementation provided by {@link - * BaseFileSystem} instead. This ensures any new methods added to the interface will be provided - * by the default implementation. - */ -@PublicApi -public interface FileSystem extends Closeable { - - /** - * Factory for {@link FileSystem}. Calling any of the {@link Factory#get()} methods in this class - * will attempt to return a cached instance of an Alluxio {@link FileSystem}. Using any of the - * {@link Factory#create} methods will always guarantee returning a new FileSystem. - */ - class Factory { - private static final Logger LOG = LoggerFactory.getLogger(Factory.class); - private static final AtomicBoolean CONF_LOGGED = new AtomicBoolean(false); - - protected static final FileSystemCache FILESYSTEM_CACHE = new FileSystemCache(); - - private Factory() {} // prevent instantiation - - /** - * @return a FileSystem from the cache, creating a new one if it doesn't yet exist - */ - public static FileSystem get() { - return get(new Subject()); // Use empty subject - } - - /** - * Get a FileSystem from the cache with a given subject. - * - * @param subject The subject to use for security-related client operations - * @return a FileSystem from the cache, creating a new one if it doesn't yet exist - */ - public static FileSystem get(Subject subject) { - return get(subject, Configuration.global()); - } - - /** - * Get a FileSystem from the cache with a given subject. - * @param subject The subject to use for security-related client operations - * @param conf the Alluxio configuration - * @return a FileSystem from the cache, creating a new one if it doesn't yet exist - */ - public static FileSystem get(Subject subject, AlluxioConfiguration conf) { - Preconditions.checkNotNull(subject, "subject"); - // TODO(gpang): should this key use the UserState instead of subject? - FileSystemCache.Key key = - new FileSystemCache.Key(UserState.Factory.create(conf, subject).getSubject(), conf); - return FILESYSTEM_CACHE.get(key); - } - - /** - * @return a new FileSystem instance - */ - public static FileSystem create() { - return create(FileSystemContext.create()); - } - - /** - * @param alluxioConf the configuration to utilize with the FileSystem - * @return a new FileSystem instance - */ - public static FileSystem create(AlluxioConfiguration alluxioConf) { - return create(FileSystemContext.create(alluxioConf)); - } - - /** - * @param ctx the context with the subject and configuration to utilize with the FileSystem - * @return a new FileSystem instance - */ - public static FileSystem create(ClientContext ctx) { - return create(FileSystemContext.create(ctx)); - } - - /** - * @param context the FileSystemContext to use with the FileSystem - * @return a new FileSystem instance - */ - public static FileSystem create(FileSystemContext context) { - return create(context, FileSystemOptions.create(context.getClusterConf())); - } - - /** - * @param context the FileSystemContext to use with the FileSystem - * @return a new FileSystem instance - */ - public static FileSystem create(FileSystemContext context, FileSystemOptions options) { - AlluxioConfiguration conf = context.getClusterConf(); - checkSortConf(conf); - FileSystem fs = options.getUfsFileSystemOptions().isPresent() - ? new UfsBaseFileSystem(context, options.getUfsFileSystemOptions().get()) - : new BaseFileSystem(context); - if (options.isMetadataCacheEnabled()) { - fs = new MetadataCachingFileSystem(fs, context); - } - if (options.isDataCacheEnabled() - && CommonUtils.PROCESS_TYPE.get() == CommonUtils.ProcessType.CLIENT) { - try { - CacheManager cacheManager = CacheManager.Factory.get(conf); - return new LocalCacheFileSystem(cacheManager, fs, conf); - } catch (IOException e) { - LOG.error("Fallback without client caching: ", e); - } - } - return fs; - } - - static void checkSortConf(AlluxioConfiguration conf) { - if (LOG.isDebugEnabled() && !CONF_LOGGED.getAndSet(true)) { - // Sort properties by name to keep output ordered. - List keys = new ArrayList<>(conf.keySet()); - keys.sort(Comparator.comparing(PropertyKey::getName)); - for (PropertyKey key : keys) { - Object value = conf.getOrDefault(key, null); - Source source = conf.getSource(key); - LOG.debug("{}={} ({})", key.getName(), value, source); - } - } - } - } - - /** - * If there are operations currently running and close is called concurrently the behavior is - * undefined. After closing a FileSystem, any operations that are performed result in undefined - * behavior. - * - * @return whether or not this FileSystem has been closed - */ - boolean isClosed(); - - /** - * Checks access to a path. - * - * @param path the path of the directory to create in Alluxio space - * @param options options to associate with this operation - * @throws InvalidPathException if the path is invalid - * @throws alluxio.exception.AccessControlException if the access is denied - */ - void checkAccess(AlluxioURI path, CheckAccessPOptions options) - throws InvalidPathException, IOException, AlluxioException; - - /** - * Convenience method for {@link #createDirectory(AlluxioURI, CreateDirectoryPOptions)} with - * default options. - * - * @param path the path of the directory to create in Alluxio space - * @throws FileAlreadyExistsException if there is already a file or directory at the given path - * @throws InvalidPathException if the path is invalid - */ - default void createDirectory(AlluxioURI path) - throws FileAlreadyExistsException, InvalidPathException, IOException, AlluxioException { - createDirectory(path, CreateDirectoryPOptions.getDefaultInstance()); - } - - /** - * Creates a directory. - * - * @param path the path of the directory to create in Alluxio space - * @param options options to associate with this operation - * @throws FileAlreadyExistsException if there is already a file or directory at the given path - * @throws InvalidPathException if the path is invalid - */ - void createDirectory(AlluxioURI path, CreateDirectoryPOptions options) - throws FileAlreadyExistsException, InvalidPathException, IOException, AlluxioException; - - /** - * Convenience method for {@link #createFile(AlluxioURI, CreateFilePOptions)} with default - * options. - * - * @param path the path of the file to create in Alluxio space - * @return a {@link FileOutStream} which will write data to the newly created file - * @throws FileAlreadyExistsException if there is already a file at the given path - * @throws InvalidPathException if the path is invalid - */ - default FileOutStream createFile(AlluxioURI path) - throws FileAlreadyExistsException, InvalidPathException, IOException, AlluxioException { - return createFile(path, CreateFilePOptions.getDefaultInstance()); - } - - /** - * Creates a file. - * - * @param path the path of the file to create in Alluxio space - * @param options options to associate with this operation - * @return a {@link FileOutStream} which will write data to the newly created file - * @throws FileAlreadyExistsException if there is already a file at the given path - * @throws InvalidPathException if the path is invalid - */ - FileOutStream createFile(AlluxioURI path, CreateFilePOptions options) - throws FileAlreadyExistsException, InvalidPathException, IOException, AlluxioException; - - /** - * Convenience method for {@link #delete(AlluxioURI, DeletePOptions)} with default options. - * - * @param path the path to delete in Alluxio space - * @throws FileDoesNotExistException if the given path does not exist - * @throws DirectoryNotEmptyException if recursive is false and the path is a nonempty directory - */ - default void delete(AlluxioURI path) - throws DirectoryNotEmptyException, FileDoesNotExistException, IOException, AlluxioException { - delete(path, DeletePOptions.getDefaultInstance()); - } - - /** - * Deletes a file or a directory. - * - * @param path the path to delete in Alluxio space - * @param options options to associate with this operation - * @throws FileDoesNotExistException if the given path does not exist - * @throws DirectoryNotEmptyException if recursive is false and the path is a nonempty directory - */ - void delete(AlluxioURI path, DeletePOptions options) - throws DirectoryNotEmptyException, FileDoesNotExistException, IOException, AlluxioException; - - /** - * Convenience method for {@link #exists(AlluxioURI, ExistsPOptions)} with default options. - * - * @param path the path in question - * @return true if the path exists, false otherwise - * @throws InvalidPathException if the path is invalid - */ - default boolean exists(AlluxioURI path) - throws InvalidPathException, IOException, AlluxioException { - return exists(path, ExistsPOptions.getDefaultInstance()); - } - - /** - * Checks whether a path exists in Alluxio space. - * - * @param path the path in question - * @param options options to associate with this operation - * @return true if the path exists, false otherwise - * @throws InvalidPathException if the path is invalid - */ - boolean exists(AlluxioURI path, ExistsPOptions options) - throws InvalidPathException, IOException, AlluxioException; - - /** - * Convenience method for {@link #free(AlluxioURI, FreePOptions)} with default options. - * - * @param path the path to free in Alluxio space - * @throws FileDoesNotExistException if the given path does not exist - */ - default void free(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - free(path, FreePOptions.getDefaultInstance()); - } - - /** - * Evicts any data under the given path from Alluxio space, but does not delete the data from the - * UFS. The metadata will still be present in Alluxio space after this operation. - * - * @param path the path to free in Alluxio space - * @param options options to associate with this operation - * @throws FileDoesNotExistException if the given path does not exist - */ - void free(AlluxioURI path, FreePOptions options) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Builds a list of {@link BlockLocationInfo} for the given file. Each list item contains a list - * of {@link WorkerNetAddress} which allows a user to determine the physical location of a block - * of the given file stored within Alluxio. In the case where data is stored in a UFS, but not in - * Alluxio this function will only include a {@link WorkerNetAddress} if the block stored in the - * UFS is co-located with an Alluxio worker. - * However if there are no co-located Alluxio workers for the block, then the behavior is - * controlled by the {@link PropertyKey#USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED} . If - * this property is set to {@code true} then every Alluxio worker will be returned. - * Blocks which are stored in the UFS and are *not* co-located with any Alluxio worker will return - * an empty list. If the file block is within Alluxio *and* the UFS then this will only return - * Alluxio workers which currently store the block. - * - * @param path the path to get block info for - * @return a list of blocks with the workers whose hosts have the blocks. The blocks may not - * necessarily be stored in Alluxio. The blocks are returned in the order of their - * sequences in file. - * @throws FileDoesNotExistException if the given path does not exist - */ - default List getBlockLocations(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - return getBlockLocations(getStatus(path)); - } - - /** - * Builds a list of {@link BlockLocationInfo} for the given file. - * - * @param status the path status - * @return a list of blocks with the workers whose hosts have the blocks. The blocks may not - * necessarily be stored in Alluxio. The blocks are returned in the order of their - * sequences in file. - */ - List getBlockLocations(URIStatus status) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * @return the configuration which the FileSystem is using to connect to Alluxio - */ - AlluxioConfiguration getConf(); - - /** - * Convenience method for {@link #getStatus(AlluxioURI, GetStatusPOptions)} with default options. - * - * @param path the path to obtain information about - * @return the {@link URIStatus} of the file - * @throws FileDoesNotExistException if the path does not exist - */ - default URIStatus getStatus(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - return getStatus(path, GetStatusPOptions.getDefaultInstance()); - } - - /** - * Gets the {@link URIStatus} object that represents the metadata of an Alluxio path. - * - * @param path the path to obtain information about - * @param options options to associate with this operation - * @return the {@link URIStatus} of the file - * @throws FileDoesNotExistException if the path does not exist - */ - URIStatus getStatus(AlluxioURI path, GetStatusPOptions options) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Performs a specific action on each {@code URIStatus} in the result of {@link #listStatus}. - * This method is preferred when iterating over directories with a large number of files or - * sub-directories inside. The caller can proceed with partial result without waiting for all - * result returned. - * - * @param path the path to list information about - * @param action action to apply on each {@code URIStatus} - * @throws FileDoesNotExistException if the given path does not exist - */ - default void iterateStatus(AlluxioURI path, Consumer action) - throws FileDoesNotExistException, IOException, AlluxioException { - iterateStatus(path, ListStatusPOptions.getDefaultInstance(), action); - } - - /** - * Performs a specific action on each {@code URIStatus} in the result of {@link #listStatus}. - * This method is preferred when iterating over directories with a large number of files or - * sub-directories inside. The caller can proceed with partial result without waiting for all - * result returned. - * - * @param path the path to list information about - * @param options options to associate with this operation - * @param action action to apply on each {@code URIStatus} - * @throws FileDoesNotExistException if the given path does not exist - */ - void iterateStatus(AlluxioURI path, ListStatusPOptions options, - Consumer action) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Convenience method for {@link #listStatus(AlluxioURI, ListStatusPOptions)} with default - * options. - * - * @param path the path to list information about - * @return a list of {@link URIStatus}s containing information about the files and directories - * which are children of the given path - * @throws FileDoesNotExistException if the given path does not exist - */ - default List listStatus(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - return listStatus(path, ListStatusPOptions.getDefaultInstance()); - } - - /** - * If the path is a directory, returns the {@link URIStatus} of all the direct entries in it. - * Otherwise returns a list with a single {@link URIStatus} element for the file. - * - * @param path the path to list information about - * @param options options to associate with this operation - * @return a list of {@link URIStatus}s containing information about the files and directories - * which are children of the given path - * @throws FileDoesNotExistException if the given path does not exist - */ - List listStatus(AlluxioURI path, ListStatusPOptions options) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Same as {@link FileSystem#listStatus(AlluxioURI, ListStatusPOptions)} except may - * only return a subset of the results as determined by the options parameter. - * - * @param path the path to list information about - * @param options options to associate with this operation - * @return a list of {@link URIStatus}s containing information about the files and directories - * which are children of the given path - * @throws FileDoesNotExistException if the given path does not exist - */ - ListStatusPartialResult listStatusPartial( - AlluxioURI path, ListStatusPartialPOptions options) - throws AlluxioException, IOException; - - /** - * Convenience method for {@link #loadMetadata(AlluxioURI, ListStatusPOptions)} with default - * options. - * - * @param path the path for which to load metadata from UFS - * @throws FileDoesNotExistException if the given path does not exist - */ - default void loadMetadata(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - ListStatusPOptions options = ListStatusPOptions.newBuilder() - .setLoadMetadataType(LoadMetadataPType.ALWAYS) - .setRecursive(LoadMetadataPOptions.getDefaultInstance().getRecursive()) - .setLoadMetadataOnly(true).build(); - loadMetadata(path, options); - } - - /** - * Loads metadata about a path in the UFS to Alluxio. No data will be transferred. - * - * @param path the path for which to load metadata from UFS - * @param options options to associate with this operation - * @throws FileDoesNotExistException if the given path does not exist - */ - void loadMetadata(AlluxioURI path, ListStatusPOptions options) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Convenience method for {@link #mount(AlluxioURI, AlluxioURI, MountPOptions)} with default - * options. - * - * @param alluxioPath an Alluxio path to mount the data to - * @param ufsPath a UFS path to mount the data from - */ - default void mount(AlluxioURI alluxioPath, AlluxioURI ufsPath) - throws IOException, AlluxioException { - mount(alluxioPath, ufsPath, MountPOptions.getDefaultInstance()); - } - - /** - * Mounts a UFS subtree to the given Alluxio path. The Alluxio path is expected not to exist as - * the method creates it. If the path already exists, a {@link AlluxioException} will be thrown. - * This method does not transfer any data or metadata from the UFS. It simply establishes the - * connection between the given Alluxio path and UFS path. - * - * @param alluxioPath an Alluxio path to mount the data to - * @param ufsPath a UFS path to mount the data from - * @param options options to associate with this operation - */ - void mount(AlluxioURI alluxioPath, AlluxioURI ufsPath, MountPOptions options) - throws IOException, AlluxioException; - - /** - * Updates the options for an existing mount point. - * - * @param alluxioPath the Alluxio path of the mount point - * @param options options for this mount point - */ - void updateMount(AlluxioURI alluxioPath, MountPOptions options) - throws IOException, AlluxioException; - - /** - * Lists all mount points and their corresponding under storage addresses. - * This is the same as calling {@link #getMountTable(boolean)} with true argument. - * @return a map from String to {@link MountPointInfo} - */ - default Map getMountTable() throws IOException, AlluxioException { - return getMountTable(true); - } - - /** - * Lists all mount points and their corresponding under storage addresses. - * @param checkUfs whether to get UFS usage info - * @return a map from String to {@link MountPointInfo} - */ - Map getMountTable(boolean checkUfs) - throws IOException, AlluxioException; - - /** - * Lists all the actively synced paths. - * - * @return a list of actively synced paths - */ - List getSyncPathList() throws IOException, AlluxioException; - - /** - * Convenience method for {@link #openFile(AlluxioURI, OpenFilePOptions)} with default options. - * - * @param path the file to read from - * @return a {@link FileInStream} for the given path - * @throws FileDoesNotExistException when path does not exist - * @throws OpenDirectoryException when path is a directory - * @throws FileIncompleteException when path is a file and is not completed yet - */ - default FileInStream openFile(AlluxioURI path) - throws FileDoesNotExistException, OpenDirectoryException, FileIncompleteException, - IOException, AlluxioException { - return openFile(path, OpenFilePOptions.getDefaultInstance()); - } - - /** - * Opens a file for reading. - * - * @param path the file to read from - * @param options options to associate with this operation - * @return a {@link FileInStream} for the given path - * @throws FileDoesNotExistException when path does not exist - * @throws OpenDirectoryException when path is a directory - * @throws FileIncompleteException when path is a file and is not completed yet - */ - FileInStream openFile(AlluxioURI path, OpenFilePOptions options) - throws FileDoesNotExistException, OpenDirectoryException, FileIncompleteException, - IOException, AlluxioException; - - /** - * Opens a file for reading. - * - * @param status status of the file to read from - * @param options options to associate with this operation - * @return a {@link FileInStream} for the given path - * @throws FileDoesNotExistException when path does not exist - * @throws OpenDirectoryException when path is a directory - * @throws FileIncompleteException when path is a file and is not completed yet - */ - FileInStream openFile(URIStatus status, OpenFilePOptions options) - throws FileDoesNotExistException, OpenDirectoryException, FileIncompleteException, - IOException, AlluxioException; - - /** - * Convenience method for {@link #persist(AlluxioURI, ScheduleAsyncPersistencePOptions)} which - * uses the default {@link ScheduleAsyncPersistencePOptions}. - * - * @param path the uri of the file to persist - */ - default void persist(final AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - persist(path, ScheduleAsyncPersistencePOptions.getDefaultInstance()); - } - - /** - * Schedules the given path to be asynchronously persisted to the under file system. - * - * To persist synchronously please see - * {@link FileSystemUtils#persistAndWait(FileSystem, AlluxioURI, long)}. - * - * @param path the uri of the file to persist - * @param options the options to use when submitting persist the path - */ - void persist(AlluxioURI path, ScheduleAsyncPersistencePOptions options) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Convenience method for {@link #rename(AlluxioURI, AlluxioURI, RenamePOptions)} with default - * options. - * - * @param src the path of the source, this must already exist - * @param dst the path of the destination, this path should not exist - * @throws FileDoesNotExistException if the given file does not exist - */ - default void rename(AlluxioURI src, AlluxioURI dst) - throws FileDoesNotExistException, IOException, AlluxioException { - rename(src, dst, RenamePOptions.getDefaultInstance()); - } - - /** - * Renames an existing Alluxio path to another Alluxio path in Alluxio. This operation will be - * propagated in the underlying storage if the path is persisted. - * - * @param src the path of the source, this must already exist - * @param dst the path of the destination, this path should not exist - * @param options options to associate with this operation - * @throws FileDoesNotExistException if the given file does not exist - */ - void rename(AlluxioURI src, AlluxioURI dst, RenamePOptions options) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Reverse resolve a ufs uri. - * - * @param ufsUri the ufs uri - * @return the alluxio path for the ufsUri - * @throws AlluxioStatusException - */ - AlluxioURI reverseResolve(AlluxioURI ufsUri) throws IOException, AlluxioException; - - /** - * Convenience method for {@link #setAcl(AlluxioURI, SetAclAction, List, SetAclPOptions)} with - * default options. - * - * @param path the path to set the ACL for - * @param action the set action to perform - * @param entries the ACL entries - * @throws FileDoesNotExistException if the given file does not exist - */ - default void setAcl(AlluxioURI path, SetAclAction action, List entries) - throws FileDoesNotExistException, IOException, AlluxioException { - setAcl(path, action, entries, SetAclPOptions.getDefaultInstance()); - } - - /** - * Sets the ACL for a path. - * - * @param path the path to set the ACL for - * @param action the set action to perform - * @param entries the ACL entries - * @param options options to associate with this operation - * @throws FileDoesNotExistException if the given file does not exist - */ - void setAcl(AlluxioURI path, SetAclAction action, List entries, SetAclPOptions options) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Starts the active syncing process on an Alluxio path. - * @param path the path to sync - */ - void startSync(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Stops the active syncing process on an Alluxio path. - * @param path the path to stop syncing - */ - void stopSync(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Convenience method for {@link #setAttribute(AlluxioURI, SetAttributePOptions)} with default - * options. - * - * @param path the path to set attributes for - * @throws FileDoesNotExistException if the given file does not exist - */ - default void setAttribute(AlluxioURI path) - throws FileDoesNotExistException, IOException, AlluxioException { - setAttribute(path, SetAttributePOptions.getDefaultInstance()); - } - - /** - * Sets any number of a path's attributes, such as TTL and pin status. - * - * @param path the path to set attributes for - * @param options options to associate with this operation - * @throws FileDoesNotExistException if the given file does not exist - */ - void setAttribute(AlluxioURI path, SetAttributePOptions options) - throws FileDoesNotExistException, IOException, AlluxioException; - - /** - * Convenience method for {@link #unmount(AlluxioURI, UnmountPOptions)} with default options. - * - * @param path an Alluxio path, this must be a mount point - */ - default void unmount(AlluxioURI path) throws IOException, AlluxioException { - unmount(path, UnmountPOptions.getDefaultInstance()); - } - - /** - * Unmounts a UFS subtree identified by the given Alluxio path. The Alluxio path match a - * previously mounted path. The contents of the subtree rooted at this path are removed from - * Alluxio but the corresponding UFS subtree is left untouched. - * - * @param path an Alluxio path, this must be a mount point - * @param options options to associate with this operation - */ - void unmount(AlluxioURI path, UnmountPOptions options) throws IOException, AlluxioException; - - /** - * Marks the path in Alluxio as needing sync with the UFS. The next time the - * path or any of its children are accessed they will be synced with the UFS. - * @param path the path needing synchronization - */ - void needsSync(AlluxioURI path) throws IOException, AlluxioException; -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/FileSystemContext.java b/core/client/fs/src/main/java/alluxio/client/file/FileSystemContext.java deleted file mode 100644 index d7dd61b218c9..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/FileSystemContext.java +++ /dev/null @@ -1,812 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import static java.util.stream.Collectors.toList; - -import alluxio.AlluxioURI; -import alluxio.ClientContext; -import alluxio.client.block.BlockMasterClient; -import alluxio.client.block.BlockMasterClientPool; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.BlockLocationPolicy; -import alluxio.client.block.stream.BlockWorkerClient; -import alluxio.client.block.stream.BlockWorkerClientPool; -import alluxio.client.file.FileSystemContextReinitializer.ReinitBlockerResource; -import alluxio.client.metrics.MetricsHeartbeatContext; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.conf.ReconfigurableRegistry; -import alluxio.conf.path.SpecificPathConfiguration; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.GrpcServerAddress; -import alluxio.master.MasterClientContext; -import alluxio.master.MasterInquireClient; -import alluxio.metrics.MetricsSystem; -import alluxio.refresh.RefreshPolicy; -import alluxio.refresh.TimeoutRefresh; -import alluxio.resource.CloseableResource; -import alluxio.resource.DynamicResourcePool; -import alluxio.security.authentication.AuthenticationUtils; -import alluxio.security.user.UserState; -import alluxio.util.IdUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.WorkerInfo; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.BlockWorker; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.ThreadSafe; -import javax.security.auth.Subject; - -/** - * An object which houses resources and information for performing {@link FileSystem} operations. - * Typically, a single JVM should only need one instance of a {@link FileSystem} to connect to - * Alluxio. The reference to that client object should be shared among threads. - * - * A second {@link FileSystemContext} object should only be created when a user needs to connect to - * Alluxio with a different {@link Subject} and/or {@link AlluxioConfiguration}. - * {@link FileSystemContext} instances should be created sparingly because each instance creates - * its own thread pools of {@link FileSystemMasterClient} and {@link BlockMasterClient} which can - * lead to inefficient use of client machine resources. - * - * A {@link FileSystemContext} should be closed once the user is done performing operations with - * Alluxio and no more connections need to be made. Once a {@link FileSystemContext} is closed it - * is preferred that the user of the class create a new instance with - * {@link FileSystemContext#create} to create a new context, rather than reinitializing using the - * {@link FileSystemContext#init} method. - * - * NOTE: Each context maintains a pool of file system master clients that is already thread-safe. - * Synchronizing {@link FileSystemContext} methods could lead to deadlock: thread A attempts to - * acquire a client when there are no clients left in the pool and blocks holding a lock on the - * {@link FileSystemContext}, when thread B attempts to release a client it owns it is unable to do - * so, because thread A holds the lock on {@link FileSystemContext}. - */ -@ThreadSafe -public class FileSystemContext implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(FileSystemContext.class); - - /** - * Unique ID for each FileSystemContext. - * One example usage is to uniquely identify the heartbeat thread for ConfigHashSync. - */ - private final String mId; - - /** - * The block worker for worker internal clients to call worker operation directly - * without going through the external RPC frameworks. - */ - private final BlockWorker mBlockWorker; - - /** - * Marks whether the context has been closed, closing the context means releasing all resources - * in the context like clients and thread pools. - */ - private final AtomicBoolean mClosed = new AtomicBoolean(false); - - @GuardedBy("this") - private boolean mMetricsEnabled; - - // - // Master related resources. - // - /** - * The master client context holding the inquire client. - */ - private volatile MasterClientContext mMasterClientContext; - /** - * Master client pools. - */ - private volatile FileSystemMasterClientPool mFileSystemMasterClientPool; - private volatile BlockMasterClientPool mBlockMasterClientPool; - - // - // Worker related resources. - // - /** - * The data server channel pools. This pool will only grow and keys are not removed. - */ - private volatile ConcurrentHashMap - mBlockWorkerClientPoolMap; - - /** - * Indicates whether the {@link #mLocalWorker} field has been lazily initialized yet. - */ - @GuardedBy("this") - private boolean mLocalWorkerInitialized; - /** - * The address of any Alluxio worker running on the local machine. This is initialized lazily. - */ - @GuardedBy("this") - private WorkerNetAddress mLocalWorker; - - /** - * Reinitializer contains a daemon heartbeat thread to reinitialize this context when - * configuration hashes change. - */ - private volatile FileSystemContextReinitializer mReinitializer; - - /** Whether to do URI scheme validation for file systems using this context. */ - private boolean mUriValidationEnabled = true; - - /** Cached map for workers. */ - @GuardedBy("mWorkerInfoList") - private final AtomicReference> mWorkerInfoList = new AtomicReference<>(); - - /** The policy to refresh workers list. */ - @GuardedBy("mWorkerInfoList") - private final RefreshPolicy mWorkerRefreshPolicy; - - private final List mMasterAddresses; - - private final Map mBlockLocationPolicyMap; - - /** - * Creates a {@link FileSystemContext} with an empty subject - * , a null local block worker, and the given master addresses. - * - * @param conf Alluxio configuration - * @param masterAddresses the master addresses to use, this addresses will be - * used across reinitialization - * @return an instance of file system context with no subject associated - */ - public static FileSystemContext create( - AlluxioConfiguration conf, List masterAddresses) { - return create(ClientContext.create(conf), null, masterAddresses); - } - - /** - * Creates a {@link FileSystemContext} with an empty subject, default config - * and a null local block worker. - * - * @return an instance of file system context with no subject associated - */ - public static FileSystemContext create() { - return create(ClientContext.create()); - } - - /** - * Creates a {@link FileSystemContext} with an empty subject - * and a null local block worker. - * - * @param conf Alluxio configuration - * @return an instance of file system context with no subject associated - */ - public static FileSystemContext create(AlluxioConfiguration conf) { - Preconditions.checkNotNull(conf); - return create(ClientContext.create(conf)); - } - - /** - * @param subject the parent subject - * @param conf Alluxio configuration - * @return a context - */ - public static FileSystemContext create(Subject subject, - AlluxioConfiguration conf) { - return create(ClientContext.create(subject, conf)); - } - - /** - * @param clientContext the {@link alluxio.ClientContext} containing the subject and configuration - * @return the {@link alluxio.client.file.FileSystemContext} - */ - public static FileSystemContext create(ClientContext clientContext) { - return create(clientContext, null, null); - } - - /** - * @param ctx client context - * @param blockWorker block worker - * @return a context - */ - public static FileSystemContext create(ClientContext ctx, - @Nullable BlockWorker blockWorker) { - return create(ctx, blockWorker, null); - } - - /** - * @param ctx client context - * @param blockWorker block worker - * @param masterAddresses is non-null then the addresses used to connect to the master - * @return a context - */ - public static FileSystemContext create(ClientContext ctx, - @Nullable BlockWorker blockWorker, @Nullable List masterAddresses) { - FileSystemContext context = new FileSystemContext(ctx.getClusterConf(), blockWorker, - masterAddresses); - MasterInquireClient inquireClient; - if (masterAddresses != null) { - inquireClient = MasterInquireClient.Factory.createForAddresses(masterAddresses, - ctx.getClusterConf(), ctx.getUserState()); - } else { - inquireClient = MasterInquireClient.Factory.create( - ctx.getClusterConf(), ctx.getUserState()); - } - context.init(ctx, inquireClient); - return context; - } - - /** - * This method is provided for testing, use the {@link FileSystemContext#create} methods. The - * returned context object will not be cached automatically. - * - * @param subject the parent subject - * @param masterInquireClient the client to use for determining the master; note that if the - * context is reset, this client will be replaced with a new masterInquireClient based on - * the original configuration. - * @param alluxioConf Alluxio configuration - * @return the context - */ - @VisibleForTesting - public static FileSystemContext create(Subject subject, MasterInquireClient masterInquireClient, - AlluxioConfiguration alluxioConf) { - FileSystemContext context = new FileSystemContext(alluxioConf, null, null); - ClientContext ctx = ClientContext.create(subject, alluxioConf); - context.init(ctx, masterInquireClient); - return context; - } - - /** - * Initializes FileSystemContext ID. - * - * @param conf Alluxio configuration - * @param blockWorker block worker - */ - private FileSystemContext(AlluxioConfiguration conf, @Nullable BlockWorker blockWorker, - @Nullable List masterAddresses) { - mId = IdUtils.createFileSystemContextId(); - mBlockWorker = blockWorker; - mMasterAddresses = masterAddresses; - mWorkerRefreshPolicy = - new TimeoutRefresh(conf.getMs(PropertyKey.USER_WORKER_LIST_REFRESH_INTERVAL)); - LOG.debug("Created context with id: {}, with local block worker: {}", - mId, mBlockWorker != null); - mBlockLocationPolicyMap = new ConcurrentHashMap(); - } - - /** - * Initializes the context. Only called in the factory methods. - * - * @param masterInquireClient the client to use for determining the master - */ - private synchronized void init(ClientContext clientContext, - MasterInquireClient masterInquireClient) { - initContext(clientContext, masterInquireClient); - mReinitializer = new FileSystemContextReinitializer(this); - } - - private synchronized void initContext(ClientContext ctx, - MasterInquireClient masterInquireClient) { - mClosed.set(false); - mMasterClientContext = MasterClientContext.newBuilder(ctx) - .setMasterInquireClient(masterInquireClient).build(); - mMetricsEnabled = getClusterConf().getBoolean(PropertyKey.USER_METRICS_COLLECTION_ENABLED); - if (mMetricsEnabled) { - MetricsSystem.startSinks(getClusterConf().getString(PropertyKey.METRICS_CONF_FILE)); - MetricsHeartbeatContext.addHeartbeat(getClientContext(), masterInquireClient); - } - mFileSystemMasterClientPool = new FileSystemMasterClientPool(mMasterClientContext); - mBlockMasterClientPool = new BlockMasterClientPool(mMasterClientContext); - mBlockWorkerClientPoolMap = new ConcurrentHashMap<>(); - mUriValidationEnabled = ctx.getUriValidationEnabled(); - } - - /** - * Closes all the resources associated with the context. Make sure all the resources are released - * back to this context before calling this close. After closing the context, all the resources - * that acquired from this context might fail. Only call this when you are done with using - * the {@link FileSystem} associated with this {@link FileSystemContext}. - */ - @Override - public synchronized void close() throws IOException { - LOG.debug("Closing context with id: {}", mId); - mReinitializer.close(); - closeContext(); - LOG.debug("Closed context with id: {}", mId); - } - - private synchronized void closeContext() throws IOException { - if (!mClosed.get()) { - // Setting closed should be the first thing we do because if any of the close operations - // fail we'll only have a half-closed object and performing any more operations or closing - // again on a half-closed object can possibly result in more errors (i.e. NPE). Setting - // closed first is also recommended by the JDK that in implementations of #close() that - // developers should first mark their resources as closed prior to any exceptions being - // thrown. - mClosed.set(true); - LOG.debug("Closing fs master client pool with current size: {} for id: {}", - mFileSystemMasterClientPool.size(), mId); - mFileSystemMasterClientPool.close(); - mFileSystemMasterClientPool = null; - LOG.debug("Closing block master client pool with size: {} for id: {}", - mBlockMasterClientPool.size(), mId); - mBlockMasterClientPool.close(); - mBlockMasterClientPool = null; - for (BlockWorkerClientPool pool : mBlockWorkerClientPoolMap.values()) { - LOG.debug("Closing block worker client pool with size: {} for id: {}", pool.size(), mId); - pool.close(); - } - // Close worker group after block master clients in order to allow - // clean termination for open streams. - mBlockWorkerClientPoolMap.clear(); - mBlockWorkerClientPoolMap = null; - mLocalWorkerInitialized = false; - mLocalWorker = null; - - if (mMetricsEnabled) { - MetricsHeartbeatContext.removeHeartbeat(getClientContext()); - } - } else { - LOG.warn("Attempted to close FileSystemContext which has already been closed or not " - + "initialized."); - } - } - - /** - * Acquires the resource to block reinitialization. - * - * If reinitialization is happening, this method will block until reinitialization succeeds or - * fails, if it fails, a RuntimeException will be thrown explaining the - * reinitialization's failure and automatically closes the resource. - * - * RuntimeException is thrown because this method is called before requiring resources from the - * context, if reinitialization fails, the resources might be half closed, to prevent resource - * leaks, we thrown RuntimeException here to force callers to fail since there is no way to - * recover. - * - * @return the resource - */ - public ReinitBlockerResource blockReinit() { - try { - return mReinitializer.block(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Closes the context, updates configuration from meta master, then re-initializes the context. - * - * The reinitializer is not closed, which means the heartbeat thread inside it is not stopped. - * The reinitializer will be reset with the updated context if reinitialization succeeds, - * otherwise, the reinitializer is not reset. - * - * Blocks until there is no active RPCs. - * - * @param updateClusterConf whether cluster level configuration should be updated - * @param updatePathConf whether path level configuration should be updated - * @throws UnavailableException when failed to load configuration from master - * @throws IOException when failed to close the context - */ - public void reinit(boolean updateClusterConf, boolean updatePathConf) - throws UnavailableException, IOException { - try (Closeable r = mReinitializer.allow()) { - InetSocketAddress masterAddr; - try { - masterAddr = getMasterAddress(); - } catch (IOException e) { - throw new UnavailableException("Failed to get master address during reinitialization", e); - } - try { - getClientContext().loadConf(masterAddr, updateClusterConf, updatePathConf); - } catch (AlluxioStatusException e) { - // Failed to load configuration from meta master, maybe master is being restarted, - // or their is a temporary network problem, give up reinitialization. The heartbeat thread - // will try to reinitialize in the next heartbeat. - throw new UnavailableException(String.format("Failed to load configuration from " - + "meta master (%s) during reinitialization", masterAddr), e); - } - LOG.debug("Reinitializing FileSystemContext: update cluster conf: {}, update path conf:" - + " {}", updateClusterConf, updateClusterConf); - closeContext(); - ReconfigurableRegistry.update(); - initContext(getClientContext(), mMasterAddresses != null - ? MasterInquireClient.Factory.createForAddresses(mMasterAddresses, - getClusterConf(), getClientContext().getUserState()) - : MasterInquireClient.Factory.create(getClusterConf(), - getClientContext().getUserState())); - LOG.debug("FileSystemContext re-initialized"); - mReinitializer.onSuccess(); - } - } - - /** - * @return the unique ID of this context - */ - public String getId() { - return mId; - } - - /** - * @return the {@link MasterClientContext} backing this context - */ - public MasterClientContext getMasterClientContext() { - return mMasterClientContext; - } - - /** - * @return the {@link ClientContext} backing this {@link FileSystemContext} - */ - public ClientContext getClientContext() { - return mMasterClientContext; - } - - /** - * @return the cluster level configuration backing this {@link FileSystemContext} - */ - public AlluxioConfiguration getClusterConf() { - return getClientContext().getClusterConf(); - } - - /** - * The path level configuration is a {@link SpecificPathConfiguration}. - * - * If path level configuration has never been loaded from meta master yet, it will be loaded. - * - * @param path the path to get the configuration for - * @return the path level configuration for the specific path - */ - public AlluxioConfiguration getPathConf(AlluxioURI path) { - return new SpecificPathConfiguration(getClientContext().getClusterConf(), - getClientContext().getPathConf(), path); - } - - /** - * @return the master address - * @throws UnavailableException if the master address cannot be determined - */ - public synchronized InetSocketAddress getMasterAddress() throws UnavailableException { - return mMasterClientContext.getMasterInquireClient().getPrimaryRpcAddress(); - } - - /** - * @return {@code true} if URI validation is enabled - */ - public synchronized boolean getUriValidationEnabled() { - return mUriValidationEnabled; - } - - /** - * Acquires a file system master client from the file system master client pool. The resource is - * {@code Closeable}. - * - * @return the acquired file system master client resource - */ - public CloseableResource acquireMasterClientResource() { - try (ReinitBlockerResource r = blockReinit()) { - return acquireClosableClientResource(mFileSystemMasterClientPool); - } - } - - /** - * Acquires a block master client resource from the block master client pool. The resource is - * {@code Closeable}. - * - * @return the acquired block master client resource - */ - public CloseableResource acquireBlockMasterClientResource() { - try (ReinitBlockerResource r = blockReinit()) { - return acquireClosableClientResource(mBlockMasterClientPool); - } - } - - /** - * Acquire a client resource from {@link #mBlockMasterClientPool} or - * {@link #mFileSystemMasterClientPool}. - * - * Because it's possible for a context re-initialization to occur while the resource is - * acquired this method uses an inline class which will save the reference to the pool used to - * acquire the resource. - * - * There are three different cases to which may occur during the release of the resource - * - * 1. release while the context is re-initializing - * - The original {@link #mBlockMasterClientPool} or {@link #mFileSystemMasterClientPool} - * may be null, closed, or overwritten with a difference pool. The inner class here saves - * the original pool from being GCed because it holds a reference to the pool that was used - * to acquire the client initially. Releasing into the closed pool is harmless. - * 2. release after the context has been re-initialized - * - Similar to the above scenario the original {@link #mBlockMasterClientPool} or - * {@link #mFileSystemMasterClientPool} are going to be using an entirely new pool. Since - * this method will save the original pool reference, this method would result in releasing - * into a closed pool which is harmless - * 3. release before any re-initialization - * - This is the normal case. There are no special considerations - * - * @param pool the pool to acquire from and release to - * @param the resource type - * @return a {@link CloseableResource} - */ - private CloseableResource acquireClosableClientResource(DynamicResourcePool pool) { - try { - return new CloseableResource(pool.acquire()) { - @Override - public void closeResource() { - pool.release(get()); - } - }; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Acquires a block worker client from the client pools. If there is no available client instance - * available in the pool, it tries to create a new one. And an exception is thrown if it fails to - * create a new one. - * - * @param workerNetAddress the network address of the channel - * @return the acquired block worker resource - */ - public CloseableResource acquireBlockWorkerClient( - final WorkerNetAddress workerNetAddress) - throws IOException { - try (ReinitBlockerResource r = blockReinit()) { - return acquireBlockWorkerClientInternal(workerNetAddress, getClientContext(), - getClientContext().getUserState()); - } - } - - private CloseableResource acquireBlockWorkerClientInternal( - final WorkerNetAddress workerNetAddress, final ClientContext context, UserState userState) - throws IOException { - SocketAddress address = NetworkAddressUtils - .getDataPortSocketAddress(workerNetAddress, context.getClusterConf()); - GrpcServerAddress serverAddress = GrpcServerAddress.create(workerNetAddress.getHost(), address); - final ClientPoolKey key = new ClientPoolKey(address, AuthenticationUtils - .getImpersonationUser(userState.getSubject(), context.getClusterConf())); - final ConcurrentHashMap poolMap = - mBlockWorkerClientPoolMap; - BlockWorkerClientPool pool = poolMap.computeIfAbsent( - key, - k -> new BlockWorkerClientPool(userState, serverAddress, - context.getClusterConf().getInt(PropertyKey.USER_BLOCK_WORKER_CLIENT_POOL_MIN), - context.getClusterConf().getInt(PropertyKey.USER_BLOCK_WORKER_CLIENT_POOL_MAX), - context.getClusterConf()) - ); - return new CloseableResource(pool.acquire()) { - @Override - public void closeResource() { - releaseBlockWorkerClient(get(), key, poolMap); - } - }; - } - - /** - * Releases a block worker client to the client pools. - * - * @param client the client to release - * @param key the key in the map of the pool from which the client was acquired - * @param poolMap the client pool map - */ - private static void releaseBlockWorkerClient(BlockWorkerClient client, final ClientPoolKey key, - ConcurrentHashMap poolMap) { - if (client == null) { - return; - } - if (poolMap.containsKey(key)) { - poolMap.get(key).release(client); - } else { - LOG.warn("No client pool for key {}, closing client instead. Context may have been closed", - key); - try { - client.close(); - } catch (IOException e) { - LOG.warn("Error closing block worker client for key {}", key, e); - } - } - } - - /** - * @return if the current client is embedded in a worker - */ - public synchronized boolean hasProcessLocalWorker() { - return mBlockWorker != null; - } - - /** - * Acquires the internal block worker as a gateway for worker internal clients to communicate - * with the local worker directly without going through external RPC frameworks. - * - * @return the acquired block worker or null if this client is not interal to a block worker - */ - public Optional getProcessLocalWorker() { - return Optional.ofNullable(mBlockWorker); - } - - /** - * @return if there is a local worker running the same machine - */ - public synchronized boolean hasNodeLocalWorker() throws IOException { - if (!mLocalWorkerInitialized) { - initializeLocalWorker(); - } - return mLocalWorker != null; - } - - /** - * @return a local worker running the same machine, or null if none is found - */ - public synchronized WorkerNetAddress getNodeLocalWorker() throws IOException { - if (!mLocalWorkerInitialized) { - initializeLocalWorker(); - } - return mLocalWorker; - } - - /** - * Gets the cached worker information list. - * This method is relatively cheap as the result is cached, but may not - * be up-to-date. If up-to-date worker info list is required, - * use {@link #getAllWorkers()} instead. - * - * @return the info of all block workers eligible for reads and writes - */ - public List getCachedWorkers() throws IOException { - synchronized (mWorkerInfoList) { - if (mWorkerInfoList.get() == null || mWorkerInfoList.get().isEmpty() - || mWorkerRefreshPolicy.attempt()) { - mWorkerInfoList.set(getAllWorkers()); - } - return mWorkerInfoList.get(); - } - } - - /** - * Gets the worker information list. - * This method is more expensive than {@link #getCachedWorkers()}. - * Used when more up-to-date data is needed. - * - * @return the info of all block workers - */ - private List getAllWorkers() throws IOException { - try (CloseableResource masterClientResource = - acquireBlockMasterClientResource()) { - return masterClientResource.get().getWorkerInfoList().stream() - .map(w -> new BlockWorkerInfo(w.getAddress(), w.getCapacityBytes(), w.getUsedBytes())) - .collect(toList()); - } - } - - private void initializeLocalWorker() throws IOException { - List addresses = getWorkerAddresses(); - if (!addresses.isEmpty()) { - if (addresses.get(0).getHost().equals(NetworkAddressUtils.getClientHostName( - getClusterConf()))) { - mLocalWorker = addresses.get(0); - } - } - mLocalWorkerInitialized = true; - } - - /** - * @return if there are any local workers, the returned list will ONLY contain the local workers, - * otherwise a list of all remote workers will be returned - */ - private List getWorkerAddresses() throws IOException { - List infos; - BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire(); - try { - infos = blockMasterClient.getWorkerInfoList(); - } finally { - mBlockMasterClientPool.release(blockMasterClient); - } - if (infos.isEmpty()) { - throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()); - } - - // Convert the worker infos into net addresses, if there are local addresses, only keep those - List workerNetAddresses = new ArrayList<>(); - List localWorkerNetAddresses = new ArrayList<>(); - String localHostname = NetworkAddressUtils.getClientHostName(getClusterConf()); - for (WorkerInfo info : infos) { - WorkerNetAddress netAddress = info.getAddress(); - if (netAddress.getHost().equals(localHostname)) { - localWorkerNetAddresses.add(netAddress); - } - workerNetAddresses.add(netAddress); - } - - return localWorkerNetAddresses.isEmpty() ? workerNetAddresses : localWorkerNetAddresses; - } - - /** - * Gets the readBlockLocationPolicy. - * - * @param alluxioConf Alluxio configuration - * - * @return the readBlockLocationPolicy - */ - public BlockLocationPolicy getReadBlockLocationPolicy(AlluxioConfiguration alluxioConf) { - return mBlockLocationPolicyMap.computeIfAbsent( - alluxioConf.getClass(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY), - pc -> BlockLocationPolicy.Factory.create(pc, alluxioConf)); - } - - /** - * Gets the writeBlockLocationPolicy. - * - * @param alluxioConf Alluxio configuration - * - * @return the writeBlockLocationPolicy - */ - public BlockLocationPolicy getWriteBlockLocationPolicy(AlluxioConfiguration alluxioConf) { - return mBlockLocationPolicyMap.computeIfAbsent( - alluxioConf.getClass(PropertyKey.USER_BLOCK_WRITE_LOCATION_POLICY), - pc -> BlockLocationPolicy.Factory.create(pc, alluxioConf)); - } - - /** - * Key for block worker client pools. This requires both the worker address and the username, so - * that block workers are created for different users. - */ - private static final class ClientPoolKey { - private final SocketAddress mSocketAddress; - private final String mUsername; - - public ClientPoolKey(SocketAddress socketAddress, String username) { - mSocketAddress = socketAddress; - mUsername = username; - } - - @Override - public int hashCode() { - return Objects.hashCode(mSocketAddress, mUsername); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof ClientPoolKey)) { - return false; - } - ClientPoolKey that = (ClientPoolKey) o; - return Objects.equal(mSocketAddress, that.mSocketAddress) - && Objects.equal(mUsername, that.mUsername); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("socketAddress", mSocketAddress) - .add("username", mUsername) - .toString(); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/FileSystemMasterClient.java b/core/client/fs/src/main/java/alluxio/client/file/FileSystemMasterClient.java deleted file mode 100644 index 3d643329d21f..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/FileSystemMasterClient.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import alluxio.AlluxioURI; -import alluxio.Client; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.AlreadyExistsException; -import alluxio.exception.status.NotFoundException; -import alluxio.grpc.CheckAccessPOptions; -import alluxio.grpc.CheckConsistencyPOptions; -import alluxio.grpc.CompleteFilePOptions; -import alluxio.grpc.CreateDirectoryPOptions; -import alluxio.grpc.CreateFilePOptions; -import alluxio.grpc.DeletePOptions; -import alluxio.grpc.ExistsPOptions; -import alluxio.grpc.FreePOptions; -import alluxio.grpc.GetStatusPOptions; -import alluxio.grpc.ListStatusPOptions; -import alluxio.grpc.ListStatusPartialPOptions; -import alluxio.grpc.MountPOptions; -import alluxio.grpc.RenamePOptions; -import alluxio.grpc.ScheduleAsyncPersistencePOptions; -import alluxio.grpc.SetAclAction; -import alluxio.grpc.SetAclPOptions; -import alluxio.grpc.SetAttributePOptions; -import alluxio.grpc.UpdateUfsModePOptions; -import alluxio.master.MasterClientContext; -import alluxio.security.authorization.AclEntry; -import alluxio.wire.MountPointInfo; -import alluxio.wire.SyncPointInfo; - -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - -/** - * A client to use for interacting with a file system master. - */ -public interface FileSystemMasterClient extends Client { - - /** - * Factory for {@link FileSystemMasterClient}. - */ - class Factory { - - private Factory() {} // prevent instantiation - - /** - * Factory method for {@link FileSystemMasterClient}. - * - * @param conf master client configuration - * @return a new {@link FileSystemMasterClient} instance - */ - public static FileSystemMasterClient create(MasterClientContext conf) { - return new RetryHandlingFileSystemMasterClient(conf); - } - } - - /** - * Check access to a path. - * - * @param path the path to check - * @param options method options - * @throws alluxio.exception.AccessControlException if the access is denied - */ - void checkAccess(AlluxioURI path, CheckAccessPOptions options) - throws AlluxioStatusException; - - /** - * Checks the consistency of Alluxio metadata against the under storage for all files and - * directories in a given subtree. - * - * @param path the root of the subtree to check - * @param options method options - * @return a list of inconsistent files and directories - */ - List checkConsistency(AlluxioURI path, CheckConsistencyPOptions options) - throws AlluxioStatusException; - - /** - * Creates a new directory. - * - * @param path the directory path - * @param options method options - * @throws AlreadyExistsException if the directory already exists - */ - void createDirectory(AlluxioURI path, CreateDirectoryPOptions options) - throws AlluxioStatusException; - - /** - * Creates a new file. - * - * @param path the file path - * @param options method options - * @throws AlreadyExistsException if the file already exists - * @return the uri status of the newly created file - */ - URIStatus createFile(AlluxioURI path, CreateFilePOptions options) throws AlluxioStatusException; - - /** - * Marks a file as completed. - * - * @param path the file path - * @param options the method options - */ - void completeFile(AlluxioURI path, CompleteFilePOptions options) throws AlluxioStatusException; - - /** - * Deletes a file or a directory. - * - * @param path the path to delete - * @param options method options - */ - void delete(AlluxioURI path, DeletePOptions options) throws AlluxioStatusException; - - /** - * Checks whether a file or directory exists. - * - * @param path the file path to check existence - * @param options the method options - * @return whether the file path exists - */ - boolean exists(AlluxioURI path, ExistsPOptions options) throws AlluxioStatusException; - - /** - * Frees a file. - * - * @param path the path to free - * @param options method options - * @throws NotFoundException if the path does not exist - */ - void free(AlluxioURI path, FreePOptions options) throws AlluxioStatusException; - - /** - * @param fileId a file id - * @return the file path for the given file id - */ - String getFilePath(long fileId) throws AlluxioStatusException; - - /** - * @param path the file path - * @param options the getStatus options - * @return the file info for the given file id - * @throws NotFoundException if the path does not exist - */ - URIStatus getStatus(AlluxioURI path, GetStatusPOptions options) throws AlluxioStatusException; - - /** - * @param path the file path - * @return the next blockId for the file - */ - long getNewBlockIdForFile(AlluxioURI path) throws AlluxioStatusException; - - /** - * get the list of paths that are currently being actively synced. - * - * @return the list of paths - */ - List getSyncPathList() throws AlluxioStatusException; - - /** - * Performs a specific action on each {@code URIStatus} in the result of {@link #listStatus}. - * This method is preferred when iterating over directories with a large number of files or - * sub-directories inside. The caller can proceed with partial result without waiting for all - * result returned. - * - * @param path the path to list information about - * @param options options to associate with this operation - * @param action action to apply on each {@code URIStatus} - * @throws NotFoundException if the path does not exist - */ - void iterateStatus(AlluxioURI path, ListStatusPOptions options, - Consumer action) throws AlluxioStatusException; - - /** - * @param path the path to list - * @param options the listStatus options - * @return the list of file information for the given path - * @throws NotFoundException if the path does not exist - */ - List listStatus(AlluxioURI path, ListStatusPOptions options) - throws AlluxioStatusException; - - /** - * @param path the path to list - * @param options the listStatus partial options - * @return the list of file information for the given path - * @throws NotFoundException if the path does not exist - */ - ListStatusPartialResult listStatusPartial( - AlluxioURI path, ListStatusPartialPOptions options) - throws AlluxioStatusException; - - /** - * Mounts the given UFS path under the given Alluxio path. - * - * @param alluxioPath the Alluxio path - * @param ufsPath the UFS path - * @param options mount options - */ - void mount(AlluxioURI alluxioPath, AlluxioURI ufsPath, MountPOptions options) - throws AlluxioStatusException; - - /** - * Updates options of a mount point for the given Alluxio path. - * - * @param alluxioPath the Alluxio path - * @param options mount options - */ - void updateMount(AlluxioURI alluxioPath, MountPOptions options) throws AlluxioStatusException; - - /** - * Lists all mount points and their corresponding under storage addresses. - * This is the same as calling {@link #getMountTable(boolean)} with true argument. - * - * @return a map from String to {@link MountPointInfo} - */ - default Map getMountTable() throws AlluxioStatusException { - return getMountTable(true); - } - - /** - * Lists all mount points and their corresponding under storage addresses. - * - * @param checkUfs whether to get UFS usage info - * - * @return a map from String to {@link MountPointInfo} - */ - Map getMountTable(boolean checkUfs) throws AlluxioStatusException; - - /** - * Renames a file or a directory. - * - * @param src the path to rename - * @param dst new file path - * @throws NotFoundException if the path does not exist - */ - void rename(AlluxioURI src, AlluxioURI dst) throws AlluxioStatusException; - - /** - * Renames a file or a directory. - * - * @param src the path to rename - * @param dst new file path - * @param options rename options - * @throws NotFoundException if the path does not exist - */ - void rename(AlluxioURI src, AlluxioURI dst, RenamePOptions options) throws AlluxioStatusException; - - /** - * Reverse resolve a ufs uri. - * - * @param ufsUri the ufs uri - * @return the alluxio path for the ufsUri - * @throws AlluxioStatusException - */ - AlluxioURI reverseResolve(AlluxioURI ufsUri) throws AlluxioStatusException; - - /** - * Sets the ACL for a path. - * - * @param path the file or directory path - * @param action the set action to perform - * @param entries the ACL entries to use - * @param options the options for setting ACL - * @throws NotFoundException if the path does not exist - */ - void setAcl(AlluxioURI path, SetAclAction action, List entries, SetAclPOptions options) - throws AlluxioStatusException; - - /** - * Sets the file or directory attributes. - * - * @param path the file or directory path - * @param options the file or directory attribute options to be set - * @throws NotFoundException if the path does not exist - */ - void setAttribute(AlluxioURI path, SetAttributePOptions options) throws AlluxioStatusException; - - /** - * Start the active syncing process for a specified path. - * - * @param path the file or directory to be synced - * @throws AlluxioStatusException - */ - void startSync(AlluxioURI path) throws AlluxioStatusException; - - /** - * Stop the active syncing process for a specified path. - * - * @param path the file or directory to stop syncing - * @throws AlluxioStatusException - */ - void stopSync(AlluxioURI path) throws AlluxioStatusException; - - /** - * Schedules the async persistence of the given file. - * - * @param path the file path - * @param options options to use when scheduling the persist - */ - void scheduleAsyncPersist(AlluxioURI path, ScheduleAsyncPersistencePOptions options) - throws AlluxioStatusException; - - /** - * Unmounts the given Alluxio path. - * - * @param alluxioPath the Alluxio path - */ - void unmount(AlluxioURI alluxioPath) throws AlluxioStatusException; - - /** - * Updates the operation mode for the given ufs path. The path is required to be the scheme and - * authority only. For example, to update the mode for under storage at hdfs://ns/folder1 - * specify the argument as hdfs://ns/. Note: the mode for any other mounted under storage which - * shares the prefix (such as hdfs://ns/folder2) is also updated. - * - * @param ufsUri the ufs path - * @param options the options to update ufs operation mode - */ - void updateUfsMode(AlluxioURI ufsUri, UpdateUfsModePOptions options) - throws AlluxioStatusException; - - /** - * @return the state lock waiters and holders thread identifiers - */ - List getStateLockHolders() throws AlluxioStatusException; - - /** - * Mark a path as needed synchronization with the UFS, when this path or any - * of its children are accessed, a sync with the UFS will be performed. - * @param path the path to invalidate - */ - void needsSync(AlluxioURI path) throws AlluxioStatusException; -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java deleted file mode 100644 index 4f0e8f5c1fb4..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache; - -import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.PageReadTargetBuffer; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.resource.LockResource; - -import com.codahale.metrics.Counter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import javax.annotation.concurrent.GuardedBy; - -/** - * Interface for managing cached pages. - */ -public interface CacheManager extends AutoCloseable { - - /** - * State of a cache. - */ - enum State { - /** - * this cache is not in use. - */ - NOT_IN_USE(0), - /** - * this cache is read only. - */ - READ_ONLY(1), - /** - * this cache can both read and write. - */ - READ_WRITE(2); - - private final int mValue; - - State(int value) { - mValue = value; - } - - /** - * @return the value of the state - */ - public int getValue() { - return mValue; - } - } - - /** - * Factory class to get or create a CacheManager. - */ - class Factory { - private static final Logger LOG = LoggerFactory.getLogger(Factory.class); - private static final Lock CACHE_INIT_LOCK = new ReentrantLock(); - @GuardedBy("CACHE_INIT_LOCK") - private static final AtomicReference CACHE_MANAGER = new AtomicReference<>(); - - /** - * @param conf the Alluxio configuration - * @return current CacheManager handle, creating a new one if it doesn't yet exist or null in - * case creation takes a long time by other threads. - */ - public static CacheManager get(AlluxioConfiguration conf) throws IOException { - // TODO(feng): support multiple cache managers - if (CACHE_MANAGER.get() == null) { - try (LockResource lockResource = new LockResource(CACHE_INIT_LOCK)) { - if (CACHE_MANAGER.get() == null) { - CACHE_MANAGER.set( - create(conf)); - } - } catch (IOException e) { - Metrics.CREATE_ERRORS.inc(); - throw new IOException("Failed to create CacheManager", e); - } - } - return CACHE_MANAGER.get(); - } - - /** - * @param conf the Alluxio configuration - * @return an instance of {@link CacheManager} - */ - public static CacheManager create(AlluxioConfiguration conf) throws IOException { - CacheManagerOptions options = CacheManagerOptions.create(conf); - return create(conf, options, PageMetaStore.create(options)); - } - - /** - * @param conf the Alluxio configuration - * @param options the options for local cache manager - * @param pageMetaStore meta store for pages - * @return an instance of {@link CacheManager} - */ - public static CacheManager create(AlluxioConfiguration conf, - CacheManagerOptions options, PageMetaStore pageMetaStore) throws IOException { - try { - boolean isShadowCacheEnabled = - conf.getBoolean(PropertyKey.USER_CLIENT_CACHE_SHADOW_ENABLED); - if (isShadowCacheEnabled) { - return new NoExceptionCacheManager( - new CacheManagerWithShadowCache(LocalCacheManager.create(options, pageMetaStore), - conf)); - } - return new NoExceptionCacheManager(LocalCacheManager.create(options, pageMetaStore)); - } catch (IOException e) { - Metrics.CREATE_ERRORS.inc(); - LOG.error("Failed to create CacheManager", e); - throw e; - } - } - - /** - * Removes the current {@link CacheManager} if it exists. - */ - static void clear() { - try (LockResource r = new LockResource(CACHE_INIT_LOCK)) { - CacheManager manager = CACHE_MANAGER.getAndSet(null); - if (manager != null) { - manager.close(); - } - } catch (Exception e) { - LOG.warn("Failed to close CacheManager: {}", e.toString()); - } - } - - private Factory() {} // prevent instantiation - - private static final class Metrics { - // Note that only counter can be added here. - // Both meter and timer need to be used inline - // because new meter and timer will be created after {@link MetricsSystem.resetAllMetrics()} - /** Errors when creating cache. */ - private static final Counter CREATE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_CREATE_ERRORS.getName()); - - private Metrics() {} // prevent instantiation - } - } - - /** - * Puts a page into the cache manager. This method is best effort. It is possible that this put - * operation returns without page written. - * - * @param pageId page identifier - * @param page page data - * @return true if the put was successful, false otherwise - */ - default boolean put(PageId pageId, byte[] page) { - return put(pageId, page, CacheContext.defaults()); - } - - /** - * Puts a page into the cache manager. This method is best effort. It is possible that this put - * operation returns without page written. - * - * @param pageId page identifier - * @param page page data - * @return true if the put was successful, false otherwise - */ - default boolean put(PageId pageId, ByteBuffer page) { - return put(pageId, page, CacheContext.defaults()); - } - - /** - * Puts a page into the cache manager with scope and quota respected. This method is best effort. - * It is possible that this put operation returns without page written. - * - * @param pageId page identifier - * @param page page data - * @param cacheContext cache related context - * @return true if the put was successful, false otherwise - */ - default boolean put(PageId pageId, byte[] page, CacheContext cacheContext) { - return put(pageId, ByteBuffer.wrap(page), cacheContext); - } - - /** - * Puts a page into the cache manager with scope and quota respected. This method is best effort. - * It is possible that this put operation returns without page written. - * - * @param pageId page identifier - * @param page page data - * @param cacheContext cache related context - * @return true if the put was successful, false otherwise - */ - boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext); - - /** - * Reads the entire page if the queried page is found in the cache, stores the result in buffer. - * - * @param pageId page identifier - * @param bytesToRead number of bytes to read in this page - * @param buffer destination buffer to write - * @param offsetInBuffer offset in the destination buffer to write - * @return number of bytes read, 0 if page is not found, -1 on errors - */ - default int get(PageId pageId, int bytesToRead, byte[] buffer, int offsetInBuffer) { - return get(pageId, 0, bytesToRead, buffer, offsetInBuffer); - } - - /** - * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. - * - * @param pageId page identifier - * @param pageOffset offset into the page - * @param bytesToRead number of bytes to read in this page - * @param buffer destination buffer to write - * @param offsetInBuffer offset in the destination buffer to write - * @return number of bytes read, 0 if page is not found, -1 on errors - */ - default int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, - int offsetInBuffer) { - return get(pageId, pageOffset, bytesToRead, buffer, offsetInBuffer, CacheContext.defaults()); - } - - /** - * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. - * - * @param pageId page identifier - * @param pageOffset offset into the page - * @param bytesToRead number of bytes to read in this page - * @param buffer destination buffer to write - * @param offsetInBuffer offset in the destination buffer to write - * @param cacheContext cache related context - * @return number of bytes read, 0 if page is not found, -1 on errors - */ - default int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, int offsetInBuffer, - CacheContext cacheContext) { - return get(pageId, pageOffset, bytesToRead, new ByteArrayTargetBuffer(buffer, offsetInBuffer), - cacheContext); - } - - /** - * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. - * - * @param pageId page identifier - * @param pageOffset offset into the page - * @param bytesToRead number of bytes to read in this page - * @param buffer destination buffer to write - * @param cacheContext cache related context - * @return number of bytes read, 0 if page is not found, -1 on errors - */ - int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, - CacheContext cacheContext); - - /** - * Get page ids by the given file id. - * @param fileId file identifier - * @param fileLength file length (this will not be needed after we have per-file metadata) - * @return a list of page ids which belongs to the file - */ - default List getCachedPageIdsByFileId(String fileId, long fileLength) { - throw new UnsupportedOperationException(); - } - - /** - * Deletes a page from the cache. - * - * @param pageId page identifier - * @return true if the page is successfully deleted, false otherwise - */ - boolean delete(PageId pageId); - - /** - * @return state of this cache - */ - State state(); - - /** - * - * @param pageId - * @param appendAt - * @param page - * @param cacheContext - * @return true if append was successful - */ - boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cacheContext); -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java deleted file mode 100644 index 1c8e2b748119..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache; - -import static java.util.Objects.requireNonNull; - -import alluxio.client.file.cache.allocator.Allocator; -import alluxio.client.file.cache.allocator.HashAllocator; -import alluxio.client.file.cache.evictor.CacheEvictor; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.client.quota.CacheScope; -import alluxio.collections.IndexDefinition; -import alluxio.collections.IndexedSet; -import alluxio.exception.PageNotFoundException; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; - -import com.codahale.metrics.Counter; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * The default implementation of a metadata store for pages stored in cache. This implementation - * is not thread safe and requires synchronizations on external callers by acquiring the associated - * lock. - */ -@NotThreadSafe -public class DefaultPageMetaStore implements PageMetaStore { - private static final Logger LOG = LoggerFactory.getLogger(DefaultPageMetaStore.class); - /** A map from PageId to page info. */ - private final IndexedSet mPages = new IndexedSet<>(INDEX_PAGE_ID, INDEX_FILE_ID); - private final ImmutableList mDirs; - /** The number of logical bytes used. */ - private final AtomicLong mBytes = new AtomicLong(0); - - protected final ReentrantReadWriteLock mLock = new ReentrantReadWriteLock(); - private final Allocator mAllcator; - - private static final IndexDefinition INDEX_PAGE_ID = - IndexDefinition.ofUnique(PageInfo::getPageId); - private static final IndexDefinition INDEX_FILE_ID = - IndexDefinition.ofNonUnique(pageInfo -> pageInfo.getPageId().getFileId()); - - /** - * @param dirs storage directories - */ - public DefaultPageMetaStore(List dirs) { - this(dirs, new HashAllocator(dirs)); - } - - /** - * Constructor of DefaultMetaStore. - * - * @param dirs storage directories - * @param allocator storage allocator - */ - public DefaultPageMetaStore(List dirs, Allocator allocator) { - mDirs = ImmutableList.copyOf(requireNonNull(dirs)); - mAllcator = requireNonNull(allocator); - //metrics for the num of pages stored in the cache - MetricsSystem.registerGaugeIfAbsent(MetricKey.CLIENT_CACHE_PAGES.getName(), - mPages::size); - } - - @Override - public ReentrantReadWriteLock getLock() { - return mLock; - } - - @Override - @GuardedBy("getLock()") - public boolean hasPage(PageId pageId) { - return mPages.contains(INDEX_PAGE_ID, pageId); - } - - @Override - @GuardedBy("getLock()") - public void addPage(PageId pageId, PageInfo pageInfo) { - addPageInternal(pageId, pageInfo); - pageInfo.getLocalCacheDir().putPage(pageInfo); - } - - private void addPageInternal(PageId pageId, PageInfo pageInfo) { - Preconditions.checkArgument(pageId.equals(pageInfo.getPageId()), "page id mismatch"); - mPages.add(pageInfo); - mBytes.addAndGet(pageInfo.getPageSize()); - Metrics.SPACE_USED.inc(pageInfo.getPageSize()); - } - - @Override - @GuardedBy("getLock()") - public void addTempPage(PageId pageId, PageInfo pageInfo) { - addPageInternal(pageId, pageInfo); - pageInfo.getLocalCacheDir().putTempPage(pageInfo); - } - - @Override - @GuardedBy("getLock().writeLock()") - public void commitFile(String fileId, String newFileId) throws PageNotFoundException { - Set pages = mPages.getByField(INDEX_FILE_ID, fileId); - if (pages.size() == 0) { - throw new PageNotFoundException( - String.format("No Pages found for file %s when committing", fileId)); - } - for (PageInfo oldPage : pages) { - PageId newPageId = new PageId(newFileId, oldPage.getPageId().getPageIndex()); - PageInfo newPageInfo = new PageInfo(newPageId, oldPage.getPageSize(), oldPage.getScope(), - oldPage.getLocalCacheDir()); - mPages.remove(oldPage); - mPages.add(newPageInfo); - } - } - - @Override - public List getStoreDirs() { - return mDirs; - } - - @Override - public PageStoreDir allocate(String fileId, long fileLength) { - return mAllcator.allocate(fileId, fileLength); - } - - @Override - @GuardedBy("getLock()") - public PageInfo getPageInfo(PageId pageId) throws PageNotFoundException { - if (!mPages.contains(INDEX_PAGE_ID, pageId)) { - throw new PageNotFoundException(String.format("Page %s could not be found", pageId)); - } - PageInfo pageInfo = mPages.getFirstByField(INDEX_PAGE_ID, pageId); - pageInfo.getLocalCacheDir().getEvictor().updateOnGet(pageId); - return pageInfo; - } - - @Override - @GuardedBy("getLock()") - public PageInfo removePage(PageId pageId) throws PageNotFoundException { - if (!mPages.contains(INDEX_PAGE_ID, pageId)) { - throw new PageNotFoundException(String.format("Page %s could not be found", pageId)); - } - - PageInfo pageInfo = mPages.getFirstByField(INDEX_PAGE_ID, pageId); - mPages.remove(pageInfo); - mBytes.addAndGet(-pageInfo.getPageSize()); - Metrics.SPACE_USED.dec(pageInfo.getPageSize()); - pageInfo.getLocalCacheDir().deletePage(pageInfo); - return pageInfo; - } - - @Override - public long bytes() { - return mBytes.get(); - } - - @Override - @GuardedBy("getLock()") - public long numPages() { - return mPages.size(); - } - - @Override - @GuardedBy("getLock()") - public void reset() { - mBytes.set(0); - Metrics.SPACE_USED.dec(Metrics.SPACE_USED.getCount()); - mPages.clear(); - } - - @Override - @Nullable - @GuardedBy("getLock()") - public PageInfo evict(CacheScope scope, PageStoreDir pageStoreDir) { - return evictInternal(pageStoreDir.getEvictor()); - } - - PageInfo evictInternal(CacheEvictor evictor) { - PageId victim = evictor.evict(); - if (victim == null) { - return null; - } - PageInfo victimInfo = mPages.getFirstByField(INDEX_PAGE_ID, victim); - if (victimInfo == null) { - LOG.error("Invalid result returned by evictor: page {} not available", victim); - evictor.updateOnDelete(victim); - return null; - } - return victimInfo; - } - - private static final class Metrics { - // Note that only counter can be added here. - // Both meter and timer need to be used inline - // because new meter and timer will be created after {@link MetricsSystem.resetAllMetrics()} - /** Bytes used in the cache. */ - private static final Counter SPACE_USED = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_SPACE_USED_COUNT.getName()); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java deleted file mode 100644 index a54ef7eaacc8..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache; - -import alluxio.AlluxioURI; -import alluxio.client.file.DelegatingFileSystem; -import alluxio.client.file.FileInStream; -import alluxio.client.file.FileSystem; -import alluxio.client.file.URIStatus; -import alluxio.conf.AlluxioConfiguration; -import alluxio.exception.AlluxioException; -import alluxio.grpc.OpenFilePOptions; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -/** - * A FileSystem implementation with a local cache. - */ -public class LocalCacheFileSystem extends DelegatingFileSystem { - private static final Logger LOG = LoggerFactory.getLogger(LocalCacheFileSystem.class); - private final CacheManager mCacheManager; - private final AlluxioConfiguration mConf; - - /** - * @param cacheManage cache manager - * @param fs a FileSystem instance to query on local cache miss - * @param conf the configuration, only respected for the first call - */ - public LocalCacheFileSystem(CacheManager cacheManage, FileSystem fs, AlluxioConfiguration conf) { - super(fs); - mCacheManager = Preconditions.checkNotNull(cacheManage, "cacheManager"); - mConf = Preconditions.checkNotNull(conf, "conf"); - } - - @Override - public AlluxioConfiguration getConf() { - return mDelegatedFileSystem.getConf(); - } - - @Override - public FileInStream openFile(AlluxioURI path, OpenFilePOptions options) - throws IOException, AlluxioException { - if (mCacheManager == null || mCacheManager.state() == CacheManager.State.NOT_IN_USE) { - return mDelegatedFileSystem.openFile(path, options); - } - return openFile(mDelegatedFileSystem.getStatus(path), options); - } - - @Override - public FileInStream openFile(URIStatus status, OpenFilePOptions options) - throws IOException, AlluxioException { - if (mCacheManager == null || mCacheManager.state() == CacheManager.State.NOT_IN_USE) { - return mDelegatedFileSystem.openFile(status, options); - } - return new LocalCacheFileInStream(status, - uriStatus -> mDelegatedFileSystem.openFile(status, options), mCacheManager, mConf); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java deleted file mode 100644 index a1dad88b169b..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java +++ /dev/null @@ -1,765 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache; - -import static alluxio.client.file.cache.CacheManager.State.NOT_IN_USE; -import static alluxio.client.file.cache.CacheManager.State.READ_ONLY; -import static alluxio.client.file.cache.CacheManager.State.READ_WRITE; - -import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.PageReadTargetBuffer; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.client.quota.CacheQuota; -import alluxio.client.quota.CacheScope; -import alluxio.collections.ConcurrentHashSet; -import alluxio.collections.Pair; -import alluxio.exception.PageNotFoundException; -import alluxio.exception.status.ResourceExhaustedException; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.resource.LockResource; - -import com.codahale.metrics.Counter; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A class to manage & serve cached pages. This class coordinates various components to respond for - * thread-safety and enforce cache replacement policies. - * - * The idea of creating a client-side cache is followed after "Improving In-Memory File System - * Reading Performance by Fine-Grained User-Space Cache Mechanisms" by Gu et al, which illustrates - * performance benefits for various read workloads. This class also introduces paging as a caching - * unit. - * - * Lock hierarchy in this class: All operations must follow this order to operate on pages: - *
    - *
  1. Acquire corresponding page lock
    2. - *
  2. Acquire metastore lock mMetaLock
    3. - *
  3. Update metastore
    4. - *
  4. Release metastore lock mMetaLock
    5. - *
  5. Update the pagestore and evictor
    6. - *
  6. Release corresponding page lock
    7. - *
- */ -@ThreadSafe -public class LocalCacheManager implements CacheManager { - private static final Logger LOG = LoggerFactory.getLogger(LocalCacheManager.class); - - private static final int LOCK_SIZE = 1024; - private final long mCacheSize; - /** A readwrite lock pool to guard individual pages based on striping. */ - private final ReadWriteLock[] mPageLocks = new ReentrantReadWriteLock[LOCK_SIZE]; - private final List mPageStoreDirs; - @GuardedBy("PageMetaStore.getLock()") - private final PageMetaStore mPageMetaStore; - /** Executor service for execute the init tasks. */ - private final Optional mInitService; - /** Executor service for execute the async cache tasks. */ - private final Optional mAsyncCacheExecutor; - private final ConcurrentHashSet mPendingRequests; - /** State of this cache. */ - private final AtomicReference mState = new AtomicReference<>(); - private final CacheManagerOptions mOptions; - - /** - * @param options the options of local cache manager - * @param pageMetaStore the metadata store for local cache - * @return an instance of {@link LocalCacheManager} - */ - public static LocalCacheManager create(CacheManagerOptions options, - PageMetaStore pageMetaStore) - throws IOException { - LocalCacheManager manager = new LocalCacheManager(options, pageMetaStore); - List pageStoreDirs = pageMetaStore.getStoreDirs(); - if (manager.mInitService.isPresent()) { - manager.mInitService.get().submit(() -> { - try { - manager.restoreOrInit(pageStoreDirs); - } catch (IOException e) { - LOG.error("Failed to restore LocalCacheManager", e); - } - }); - } else { - manager.restoreOrInit(pageStoreDirs); - } - return manager; - } - - /** - * @param options the options of local cache manager - * @param pageMetaStore the meta store manages the metadata - */ - @VisibleForTesting - LocalCacheManager(CacheManagerOptions options, PageMetaStore pageMetaStore) { - mPageMetaStore = pageMetaStore; - mPageStoreDirs = pageMetaStore.getStoreDirs(); - mOptions = options; - mCacheSize = mPageStoreDirs.stream().map(PageStoreDir::getCapacityBytes).reduce(0L, Long::sum); - for (int i = 0; i < LOCK_SIZE; i++) { - mPageLocks[i] = new ReentrantReadWriteLock(true /* fair ordering */); - } - mPendingRequests = new ConcurrentHashSet<>(); - mAsyncCacheExecutor = - options.isAsyncWriteEnabled() - ? Optional.of( - new ThreadPoolExecutor(mOptions.getAsyncWriteThreads(), - mOptions.getAsyncWriteThreads(), 60, TimeUnit.SECONDS, - new SynchronousQueue<>(), new ThreadPoolExecutor.CallerRunsPolicy())) - : Optional.empty(); - mInitService = - options.isAsyncRestoreEnabled() ? Optional.of(Executors.newSingleThreadExecutor()) : - Optional.empty(); - Metrics.registerGauges(mCacheSize, mPageMetaStore); - mState.set(READ_ONLY); - Metrics.STATE.inc(); - } - - /** - * @param pageId page identifier - * @return the page lock id - */ - private int getPageLockId(PageId pageId) { - return Math.floorMod((int) (pageId.getFileId().hashCode() + pageId.getPageIndex()), LOCK_SIZE); - } - - /** - * Gets the lock for a particular page. Note that multiple pages may share the same lock as lock - * striping is used to reduce resource overhead for locks. - * - * @param pageId page identifier - * @return the corresponding page lock - */ - private ReadWriteLock getPageLock(PageId pageId) { - return mPageLocks[getPageLockId(pageId)]; - } - - /** - * Gets a pair of locks to operate two given pages. One MUST acquire the first lock followed by - * the second lock. - * - * @param pageId1 first page identifier - * @param pageId2 second page identifier - * @return the corresponding page lock pair - */ - private Pair getPageLockPair(PageId pageId1, PageId pageId2) { - int lockId1 = getPageLockId(pageId1); - int lockId2 = getPageLockId(pageId2); - if (lockId1 < lockId2) { - return new Pair<>(mPageLocks[lockId1], mPageLocks[lockId2]); - } else { - return new Pair<>(mPageLocks[lockId2], mPageLocks[lockId1]); - } - } - - /** - * Results of Put. - */ - enum PutResult { - BENIGN_RACING, - INSUFFICIENT_SPACE_EVICTED, - NO_SPACE_LEFT, - OK, - OTHER, - } - - /** - * @return which scope to evict a page or null if space is sufficient - */ - @Nullable - private CacheScope checkScopeToEvict(int pageSize, - PageStoreDir pageStoreDir, - CacheScope scope, CacheQuota quota, - boolean forcedToEvict) { - if (mOptions.isQuotaEnabled()) { - // Check quota usage for each scope - for (CacheScope currentScope = scope; currentScope != null; - currentScope = currentScope.parent()) { - if (((QuotaPageMetaStore) mPageMetaStore).bytes(currentScope) + pageSize - > quota.getQuota(currentScope)) { - return currentScope; - } - } - } - // Check cache space usage - if (forcedToEvict - || pageStoreDir.getCachedBytes() + pageSize > pageStoreDir.getCapacityBytes()) { - return CacheScope.GLOBAL; - } - return null; - } - - @Override - public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { - LOG.debug("put({},{} bytes) enters", pageId, page.remaining()); - if (mState.get() != READ_WRITE) { - Metrics.PUT_NOT_READY_ERRORS.inc(); - Metrics.PUT_ERRORS.inc(); - return false; - } - int originPosition = page.position(); - if (!mOptions.isAsyncWriteEnabled()) { - boolean ok = putInternal(pageId, page, cacheContext); - LOG.debug("put({},{} bytes) exits: {}", pageId, page.position() - originPosition, ok); - if (!ok) { - Metrics.PUT_ERRORS.inc(); - } - return ok; - } - - if (!mPendingRequests.add(pageId)) { // already queued - return false; - } - try { - mAsyncCacheExecutor.get().submit(() -> { - try { - boolean ok = putInternal(pageId, page, cacheContext); - if (!ok) { - Metrics.PUT_ERRORS.inc(); - } - } finally { - mPendingRequests.remove(pageId); - } - }); - } catch (RejectedExecutionException e) { // queue is full, skip - // RejectedExecutionException may be thrown in extreme cases when the - // highly concurrent caching workloads. In these cases, return false - mPendingRequests.remove(pageId); - Metrics.PUT_ASYNC_REJECTION_ERRORS.inc(); - Metrics.PUT_ERRORS.inc(); - LOG.debug("put({},{} bytes) fails due to full queue", pageId, - page.position() - originPosition); - return false; - } - LOG.debug("put({},{} bytes) exits with async write", pageId, page.position() - originPosition); - return true; - } - - private boolean putInternal(PageId pageId, ByteBuffer page, CacheContext cacheContext) { - PutResult result = PutResult.OK; - boolean forcedToEvict = false; - for (int i = 0; i <= mOptions.getMaxEvictionRetries(); i++) { - result = putAttempt(pageId, page, cacheContext, forcedToEvict); - switch (result) { - case OK: - return true; - case BENIGN_RACING: - // failed put attempt due to a benign race, try again. - case INSUFFICIENT_SPACE_EVICTED: - // failed put attempt due to insufficient space, try another time. - // note that, we only evict one item a time in putAttempt. So it is possible the evicted - // page is not large enough to cover the space needed by this page. Try again - continue; - case NO_SPACE_LEFT: - // failed put attempt due to "No space left on device" error. This can happen on - // misconfiguration (e.g., cache capacity is larger than what's available), disk issues, - // or under-estimation of file system overhead (e.g., writing a file of 10B may result - // in 512 B on disk). In this case, we need to force data to be evicted in retries, - // otherwise hitratio may drop due to inability to write new data to cache. - forcedToEvict = true; - continue; - case OTHER: - // fall through intentionally - default: - return false; - } - } - if (result == PutResult.BENIGN_RACING) { - Metrics.PUT_BENIGN_RACING_ERRORS.inc(); - } else if (result == PutResult.INSUFFICIENT_SPACE_EVICTED) { - Metrics.PUT_INSUFFICIENT_SPACE_ERRORS.inc(); - } - return false; - } - - private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheContext, - boolean forcedToEvict) { - LOG.debug("putInternal({},{} bytes) enters", pageId, page.remaining()); - PageInfo victimPageInfo = null; - CacheScope scopeToEvict; - ReadWriteLock pageLock = getPageLock(pageId); - PageStoreDir pageStoreDir; - try (LockResource r = new LockResource(pageLock.writeLock())) { - try (LockResource r2 = new LockResource(mPageMetaStore.getLock().writeLock())) { - if (mPageMetaStore.hasPage(pageId)) { - LOG.debug("{} is already inserted before", pageId); - // TODO(binfan): we should return more informative result in the future - return PutResult.OK; - } - pageStoreDir = mPageMetaStore.allocate(pageId.getFileId(), page.remaining()); - scopeToEvict = checkScopeToEvict(page.remaining(), pageStoreDir, - cacheContext.getCacheScope(), - cacheContext.getCacheQuota(), forcedToEvict); - if (scopeToEvict == null) { - addPageToMetaStore(pageId, page, cacheContext, pageStoreDir); - } else { - if (mOptions.isQuotaEnabled()) { - victimPageInfo = - ((QuotaPageMetaStore) mPageMetaStore).evict(scopeToEvict, pageStoreDir); - } else { - victimPageInfo = mPageMetaStore.evict(pageStoreDir); - } - if (victimPageInfo == null) { - LOG.error("Unable to find page to evict: space used {}, page length {}, cache size {}", - mPageMetaStore.bytes(), page.remaining(), mCacheSize); - Metrics.PUT_EVICTION_ERRORS.inc(); - return PutResult.OTHER; - } - } - } - if (scopeToEvict == null) { - try { - int bytesToWrite = page.remaining(); - pageStoreDir.getPageStore().put(pageId, page, cacheContext.isTemporary()); - // Bytes written to the cache - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_WRITTEN_CACHE.getName()) - .mark(bytesToWrite); - return PutResult.OK; - } catch (ResourceExhaustedException e) { - undoAddPage(pageId); - LOG.error("Failed to add page {} to pageStore", pageId, e); - Metrics.PUT_STORE_WRITE_NO_SPACE_ERRORS.inc(); - return PutResult.NO_SPACE_LEFT; - } catch (IOException e) { - undoAddPage(pageId); - LOG.error("Failed to add page {} to pageStore", pageId, e); - Metrics.PUT_STORE_WRITE_ERRORS.inc(); - return PutResult.OTHER; - } - } - } - - Pair pageLockPair = - getPageLockPair(pageId, victimPageInfo.getPageId()); - try (LockResource r1 = new LockResource(pageLockPair.getFirst().writeLock()); - LockResource r2 = new LockResource(pageLockPair.getSecond().writeLock())) { - // Excise a two-phase commit to evict victim and add new page: - // phase1: remove victim and add new page in metastore in a critical section protected by - // metalock. Evictor will be updated inside metastore. - try (LockResource r3 = new LockResource(mPageMetaStore.getLock().writeLock())) { - if (mPageMetaStore.hasPage(pageId)) { - return PutResult.OK; - } - try { - mPageMetaStore.removePage(victimPageInfo.getPageId()); - } catch (PageNotFoundException e) { - LOG.debug("Page {} is unavailable to evict, likely due to a benign race", - victimPageInfo.getPageId()); - return PutResult.BENIGN_RACING; - } - // Check if we are able to insert page after evicting victim page - scopeToEvict = checkScopeToEvict(page.remaining(), pageStoreDir, - cacheContext.getCacheScope(), cacheContext.getCacheQuota(), false); - if (scopeToEvict == null) { - addPageToMetaStore(pageId, page, cacheContext, pageStoreDir); - } - } - // phase2: remove victim and add new page in pagestore - // Regardless of enoughSpace, delete the victim as it has been removed from the metastore - PageId victim = victimPageInfo.getPageId(); - try { - pageStoreDir.getPageStore().delete(victim); - // Bytes evicted from the cache - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_EVICTED.getName()) - .mark(victimPageInfo.getPageSize()); - // Errors when adding pages - MetricsSystem.meter(MetricKey.CLIENT_CACHE_PAGES_EVICTED.getName()).mark(); - } catch (IOException | PageNotFoundException e) { - if (scopeToEvict == null) { - // Failed to evict page, remove new page from metastore as there will not be enough space - undoAddPage(pageId); - } - if (e instanceof PageNotFoundException) { - //The victim page got deleted by other thread, likely due to a benign racing. Will retry. - return PutResult.BENIGN_RACING; - } - LOG.error("Failed to delete page {} from pageStore", pageId, e); - Metrics.PUT_STORE_DELETE_ERRORS.inc(); - return PutResult.OTHER; - } - if (scopeToEvict != null) { - return PutResult.INSUFFICIENT_SPACE_EVICTED; - } - try { - int bytesToWrite = page.remaining(); - pageStoreDir.getPageStore().put(pageId, page, cacheContext.isTemporary()); - // Bytes written to the cache - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_WRITTEN_CACHE.getName()) - .mark(bytesToWrite); - return PutResult.OK; - } catch (ResourceExhaustedException e) { - undoAddPage(pageId); - LOG.error("Failed to add page {} to pageStore", pageId, e); - Metrics.PUT_STORE_WRITE_NO_SPACE_ERRORS.inc(); - return PutResult.NO_SPACE_LEFT; - } catch (IOException e) { - // Failed to add page, remove new page from metastoree - undoAddPage(pageId); - LOG.error("Failed to add page {} to pageStore", pageId, e); - Metrics.PUT_STORE_WRITE_ERRORS.inc(); - return PutResult.OTHER; - } - } - } - - private void addPageToMetaStore(PageId pageId, ByteBuffer page, CacheContext cacheContext, - PageStoreDir pageStoreDir) { - PageInfo pageInfo = - new PageInfo(pageId, page.remaining(), cacheContext.getCacheScope(), pageStoreDir); - if (cacheContext.isTemporary()) { - mPageMetaStore.addTempPage(pageId, pageInfo); - } else { - mPageMetaStore.addPage(pageId, pageInfo); - } - } - - private void undoAddPage(PageId pageId) { - try (LockResource r3 = new LockResource(mPageMetaStore.getLock().writeLock())) { - mPageMetaStore.removePage(pageId); - } catch (Exception e) { - // best effort to remove this page from meta store and ignore the exception - Metrics.CLEANUP_PUT_ERRORS.inc(); - LOG.error("Failed to undo page add {}", pageId, e); - } - } - - @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, - CacheContext cacheContext) { - Preconditions.checkArgument(pageOffset <= mOptions.getPageSize(), - "Read exceeds page boundary: offset=%s size=%s", pageOffset, mOptions.getPageSize()); - Preconditions.checkArgument(bytesToRead <= buffer.remaining(), - "buffer does not have enough space: bufferRemaining=%s bytesToRead=%s", - buffer.remaining(), bytesToRead); - LOG.debug("get({},pageOffset={}) enters", pageId, pageOffset); - if (mState.get() == NOT_IN_USE) { - Metrics.GET_NOT_READY_ERRORS.inc(); - Metrics.GET_ERRORS.inc(); - return -1; - } - ReadWriteLock pageLock = getPageLock(pageId); - try (LockResource r = new LockResource(pageLock.readLock())) { - PageInfo pageInfo; - try (LockResource r2 = new LockResource(mPageMetaStore.getLock().readLock())) { - pageInfo = mPageMetaStore.getPageInfo(pageId); //check if page exists and refresh LRU items - } catch (PageNotFoundException e) { - LOG.debug("get({},pageOffset={}) fails due to page not found", pageId, pageOffset); - return 0; - } - int bytesRead = - getPage(pageInfo, pageOffset, bytesToRead, buffer, cacheContext); - if (bytesRead <= 0) { - Metrics.GET_ERRORS.inc(); - Metrics.GET_STORE_READ_ERRORS.inc(); - // something is wrong to read this page, let's remove it from meta store - try (LockResource r2 = new LockResource(mPageMetaStore.getLock().writeLock())) { - mPageMetaStore.removePage(pageId); - } catch (PageNotFoundException e) { - // best effort to remove this page from meta store and ignore the exception - Metrics.CLEANUP_GET_ERRORS.inc(); - } - return -1; - } - LOG.debug("get({},pageOffset={}) exits", pageId, pageOffset); - return bytesRead; - } - } - - @Override - public boolean delete(PageId pageId) { - LOG.debug("delete({}) enters", pageId); - if (mState.get() != READ_WRITE) { - Metrics.DELETE_NOT_READY_ERRORS.inc(); - Metrics.DELETE_ERRORS.inc(); - return false; - } - ReadWriteLock pageLock = getPageLock(pageId); - try (LockResource r = new LockResource(pageLock.writeLock())) { - PageInfo pageInfo; - try (LockResource r1 = new LockResource(mPageMetaStore.getLock().writeLock())) { - try { - pageInfo = mPageMetaStore.removePage(pageId); - } catch (PageNotFoundException e) { - LOG.error("Failed to delete page {} from metaStore ", pageId, e); - Metrics.DELETE_NON_EXISTING_PAGE_ERRORS.inc(); - Metrics.DELETE_ERRORS.inc(); - return false; - } - } - boolean ok = deletePage(pageInfo); - LOG.debug("delete({}) exits, success: {}", pageId, ok); - if (!ok) { - Metrics.DELETE_STORE_DELETE_ERRORS.inc(); - Metrics.DELETE_ERRORS.inc(); - } - return ok; - } - } - - @Override - public State state() { - return mState.get(); - } - - @Override - public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cacheContext) { - if (mState.get() != READ_WRITE) { - Metrics.PUT_NOT_READY_ERRORS.inc(); - Metrics.PUT_ERRORS.inc(); - return false; - } - if (appendAt > 0) { - byte[] newPage = new byte[appendAt + page.length]; - get(pageId, 0, appendAt, new ByteArrayTargetBuffer(newPage, 0), cacheContext); - delete(pageId); - System.arraycopy(page, 0, newPage, appendAt, page.length); - return put(pageId, newPage, cacheContext); - } - return put(pageId, page, cacheContext); - } - - /** - * Restores a page store at the configured location, updating meta store accordingly. - * If restore process fails, cleanup the location and create a new page store. - * This method is synchronized to ensure only one thread can enter and operate. - * @param pageStoreDirs - */ - private void restoreOrInit(List pageStoreDirs) throws IOException { - Preconditions.checkState(mState.get() == READ_ONLY); - for (PageStoreDir pageStoreDir : pageStoreDirs) { - if (!restore(pageStoreDir)) { - try (LockResource r = new LockResource(mPageMetaStore.getLock().writeLock())) { - mPageMetaStore.reset(); - } - try { - pageStoreDir.reset(); - } catch (IOException e) { - LOG.error("Cache is in NOT_IN_USE."); - mState.set(NOT_IN_USE); - Metrics.STATE.dec(); - throw e; - } - } - } - LOG.info("Cache is in READ_WRITE."); - mState.set(READ_WRITE); - Metrics.STATE.inc(); - } - - private boolean restore(PageStoreDir pageStoreDir) { - LOG.info("Restoring PageStoreDir ({})", pageStoreDir.getRootPath()); - if (!Files.exists(pageStoreDir.getRootPath())) { - LOG.error("Failed to restore PageStore: Directory {} does not exist", - pageStoreDir.getRootPath()); - return false; - } - try { - pageStoreDir.scanPages(pageInfo -> { - addPageToDir(pageStoreDir, pageInfo.get()); - }); - } catch (IOException | RuntimeException e) { - LOG.error("Failed to restore PageStore", e); - return false; - } - LOG.info("PageStore ({}) restored with {} pages ({} bytes), " - + "discarded {} pages ({} bytes)", - pageStoreDir.getRootPath(), mPageMetaStore.numPages(), mPageMetaStore.bytes(), - Metrics.PAGE_DISCARDED.getCount(), Metrics.BYTE_DISCARDED); - return true; - } - - private void addPageToDir(PageStoreDir pageStoreDir, PageInfo pageInfo) { - PageId pageId = pageInfo.getPageId(); - ReadWriteLock pageLock = getPageLock(pageId); - try (LockResource r = new LockResource(pageLock.writeLock())) { - boolean enoughSpace; - try (LockResource r2 = new LockResource(mPageMetaStore.getLock().writeLock())) { - enoughSpace = pageStoreDir.getCachedBytes() + pageInfo.getPageSize() - <= pageStoreDir.getCapacityBytes(); - if (enoughSpace) { - mPageMetaStore.addPage(pageId, pageInfo); - } - } - if (!enoughSpace) { - try { - pageStoreDir.getPageStore().delete(pageId); - } catch (IOException | PageNotFoundException e) { - throw new RuntimeException("Failed to delete page", e); - } - Metrics.PAGE_DISCARDED.inc(); - Metrics.BYTE_DISCARDED.inc(pageInfo.getPageSize()); - } - } - } - - @Override - public List getCachedPageIdsByFileId(String fileId, long fileLength) { - //ceiling round the result - int numOfPages = (int) ((fileLength - 1) / mOptions.getPageSize()) + 1; - List pageIds = new ArrayList<>(numOfPages); - try (LockResource r = new LockResource(mPageMetaStore.getLock().readLock())) { - for (long pageIndex = 0; pageIndex < numOfPages; pageIndex++) { - PageId pageId = new PageId(fileId, pageIndex); - if (mPageMetaStore.hasPage(pageId)) { - pageIds.add(pageId); - } - } - } - return pageIds; - } - - @Override - public void close() throws Exception { - for (PageStoreDir pageStoreDir: mPageStoreDirs) { - pageStoreDir.close(); - } - mPageMetaStore.reset(); - mInitService.ifPresent(ExecutorService::shutdownNow); - mAsyncCacheExecutor.ifPresent(ExecutorService::shutdownNow); - } - - /** - * Attempts to delete a page from the page store. The page lock must be acquired before calling - * this method. The metastore must be updated before calling this method. - * - * @param pageInfo page info - * @return true if successful, false otherwise - */ - private boolean deletePage(PageInfo pageInfo) { - try { - pageInfo.getLocalCacheDir().getPageStore().delete(pageInfo.getPageId()); - } catch (IOException | PageNotFoundException e) { - LOG.error("Failed to delete page {} from pageStore", pageInfo.getPageId(), e); - return false; - } - return true; - } - - private int getPage(PageInfo pageInfo, int pageOffset, int bytesToRead, - PageReadTargetBuffer target, CacheContext cacheContext) { - try { - int ret = pageInfo.getLocalCacheDir().getPageStore() - .get(pageInfo.getPageId(), pageOffset, bytesToRead, target, - cacheContext.isTemporary()); - if (ret != bytesToRead) { - // data read from page store is inconsistent from the metastore - LOG.error("Failed to read page {}: supposed to read {} bytes, {} bytes actually read", - pageInfo.getPageId(), bytesToRead, ret); - return -1; - } - } catch (IOException | PageNotFoundException e) { - LOG.error("Failed to get existing page {} from pageStore", pageInfo.getPageId(), e); - return -1; - } - return bytesToRead; - } - - private static final class Metrics { - // Note that only counter/guage can be added here. - // Both meter and timer need to be used inline - // because new meter and timer will be created after {@link MetricsSystem.resetAllMetrics()} - /** Total number of bytes discarded when restoring the page store. */ - private static final Counter BYTE_DISCARDED = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_BYTES_DISCARDED.getName()); - /** Errors when cleaning up a failed get operation. */ - private static final Counter CLEANUP_GET_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_CLEANUP_GET_ERRORS.getName()); - /** Errors when cleaning up a failed put operation. */ - private static final Counter CLEANUP_PUT_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_CLEANUP_PUT_ERRORS.getName()); - /** Errors when deleting pages. */ - private static final Counter DELETE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_ERRORS.getName()); - /** Errors when deleting pages due to absence. */ - private static final Counter DELETE_NON_EXISTING_PAGE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_NON_EXISTING_PAGE_ERRORS.getName()); - /** Errors when cache is not ready to delete pages. */ - private static final Counter DELETE_NOT_READY_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_NOT_READY_ERRORS.getName()); - /** Errors when deleting pages due to failed delete in page stores. */ - private static final Counter DELETE_STORE_DELETE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_FROM_STORE_ERRORS.getName()); - /** Errors when getting pages. */ - private static final Counter GET_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_ERRORS.getName()); - /** Errors when cache is not ready to get pages. */ - private static final Counter GET_NOT_READY_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_NOT_READY_ERRORS.getName()); - /** Errors when getting pages due to failed read from page stores. */ - private static final Counter GET_STORE_READ_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_STORE_READ_ERRORS.getName()); - /** Total number of pages discarded when restoring the page store. */ - private static final Counter PAGE_DISCARDED = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PAGES_DISCARDED.getName()); - /** Errors when adding pages. */ - private static final Counter PUT_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_ERRORS.getName()); - /** Errors when adding pages due to failed injection to async write queue. */ - private static final Counter PUT_ASYNC_REJECTION_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_ASYNC_REJECTION_ERRORS.getName()); - /** Errors when adding pages due to failed eviction. */ - private static final Counter PUT_EVICTION_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_EVICTION_ERRORS.getName()); - /** Errors when adding pages due to benign racing eviction. */ - private static final Counter PUT_BENIGN_RACING_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_BENIGN_RACING_ERRORS.getName()); - /** Errors when adding pages due to insufficient space made after eviction. */ - private static final Counter PUT_INSUFFICIENT_SPACE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_INSUFFICIENT_SPACE_ERRORS.getName()); - /** Errors when cache is not ready to add pages. */ - private static final Counter PUT_NOT_READY_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_NOT_READY_ERRORS.getName()); - /** Errors when adding pages due to failed deletes in page store. */ - private static final Counter PUT_STORE_DELETE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_DELETE_ERRORS.getName()); - /** Errors when adding pages due to failed writes to page store. */ - private static final Counter PUT_STORE_WRITE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_WRITE_ERRORS.getName()); - /** Errors when adding pages due to failed writes but before reaching cache capacity. */ - private static final Counter PUT_STORE_WRITE_NO_SPACE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_WRITE_NO_SPACE_ERRORS.getName()); - /** State of the cache. */ - private static final Counter STATE = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_STATE.getName()); - - private static void registerGauges(long cacheSize, PageMetaStore pageMetaStore) { - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.CLIENT_CACHE_SPACE_AVAILABLE.getName()), - () -> cacheSize - pageMetaStore.bytes()); - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.CLIENT_CACHE_SPACE_USED.getName()), - pageMetaStore::bytes); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java deleted file mode 100644 index 4438b496fd61..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache; - -import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.PageReadTargetBuffer; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; - -import com.codahale.metrics.Counter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.List; - -/** - * A wrapper class of CacheManager without throwing unchecked exceptions. - */ -public class NoExceptionCacheManager implements CacheManager { - private static final Logger LOG = LoggerFactory.getLogger(NoExceptionCacheManager.class); - - private final CacheManager mCacheManager; - - /** - * @param cacheManager delegated cache manager - */ - public NoExceptionCacheManager(CacheManager cacheManager) { - mCacheManager = cacheManager; - } - - @Override - public boolean put(PageId pageId, byte[] page) { - try { - return mCacheManager.put(pageId, page); - } catch (Exception e) { - LOG.error("Failed to put page {}", pageId, e); - Metrics.PUT_ERRORS.inc(); - return false; - } - } - - @Override - public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { - try { - return mCacheManager.put(pageId, page, cacheContext); - } catch (Exception e) { - LOG.error("Failed to put page {}, cacheContext {}", pageId, cacheContext, e); - Metrics.PUT_ERRORS.inc(); - return false; - } - } - - @Override - public int get(PageId pageId, int bytesToRead, byte[] buffer, int offsetInBuffer) { - try { - return mCacheManager.get(pageId, bytesToRead, buffer, offsetInBuffer); - } catch (Exception e) { - LOG.error("Failed to get page {}", pageId, e); - Metrics.GET_ERRORS.inc(); - return -1; - } - } - - @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, - int offsetInBuffer) { - try { - return mCacheManager.get(pageId, pageOffset, bytesToRead, buffer, offsetInBuffer); - } catch (Exception e) { - LOG.error("Failed to get page {}, offset {}", pageId, pageOffset, e); - Metrics.GET_ERRORS.inc(); - return -1; - } - } - - @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, - int offsetInBuffer, CacheContext cacheContext) { - try { - return mCacheManager - .get(pageId, pageOffset, bytesToRead, buffer, offsetInBuffer, cacheContext); - } catch (Exception e) { - LOG.error("Failed to get page {}, offset {} cacheContext {}", pageId, pageOffset, - cacheContext, e); - Metrics.GET_ERRORS.inc(); - return -1; - } - } - - @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, - CacheContext cacheContext) { - try { - return mCacheManager - .get(pageId, pageOffset, bytesToRead, buffer, cacheContext); - } catch (Exception e) { - LOG.error("Failed to get page {}, offset {} cacheContext {}", pageId, pageOffset, - cacheContext, e); - Metrics.GET_ERRORS.inc(); - return -1; - } - } - - @Override - public boolean delete(PageId pageId) { - try { - return mCacheManager.delete(pageId); - } catch (Exception e) { - LOG.error("Failed to delete page {}", pageId, e); - Metrics.DELETE_ERRORS.inc(); - return false; - } - } - - @Override - public State state() { - return mCacheManager.state(); - } - - @Override - public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cacheContext) { - return mCacheManager.append(pageId, appendAt, page, cacheContext); - } - - @Override - public void close() throws Exception { - try { - mCacheManager.close(); - } catch (Exception e) { - LOG.error("Failed to close CacheManager", e); - } - } - - @Override - public List getCachedPageIdsByFileId(String fileId, long fileLength) { - return mCacheManager.getCachedPageIdsByFileId(fileId, fileLength); - } - - private static final class Metrics { - // Note that only counter/guage can be added here. - // Both meter and timer need to be used inline - // because new meter and timer will be created after {@link MetricsSystem.resetAllMetrics()} - /** Errors when deleting pages. */ - private static final Counter DELETE_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_ERRORS.getName()); - /** Errors when getting pages. */ - private static final Counter GET_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_ERRORS.getName()); - /** Errors when adding pages. */ - private static final Counter PUT_ERRORS = - MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_ERRORS.getName()); - - private Metrics() {} // prevent instantiation - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java deleted file mode 100644 index 673955559fac..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache; - -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.client.quota.CacheScope; -import alluxio.exception.PageNotFoundException; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.locks.ReadWriteLock; - -/** - * The metadata store for pages stored in cache. - */ -public interface PageMetaStore { - - /** - * @param options the options of cache - * @return an instance of MetaStore - */ - static PageMetaStore create(CacheManagerOptions options) throws IOException { - List dirs = PageStoreDir.createPageStoreDirs(options); - if (options.isQuotaEnabled()) { - return new QuotaPageMetaStore(options.getCacheEvictorOptions(), dirs); - } - return new DefaultPageMetaStore(dirs); - } - - /** - * @return the associated lock - */ - ReadWriteLock getLock(); - - /** - * @param pageId page identifier - * @return if a page is stored in cache - */ - boolean hasPage(PageId pageId); - - /** - * Adds a new page to the cache. - * - * @param pageId page identifier - * @param pageInfo info of the page - */ - void addPage(PageId pageId, PageInfo pageInfo); - - /** - * Adds a new temp page to the cache. - * - * @param pageId page identifier - * @param pageInfo info of the page - */ - void addTempPage(PageId pageId, PageInfo pageInfo); - - /** - * Commits a temp file so that all its pages become permanent. - * - * @param fileId the temp file to commit - * @param newFileId the new file name of the file after committing - */ - void commitFile(String fileId, String newFileId) throws PageNotFoundException; - - /** - * Gets the storage directories. - * - * @return the storage directories - */ - List getStoreDirs(); - - /** - * @param fileId - * @param fileLength - * @return the storage directory - */ - PageStoreDir allocate(String fileId, long fileLength); - - /** - * @param pageId page identifier - * @return page info - */ - PageInfo getPageInfo(PageId pageId) throws PageNotFoundException; - - /** - * Removes a page. - * - * @param pageId page identifier - * @return page info removed - */ - PageInfo removePage(PageId pageId) throws PageNotFoundException; - - /** - * @return the total size of pages stored in bytes - */ - long bytes(); - - /** - * @return the number of pages stored - */ - long numPages(); - - /** - * Resets the meta store. - */ - void reset(); - - /** - * @param pageStoreDir - * @return a page to evict - */ - default PageInfo evict(PageStoreDir pageStoreDir) { - return evict(CacheScope.GLOBAL, pageStoreDir); - } - - /** - * @param cacheScope - * @param pageStoreDir - * @return a page to evict - */ - PageInfo evict(CacheScope cacheScope, PageStoreDir pageStoreDir); -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/ByteArrayTargetBuffer.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/ByteArrayTargetBuffer.java deleted file mode 100644 index 8fa31aa4ee8b..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/ByteArrayTargetBuffer.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import alluxio.annotation.SuppressFBWarnings; - -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; - -/** - * Target buffer backed by bytes array for zero-copy read from page store. - */ -@SuppressFBWarnings( - value = "EI_EXPOSE_REP2", - justification = "The target byte array is exposed as we expect.") -public class ByteArrayTargetBuffer implements PageReadTargetBuffer { - private final byte[] mTarget; - private int mOffset; - - /** - * Constructor. - * @param target - * @param offset - */ - public ByteArrayTargetBuffer(byte[] target, int offset) { - mTarget = target; - mOffset = offset; - } - - @Override - public byte[] byteArray() { - return mTarget; - } - - @Override - public ByteBuffer byteBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public long offset() { - return mOffset; - } - - @Override - public long remaining() { - return mTarget.length - mOffset; - } - - @Override - public void writeBytes(byte[] srcArray, int srcOffset, int length) { - System.arraycopy(srcArray, srcOffset, mTarget, mOffset, length); - mOffset += length; - } - - @Override - public int readFromFile(RandomAccessFile file, int length) throws IOException { - int bytesRead = file.read(mTarget, mOffset, length); - if (bytesRead != -1) { - mOffset += bytesRead; - } - return bytesRead; - } - - @Override - public WritableByteChannel byteChannel() { - throw new UnsupportedOperationException(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/ByteBufferTargetBuffer.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/ByteBufferTargetBuffer.java deleted file mode 100644 index 46c71d9eacf0..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/ByteBufferTargetBuffer.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; - -/** - * Target buffer backed by nio ByteBuffer for zero-copy read from page store. - */ -public class ByteBufferTargetBuffer implements PageReadTargetBuffer { - private final ByteBuffer mTarget; - - /** - * Constructor. - * @param target - */ - public ByteBufferTargetBuffer(ByteBuffer target) { - mTarget = target; - } - - @Override - public byte[] byteArray() { - throw new UnsupportedOperationException(); - } - - @Override - public ByteBuffer byteBuffer() { - return mTarget; - } - - @Override - public long offset() { - return mTarget.position(); - } - - @Override - public WritableByteChannel byteChannel() { - throw new UnsupportedOperationException(); - } - - @Override - public long remaining() { - return mTarget.remaining(); - } - - @Override - public void writeBytes(byte[] srcArray, int srcOffset, int length) { - mTarget.put(srcArray, srcOffset, length); - } - - @Override - public int readFromFile(RandomAccessFile file, int length) throws IOException { - int bytesToRead = Math.min(length, mTarget.remaining()); - ByteBuffer slice = mTarget.slice(); - slice.limit(bytesToRead); - int bytesRead = file.getChannel().read(slice); - mTarget.position(mTarget.position() + bytesRead); - return bytesRead; - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java deleted file mode 100644 index 923c4d653c41..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import static alluxio.client.file.cache.store.PageStoreDir.getFileBucket; - -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageStore; -import alluxio.exception.PageNotFoundException; -import alluxio.exception.status.ResourceExhaustedException; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.commons.io.FileUtils; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * The {@link LocalPageStore} is an implementation of {@link PageStore} which - * stores all pages in a directory somewhere on the local disk. - */ -@NotThreadSafe -public class LocalPageStore implements PageStore { - private static final String ERROR_NO_SPACE_LEFT = "No space left on device"; - public static final String TEMP_DIR = "TEMP"; - private final Path mRoot; - private final long mPageSize; - private final long mCapacity; - private final int mFileBuckets; - - /** - * Creates a new instance of {@link LocalPageStore}. - * - * @param options options for the local page store - */ - public LocalPageStore(PageStoreOptions options) { - mRoot = options.getRootDir(); - mPageSize = options.getPageSize(); - mCapacity = (long) (options.getCacheSize() / (1 + options.getOverheadRatio())); - mFileBuckets = options.getFileBuckets(); - } - - @Override - public void put(PageId pageId, - ByteBuffer page, - boolean isTemporary) throws ResourceExhaustedException, IOException { - Path pagePath = getPagePath(pageId, isTemporary); - try { - if (!Files.exists(pagePath)) { - Path parent = Preconditions.checkNotNull(pagePath.getParent(), - "parent of cache file should not be null"); - Files.createDirectories(parent); - Files.createFile(pagePath); - } - // extra try to ensure output stream is closed - try (FileOutputStream fos = new FileOutputStream(pagePath.toFile(), false)) { - fos.getChannel().write(page); - } - } catch (Exception e) { - Files.deleteIfExists(pagePath); - if (e.getMessage().contains(ERROR_NO_SPACE_LEFT)) { - throw new ResourceExhaustedException( - String.format("%s is full, configured with %d bytes", mRoot, mCapacity), e); - } - throw new IOException("Failed to write file " + pagePath + " for page " + pageId); - } - } - - @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, - boolean isTemporary) throws IOException, PageNotFoundException { - Preconditions.checkArgument(pageOffset >= 0, "page offset should be non-negative"); - Path pagePath = getPagePath(pageId, isTemporary); - if (!Files.exists(pagePath)) { - throw new PageNotFoundException(pagePath.toString()); - } - long pageLength = pagePath.toFile().length(); - Preconditions.checkArgument(pageOffset <= pageLength, "page offset %s exceeded page size %s", - pageOffset, pageLength); - try (RandomAccessFile localFile = new RandomAccessFile(pagePath.toString(), "r")) { - int bytesSkipped = localFile.skipBytes(pageOffset); - if (pageOffset != bytesSkipped) { - throw new IOException( - String.format("Failed to read page %s (%s) from offset %s: %s bytes skipped", - pageId, pagePath, pageOffset, bytesSkipped)); - } - int bytesRead = 0; - int bytesLeft = (int) Math.min(pageLength - pageOffset, target.remaining()); - bytesLeft = Math.min(bytesLeft, bytesToRead); - while (bytesLeft > 0) { - int bytes = target.readFromFile(localFile, bytesLeft); - if (bytes <= 0) { - break; - } - bytesRead += bytes; - bytesLeft -= bytes; - } - return bytesRead; - } - } - - @Override - public void delete(PageId pageId) throws IOException, PageNotFoundException { - Path pagePath = getPagePath(pageId, false); - if (!Files.exists(pagePath)) { - throw new PageNotFoundException(pagePath.toString()); - } - Files.delete(pagePath); - // Cleaning up parent directory may lead to a race condition if one thread is removing a page as - // well as its parent dir corresponding to the fileId, while another thread is adding - // a different page from the same file in the same directory. - // Note that, because (1) the chance of this type of racing is really low and - // (2) even a race happens, the only penalty is an extra cache put failure; - // whereas without the cleanup, there can be an unbounded amount of empty directories - // uncleaned which takes an unbounded amount of space possibly. - // We have seen the overhead goes up to a few hundred GBs due to inode storage overhead - // TODO(binfan): remove the coupled fileId/pagIdex encoding with storage path, so the total - // number of directories can be bounded. - Path parent = - Preconditions.checkNotNull(pagePath.getParent(), "parent of cache file should not be null"); - try (DirectoryStream stream = Files.newDirectoryStream(parent)) { - if (!stream.iterator().hasNext()) { - Files.delete(parent); - } - } catch (NoSuchFileException e) { - //Parent path is deleted by other thread in a benign race, ignore exception and continue - if (LOG.isDebugEnabled()) { - LOG.debug("Parent path is deleted by other thread, ignore and continue.", e); - } - } - } - - @Override - public void commit(String fileId, String newFileId) throws IOException { - Path filePath = getFilePath(newFileId); - Path bucketPath = Preconditions.checkNotNull(filePath.getParent(), - "%s does not have a parent path", filePath); - if (!Files.exists(bucketPath)) { - Files.createDirectories(bucketPath); - } - Files.move( - getTempFilePath(fileId), - filePath, StandardCopyOption.ATOMIC_MOVE); - } - - @Override - public void abort(String fileId) throws IOException { - FileUtils.deleteDirectory(getTempFilePath(fileId).toFile()); - } - - private Path getTempFilePath(String fileId) { - return Paths.get(mRoot.toString(), Long.toString(mPageSize), TEMP_DIR, fileId); - } - - private Path getFilePath(String fileId) { - return Paths.get(mRoot.toString(), Long.toString(mPageSize), - getFileBucket(mFileBuckets, fileId), fileId); - } - - /** - * @param pageId page Id - * @param isTemporary - * @return the local file system path to store this page - */ - @VisibleForTesting - public Path getPagePath(PageId pageId, boolean isTemporary) { - // TODO(feng): encode fileId with URLEncoder to escape invalid characters for file name - Path filePath = - isTemporary ? getTempFilePath(pageId.getFileId()) : getFilePath(pageId.getFileId()); - return filePath.resolve(Long.toString(pageId.getPageIndex())); - } - - @Override - public void close() { - // no-op - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java deleted file mode 100644 index 7426af1c9bf4..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import static alluxio.client.file.cache.store.PageStoreDir.getFileBucket; - -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageInfo; -import alluxio.client.file.cache.PageStore; -import alluxio.client.file.cache.evictor.CacheEvictor; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * - */ -public class LocalPageStoreDir extends QuotaManagedPageStoreDir { - private static final Logger LOG = LoggerFactory.getLogger(LocalPageStoreDir.class); - - private final PageStoreOptions mPageStoreOptions; - private final int mFileBuckets; - private final Pattern mPagePattern; - - private PageStore mPageStore; - - /** - * Constructor for LocalCacheDir. - * @param pageStoreOptions - * @param pageStore - * @param evictor - */ - public LocalPageStoreDir(PageStoreOptions pageStoreOptions, - PageStore pageStore, - CacheEvictor evictor) { - super(pageStoreOptions.getRootDir(), - (long) (pageStoreOptions.getCacheSize() / (1 + pageStoreOptions.getOverheadRatio())), - evictor); - mPageStoreOptions = pageStoreOptions; - mPageStore = pageStore; - mFileBuckets = pageStoreOptions.getFileBuckets(); - // pattern encoding root_path/page_size(ulong)/bucket(uint)/file_id(str)/page_idx(ulong)/ - mPagePattern = Pattern.compile( - String.format("%s/%d/(\\d+)/([^/]+)/(\\d+)", - Pattern.quote(pageStoreOptions.getRootDir().toString()), - pageStoreOptions.getPageSize())); - } - - /** - * Getter for pageStore. - * @return pageStore - */ - @Override - public PageStore getPageStore() { - return mPageStore; - } - - /** - * Reset page store. - */ - @Override - public void reset() throws IOException { - close(); - // when cache is large, e.g. millions of pages, the clear may take a while on deletion - PageStoreDir.clear(getRootPath()); - mPageStore = PageStore.create(mPageStoreOptions); - } - - /** - * Gets a stream of all pages from the page store. This stream needs to be closed as it may - * open IO resources. - * - * @throws IOException if any error occurs - */ - @Override - public void scanPages(Consumer> pageInfoConsumer) throws IOException { - Files.walk(getRootPath()).filter(Files::isRegularFile).map(this::getPageInfo) - .forEach(pageInfoConsumer); - } - - /** - * @param path path of a file - * @return the corresponding page info for the file otherwise null - */ - private Optional getPageInfo(Path path) { - Optional pageId = getPageId(path); - if (pageId.isPresent()) { - long pageSize; - try { - pageSize = Files.size(path); - } catch (IOException e) { - LOG.error("Failed to get file size for " + path, e); - return Optional.empty(); - } - return Optional.of(new PageInfo(pageId.get(), pageSize, this)); - } - return Optional.empty(); - } - - /** - * @param path path of a file - * @return the corresponding page id, or null if the file name does not match the pattern - */ - private Optional getPageId(Path path) { - Matcher matcher = mPagePattern.matcher(path.toString()); - if (!matcher.matches()) { - LOG.error("Unrecognized page file " + path); - return Optional.empty(); - } - try { - String fileBucket = Preconditions.checkNotNull(matcher.group(1)); - String fileId = Preconditions.checkNotNull(matcher.group(2)); - if (!fileBucket.equals(getFileBucket(mFileBuckets, fileId))) { - LOG.error("Bucket number mismatch " + path); - return Optional.empty(); - } - String fileName = Preconditions.checkNotNull(matcher.group(3)); - long pageIndex = Long.parseLong(fileName); - return Optional.of(new PageId(fileId, pageIndex)); - } catch (NumberFormatException e) { - LOG.error("Illegal numbers in path " + path); - return Optional.empty(); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageReadTargetBuffer.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageReadTargetBuffer.java deleted file mode 100644 index bbc826a3520e..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageReadTargetBuffer.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; - -/** - * TargetBuffer for zero-copy read from page store. - */ -public interface PageReadTargetBuffer { - - /** - * @return the byte array - */ - byte[] byteArray(); - - /** - * - * @return the byte buffer - */ - ByteBuffer byteBuffer(); - - /** - * @return offset in the buffer - */ - long offset(); - - /** - * @return the writable channel - */ - WritableByteChannel byteChannel(); - - /** - * @return the remaining for this buffer - */ - long remaining(); - - /** - * @param srcArray - * @param srcOffset - * @param length - */ - void writeBytes(byte[] srcArray, int srcOffset, int length); - - /** - * @param file - * @param length - * @return bytes read from the file - */ - int readFromFile(RandomAccessFile file, int length) throws IOException; -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java deleted file mode 100644 index 4de49a2aff24..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import static com.google.common.base.Preconditions.checkState; - -import alluxio.client.file.cache.PageInfo; -import alluxio.client.file.cache.evictor.CacheEvictor; -import alluxio.resource.LockResource; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import javax.annotation.concurrent.GuardedBy; - -abstract class QuotaManagedPageStoreDir implements PageStoreDir { - - private final ReentrantReadWriteLock mFileIdSetLock = new ReentrantReadWriteLock(); - @GuardedBy("mFileIdSetLock") - private final Set mFileIdSet = new HashSet<>(); - - private final ReentrantReadWriteLock mTempFileIdSetLock = new ReentrantReadWriteLock(); - @GuardedBy("mTempFileIdSetLock") - private final Set mTempFileIdSet = new HashSet<>(); - - private final Path mRootPath; - private final long mCapacityBytes; - private final AtomicLong mBytesUsed = new AtomicLong(0); - - private final CacheEvictor mEvictor; - - QuotaManagedPageStoreDir(Path rootPath, long capacityBytes, CacheEvictor evictor) { - mRootPath = rootPath; - mCapacityBytes = capacityBytes; - mEvictor = evictor; - } - - @Override - public Path getRootPath() { - return mRootPath; - } - - @Override - public long getCapacityBytes() { - return mCapacityBytes; - } - - @Override - public long getCachedBytes() { - return mBytesUsed.get(); - } - - @Override - public void putPage(PageInfo pageInfo) { - mEvictor.updateOnPut(pageInfo.getPageId()); - try (LockResource lock = new LockResource(mFileIdSetLock.writeLock())) { - mFileIdSet.add(pageInfo.getPageId().getFileId()); - } - mBytesUsed.addAndGet(pageInfo.getPageSize()); - } - - @Override - public void putTempPage(PageInfo pageInfo) { - try (LockResource lock = new LockResource(mTempFileIdSetLock.readLock())) { - mTempFileIdSet.add(pageInfo.getPageId().getFileId()); - } - } - - @Override - public long deletePage(PageInfo pageInfo) { - mEvictor.updateOnDelete(pageInfo.getPageId()); - return mBytesUsed.addAndGet(-pageInfo.getPageSize()); - } - - @Override - public boolean putTempFile(String fileId) { - try (LockResource lock = new LockResource(mTempFileIdSetLock.writeLock())) { - return mTempFileIdSet.add(fileId); - } - } - - @Override - public boolean reserve(long bytes) { - long previousBytesUsed; - do { - previousBytesUsed = mBytesUsed.get(); - if (previousBytesUsed + bytes > mCapacityBytes) { - return false; - } - } while (!mBytesUsed.compareAndSet(previousBytesUsed, previousBytesUsed + bytes)); - return true; - } - - @Override - public long release(long bytes) { - return mBytesUsed.addAndGet(-bytes); - } - - @Override - public boolean hasFile(String fileId) { - try (LockResource lock = new LockResource(mFileIdSetLock.readLock())) { - return mFileIdSet.contains(fileId); - } - } - - @Override - public boolean hasTempFile(String fileId) { - try (LockResource lock = new LockResource(mTempFileIdSetLock.readLock())) { - return mTempFileIdSet.contains(fileId); - } - } - - @Override - public CacheEvictor getEvictor() { - return mEvictor; - } - - @Override - public void close() { - try { - getPageStore().close(); - mBytesUsed.set(0); - } catch (Exception e) { - throw new RuntimeException("Close page store failed for dir " + getRootPath().toString(), e); - } - } - - @Override - public void commit(String fileId, String newFileId) throws IOException { - try (LockResource tempFileIdSetlock = new LockResource(mTempFileIdSetLock.writeLock()); - LockResource fileIdSetlock = new LockResource(mFileIdSetLock.writeLock())) { - checkState(mTempFileIdSet.contains(fileId), "temp file does not exist " + fileId); - checkState(!mFileIdSet.contains(newFileId), "file already committed " + newFileId); - getPageStore().commit(fileId, newFileId); - mTempFileIdSet.remove(fileId); - mFileIdSet.add(newFileId); - } - } - - @Override - public void abort(String fileId) throws IOException { - try (LockResource tempFileIdSetlock = new LockResource(mTempFileIdSetLock.writeLock())) { - checkState(mTempFileIdSet.contains(fileId), "temp file does not exist " + fileId); - getPageStore().abort(fileId); - mTempFileIdSet.remove(fileId); - } - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java deleted file mode 100644 index 4526340d668f..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import alluxio.Constants; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageStore; -import alluxio.exception.PageNotFoundException; -import alluxio.proto.client.Cache; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.InvalidProtocolBufferException; -import org.rocksdb.ColumnFamilyDescriptor; -import org.rocksdb.ColumnFamilyHandle; -import org.rocksdb.ColumnFamilyOptions; -import org.rocksdb.CompressionType; -import org.rocksdb.DBOptions; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; -import org.rocksdb.WriteOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A page store implementation which utilizes rocksDB to persist the data. This implementation - * will not be included to client jar by default to reduce client jar size. - */ -@NotThreadSafe -public class RocksPageStore implements PageStore { - private static final Logger LOG = LoggerFactory.getLogger(RocksPageStore.class); - private static final String PAGE_COLUMN = "PAGE"; - private static final byte[] CONF_KEY = "CONF".getBytes(); - private static final int DEFAULT_COLUMN_INDEX = 0; - private static final int PAGE_COLUMN_INDEX = 1; - - private static final long WRITE_BUFFER_SIZE = 64 * Constants.MB; - private final long mCapacity; - private final RocksDB mDb; - private final ColumnFamilyHandle mDefaultColumnHandle; - private final ColumnFamilyHandle mPageColumnHandle; - private final DBOptions mRocksOptions; - - private final WriteOptions mWriteOptions = new WriteOptions(); - - /** - * @param pageStoreOptions options for the rocks page store - * @return a new instance of {@link PageStore} backed by RocksDB - * @throws IOException if I/O error happens - */ - public static RocksPageStore open(PageStoreOptions pageStoreOptions) { - RocksDB.loadLibrary(); - // The RocksObject will be closed together with the RocksPageStore - DBOptions rocksOptions = createDbOptions(); - RocksDB db = null; - List columnHandles = new ArrayList<>(); - try { - db = openDB(pageStoreOptions, rocksOptions, columnHandles); - } catch (RocksDBException e) { - try { - //clear the root dir and retry - PageStoreDir.clear(pageStoreOptions.getRootDir()); - rocksOptions = createDbOptions(); - columnHandles = new ArrayList<>(); - db = openDB(pageStoreOptions, rocksOptions, columnHandles); - } catch (IOException | RocksDBException ex) { - throw new RuntimeException("Couldn't open rocksDB database", e); - } - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException("Couldn't open rocksDB database", e); - } - return new RocksPageStore(pageStoreOptions, rocksOptions, db, - columnHandles.get(DEFAULT_COLUMN_INDEX), columnHandles.get(PAGE_COLUMN_INDEX)); - } - - private static DBOptions createDbOptions() { - DBOptions rocksOptions = new DBOptions() - .setCreateIfMissing(true) - .setCreateMissingColumnFamilies(true); - return rocksOptions; - } - - private static RocksDB openDB(PageStoreOptions pageStoreOptions, - DBOptions rocksOptions, List columnHandles) - throws RocksDBException, InvalidProtocolBufferException { - List columnDescriptors = ImmutableList.of( - new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), - new ColumnFamilyDescriptor(PAGE_COLUMN.getBytes(), - new ColumnFamilyOptions() - .setWriteBufferSize(WRITE_BUFFER_SIZE) - .setCompressionType(CompressionType.NO_COMPRESSION)) - ); - RocksDB db = - RocksDB.open(rocksOptions, pageStoreOptions.getRootDir().toString(), columnDescriptors, - columnHandles); - byte[] confData = db.get(columnHandles.get(DEFAULT_COLUMN_INDEX), CONF_KEY); - Cache.PRocksPageStoreOptions pOptions = - toProto(pageStoreOptions.getPageSize(), pageStoreOptions.getCacheSize(), - pageStoreOptions.getAlluxioVersion()); - if (confData != null) { - Cache.PRocksPageStoreOptions persistedOptions = - Cache.PRocksPageStoreOptions.parseFrom(confData); - if (!persistedOptions.equals(pOptions)) { - db.close(); - rocksOptions.close(); - throw new RocksDBException("Inconsistent configuration for RocksPageStore"); - } - } - db.put(columnHandles.get(DEFAULT_COLUMN_INDEX), CONF_KEY, pOptions.toByteArray()); - return db; - } - - private static Cache.PRocksPageStoreOptions toProto(long pageSize, long cacheSize, - String alluxioVersion) { - return Cache.PRocksPageStoreOptions.newBuilder() - .setCommonOptions(Cache.PPageStoreCommonOptions.newBuilder() - .setPageSize(pageSize) - .setCacheSize(cacheSize) - .setAlluxioVersion(alluxioVersion) - ) - .build(); - } - - /** - * Creates a new instance of {@link PageStore} backed by RocksDB. - * - * @param pageStoreOptions options for the rocks page store - * @param rocksOptions rocksdb options - * @param rocksDB RocksDB instance - * @param defaultColumnHandle default column for storing configurations - * @param pageColumnHandle page column for staring page content - */ - private RocksPageStore(PageStoreOptions pageStoreOptions, - DBOptions rocksOptions, - RocksDB rocksDB, - ColumnFamilyHandle defaultColumnHandle, - ColumnFamilyHandle pageColumnHandle) { - mCapacity = - (long) (pageStoreOptions.getCacheSize() / (1 + pageStoreOptions.getOverheadRatio())); - mRocksOptions = rocksOptions; - mDb = rocksDB; - mDefaultColumnHandle = defaultColumnHandle; - mPageColumnHandle = pageColumnHandle; - } - - @Override - public void put(PageId pageId, ByteBuffer page, boolean isTemporary) throws IOException { - try { - //TODO(beinan): support temp page for rocksdb page store - ByteBuffer key = getKeyFromPageId(pageId, page.isDirect()); - if (page.isDirect()) { - mDb.put(mPageColumnHandle, mWriteOptions, key, page); - } else { - mDb.put(mPageColumnHandle, mWriteOptions, key.array(), page.array()); - } - } catch (RocksDBException e) { - throw new IOException("Failed to store page", e); - } - } - - @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, - boolean isTemporary) throws IOException, PageNotFoundException { - Preconditions.checkArgument(pageOffset >= 0, "page offset should be non-negative"); - try { - byte[] key = getKeyFromPageId(pageId, false).array(); - byte[] page = mDb.get(mPageColumnHandle, key); - if (page == null) { - throw new PageNotFoundException(new String(key)); - } - Preconditions.checkArgument(pageOffset <= page.length, - "page offset %s exceeded page size %s", pageOffset, page.length); - int bytesLeft = - Math.min(page.length - pageOffset, Math.min((int) target.remaining(), bytesToRead)); - System.arraycopy(page, pageOffset, target.byteArray(), (int) target.offset(), bytesLeft); - return bytesLeft; - } catch (RocksDBException e) { - throw new IOException("Failed to retrieve page", e); - } - } - - @Override - public void delete(PageId pageId) throws PageNotFoundException { - try { - byte[] key = getKeyFromPageId(pageId, false).array(); - mDb.delete(mPageColumnHandle, key); - } catch (RocksDBException e) { - throw new PageNotFoundException("Failed to remove page", e); - } - } - - @Override - public void close() { - LOG.info("Closing RocksPageStore and recycling all RocksDB JNI objects"); - mDb.close(); - mRocksOptions.close(); - mDefaultColumnHandle.close(); - mPageColumnHandle.close(); - LOG.info("RocksPageStore closed"); - } - - static ByteBuffer getKeyFromPageId(PageId pageId, boolean isDirect) { - byte[] fileId = pageId.getFileId().getBytes(); - ByteBuffer buf; - if (isDirect) { - buf = ByteBuffer.allocateDirect(Long.BYTES + fileId.length); - } else { - buf = ByteBuffer.allocate(Long.BYTES + fileId.length); - } - buf.putLong(pageId.getPageIndex()); - buf.put(fileId); - return buf; - } - - /** - * @param key key of a record - * @return the corresponding page id, or null if the key does not match the pattern - */ - @Nullable - static PageId getPageIdFromKey(byte[] key) { - if (key.length < Long.BYTES) { - return null; - } - ByteBuffer buf = ByteBuffer.wrap(key); - long pageIndex = buf.getLong(); - String fileId = Charset.defaultCharset().decode(buf).toString(); - return new PageId(fileId, pageIndex); - } - - /** - * @return a new iterator for the rocksdb - */ - public RocksIterator createNewInterator() { - return mDb.newIterator(mPageColumnHandle); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java deleted file mode 100644 index 628c9b386bf3..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import static com.google.common.base.Preconditions.checkState; - -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageInfo; -import alluxio.client.file.cache.PageStore; -import alluxio.client.file.cache.evictor.CacheEvictor; -import alluxio.master.metastore.rocks.RocksUtils; -import alluxio.resource.CloseableIterator; - -import com.google.common.collect.Streams; -import org.rocksdb.RocksIterator; - -import java.io.IOException; -import java.util.Optional; -import java.util.function.Consumer; - -/** - * Represent the dir and file level metadata of a rocksdb page store. - */ -public class RocksPageStoreDir extends QuotaManagedPageStoreDir { - - private final PageStoreOptions mPageStoreOptions; - - private RocksPageStore mPageStore; - - /** - * Constructor of RocksPageStoreDir. - * @param pageStoreOptions - * @param pageStore - * @param cacheEvictor - */ - public RocksPageStoreDir(PageStoreOptions pageStoreOptions, - PageStore pageStore, - CacheEvictor cacheEvictor) { - super(pageStoreOptions.getRootDir(), pageStoreOptions.getCacheSize(), cacheEvictor); - checkState(pageStore instanceof RocksPageStore); - mPageStore = (RocksPageStore) pageStore; - mPageStoreOptions = pageStoreOptions; - } - - @Override - public PageStore getPageStore() { - return mPageStore; - } - - @Override - public void reset() throws IOException { - mPageStore.close(); - PageStoreDir.clear(getRootPath()); - // when cache is large, e.g. millions of pages, initialize may take a while on deletion - mPageStore = (RocksPageStore) PageStore.create(mPageStoreOptions); - } - - @Override - public void scanPages(Consumer> pageInfoConsumer) { - try (CloseableIterator> pageIterator = - RocksUtils.createCloseableIterator(mPageStore.createNewInterator(), this::parsePageInfo)) { - Streams.stream(pageIterator).forEach(pageInfoConsumer); - } - } - - private Optional parsePageInfo(RocksIterator iter) { - PageId id = RocksPageStore.getPageIdFromKey(iter.key()); - long size = iter.value().length; - if (id != null) { - return Optional.of(new PageInfo(id, size, this)); - } - return Optional.empty(); - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/options/FileSystemOptions.java b/core/client/fs/src/main/java/alluxio/client/file/options/FileSystemOptions.java deleted file mode 100644 index 90443c59a6e4..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/options/FileSystemOptions.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.options; - -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; - -import com.google.common.base.Preconditions; - -import java.util.Optional; - -/** - * Options for creating the {@link alluxio.client.file.FileSystem}. - */ -public class FileSystemOptions { - private final boolean mMetadataCacheEnabled; - private final boolean mDataCacheEnabled; - private final Optional mUfsFileSystemOptions; - - /** - * Creates the file system options. - * - * @param conf alluxio configuration - * @return the file system options - */ - public static FileSystemOptions create(AlluxioConfiguration conf) { - return create(conf, Optional.empty()); - } - - /** - * Creates the file system options. - * - * @param conf alluxio configuration - * @param ufsOptions the options for ufs base file system - * @return the file system options - */ - public static FileSystemOptions create(AlluxioConfiguration conf, - Optional ufsOptions) { - return new FileSystemOptions(conf.getBoolean(PropertyKey.USER_METADATA_CACHE_ENABLED), - conf.getBoolean(PropertyKey.USER_CLIENT_CACHE_ENABLED), - ufsOptions); - } - - /** - * Creates a new instance of {@link FileSystemOptions}. - * - * @param metadataCacheEnabled whether metadata cache is enabled - * @param dataCacheEnabled whether data cache is enabled - * @param ufsFileSystemOptions the ufs file system options - */ - private FileSystemOptions(boolean metadataCacheEnabled, boolean dataCacheEnabled, - Optional ufsFileSystemOptions) { - mUfsFileSystemOptions = Preconditions.checkNotNull(ufsFileSystemOptions); - mMetadataCacheEnabled = metadataCacheEnabled; - mDataCacheEnabled = dataCacheEnabled; - } - - /** - * @return the ufs file system options; - */ - public Optional getUfsFileSystemOptions() { - return mUfsFileSystemOptions; - } - - /** - * @return true if metadata cache is enabled, false otherwise - */ - public boolean isMetadataCacheEnabled() { - return mMetadataCacheEnabled; - } - - /** - * @return true if data cache is enabled, false otherwise - */ - public boolean isDataCacheEnabled() { - return mDataCacheEnabled; - } -} - diff --git a/core/client/fs/src/main/java/alluxio/client/file/ufs/UfsBaseFileSystem.java b/core/client/fs/src/main/java/alluxio/client/file/ufs/UfsBaseFileSystem.java deleted file mode 100644 index 5bc1e5ec8821..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/ufs/UfsBaseFileSystem.java +++ /dev/null @@ -1,451 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.ufs; - -import alluxio.AlluxioURI; -import alluxio.client.file.FileInStream; -import alluxio.client.file.FileOutStream; -import alluxio.client.file.FileSystem; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.ListStatusPartialResult; -import alluxio.client.file.URIStatus; -import alluxio.client.file.options.UfsFileSystemOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.exception.AlluxioException; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.grpc.CheckAccessPOptions; -import alluxio.grpc.CreateDirectoryPOptions; -import alluxio.grpc.CreateFilePOptions; -import alluxio.grpc.DeletePOptions; -import alluxio.grpc.ErrorType; -import alluxio.grpc.ExistsPOptions; -import alluxio.grpc.FreePOptions; -import alluxio.grpc.GetStatusPOptions; -import alluxio.grpc.ListStatusPOptions; -import alluxio.grpc.ListStatusPartialPOptions; -import alluxio.grpc.MountPOptions; -import alluxio.grpc.OpenFilePOptions; -import alluxio.grpc.RenamePOptions; -import alluxio.grpc.ScheduleAsyncPersistencePOptions; -import alluxio.grpc.SetAclAction; -import alluxio.grpc.SetAclPOptions; -import alluxio.grpc.SetAttributePOptions; -import alluxio.grpc.UnmountPOptions; -import alluxio.resource.CloseableResource; -import alluxio.security.authorization.AclEntry; -import alluxio.security.authorization.Mode; -import alluxio.underfs.UfsFileStatus; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UfsStatus; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.options.CreateOptions; -import alluxio.underfs.options.DeleteOptions; -import alluxio.underfs.options.ListOptions; -import alluxio.underfs.options.MkdirsOptions; -import alluxio.underfs.options.OpenOptions; -import alluxio.util.CommonUtils; -import alluxio.util.ModeUtils; -import alluxio.util.io.PathUtils; -import alluxio.wire.BlockLocationInfo; -import alluxio.wire.FileInfo; -import alluxio.wire.MountPointInfo; -import alluxio.wire.SyncPointInfo; - -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import io.grpc.Status; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; - -/** - * An implementation of the {@link FileSystem} interface that directly interacts with a target - * UFS. No Alluxio server is involved. - */ -@ThreadSafe -public class UfsBaseFileSystem implements FileSystem { - private static final Logger LOG = LoggerFactory.getLogger(UfsBaseFileSystem.class); - - /** Used to manage closeable resources. */ - private final Closer mCloser = Closer.create(); - protected final FileSystemContext mFsContext; - protected final CloseableResource mUfs; - protected final AlluxioURI mRootUFS; - protected volatile boolean mClosed = false; - - /** - * Constructs a new base file system. - * - * @param fsContext file system context - * @param options the ufs file system options - */ - public UfsBaseFileSystem(FileSystemContext fsContext, UfsFileSystemOptions options) { - Preconditions.checkNotNull(fsContext); - Preconditions.checkNotNull(options); - mFsContext = fsContext; - String ufsAddress = options.getUfsAddress(); - Preconditions.checkArgument(!ufsAddress.isEmpty(), "ufs address should not be empty"); - mRootUFS = new AlluxioURI(ufsAddress); - UfsManager.UfsClient ufsClient = new UfsManager.UfsClient( - () -> UnderFileSystem.Factory.create(ufsAddress, mFsContext.getClusterConf()), - new AlluxioURI(ufsAddress)); - mUfs = ufsClient.acquireUfsResource(); - mCloser.register(mFsContext); - mCloser.register(mUfs); - LOG.debug("Creating file system connecting to ufs address {}", ufsAddress); - } - - /** - * Shuts down the FileSystem. Closes all thread pools and resources used to perform operations. If - * any operations are called after closing the context the behavior is undefined. - */ - @Override - public synchronized void close() throws IOException { - if (!mClosed) { - mClosed = true; - } - mCloser.close(); - } - - @Override - public synchronized boolean isClosed() { - return mClosed; - } - - @Override - public void checkAccess(AlluxioURI path, CheckAccessPOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public void createDirectory(AlluxioURI path, CreateDirectoryPOptions options) { - call(() -> { - // TODO(lu) deal with other options e.g. owner/group - MkdirsOptions ufsOptions = MkdirsOptions.defaults(mFsContext.getPathConf(path)); - if (options.hasMode()) { - ufsOptions.setMode(Mode.fromProto(options.getMode())); - } - if (options.hasRecursive()) { - ufsOptions.setCreateParent(options.getRecursive()); - } - mUfs.get().mkdirs(path.getPath(), ufsOptions); - }); - } - - @Override - public FileOutStream createFile(AlluxioURI path, CreateFilePOptions options) { - return callWithReturn(() -> { - // TODO(lu) deal with other options e.g. owner/group/acl/ensureAtomic - CreateOptions ufsOptions = CreateOptions.defaults(mFsContext.getPathConf(path)); - if (options.hasMode()) { - ufsOptions.setMode(Mode.fromProto(options.getMode())); - } - if (options.hasRecursive()) { - ufsOptions.setCreateParent(options.getRecursive()); - } - return new UfsFileOutStream(mUfs.get().create(path.getPath(), ufsOptions)); - }); - } - - @Override - public void delete(AlluxioURI path, DeletePOptions options) { - call(() -> { - String ufsPath = path.getPath(); - if (mUfs.get().isFile(ufsPath)) { - mUfs.get().deleteFile(ufsPath); - return; - } - DeleteOptions ufsOptions = DeleteOptions.defaults(); - if (options.hasRecursive()) { - ufsOptions.setRecursive(options.getRecursive()); - } - mUfs.get().deleteDirectory(ufsPath, ufsOptions); - }); - } - - @Override - public boolean exists(AlluxioURI path, final ExistsPOptions options) { - return Boolean.TRUE.equals(callWithReturn(() -> mUfs.get().exists(path.getPath()))); - } - - @Override - public void free(AlluxioURI path, final FreePOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public List getBlockLocations(AlluxioURI path) { - throw new UnsupportedOperationException(); - } - - @Override - public List getBlockLocations(URIStatus status) { - throw new UnsupportedOperationException(); - } - - @Override - public AlluxioConfiguration getConf() { - return mFsContext.getClusterConf(); - } - - @Override - public URIStatus getStatus(AlluxioURI path) { - return getStatus(path, GetStatusPOptions.getDefaultInstance()); - } - - @Override - public URIStatus getStatus(AlluxioURI path, final GetStatusPOptions options) { - return callWithReturn(() -> { - String ufsPath = path.getPath(); - return transformStatus(mUfs.get().isFile(ufsPath) - ? mUfs.get().getFileStatus(ufsPath) : mUfs.get().getDirectoryStatus(ufsPath)); - }); - } - - @Override - public List listStatus(AlluxioURI path, final ListStatusPOptions options) { - return callWithReturn(() -> { - ListOptions ufsOptions = ListOptions.defaults(); - if (options.hasRecursive()) { - ufsOptions.setRecursive(options.getRecursive()); - } - UfsStatus[] ufsStatuses = mUfs.get().listStatus(path.getPath(), ufsOptions); - if (ufsStatuses == null || ufsStatuses.length == 0) { - return Collections.emptyList(); - } - return Arrays.stream(ufsStatuses).map(this::transformStatus).collect(Collectors.toList()); - }); - } - - @Override - public void iterateStatus(AlluxioURI path, final ListStatusPOptions options, - Consumer action) { - call(() -> { - ListOptions ufsOptions = ListOptions.defaults(); - if (options.hasRecursive()) { - ufsOptions.setRecursive(options.getRecursive()); - } - UfsStatus[] ufsStatuses = mUfs.get().listStatus(path.getPath(), ufsOptions); - if (ufsStatuses == null || ufsStatuses.length == 0) { - return; - } - Arrays.stream(ufsStatuses).map(this::transformStatus).forEach(action); - }); - } - - @Override - public ListStatusPartialResult listStatusPartial( - AlluxioURI path, final ListStatusPartialPOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public void loadMetadata(AlluxioURI path, final ListStatusPOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public void mount(AlluxioURI alluxioPath, AlluxioURI ufsPath, final MountPOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public void updateMount(AlluxioURI alluxioPath, final MountPOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public Map getMountTable(boolean checkUfs) { - throw new UnsupportedOperationException(); - } - - @Override - public List getSyncPathList() { - throw new UnsupportedOperationException(); - } - - @Override - public void persist(final AlluxioURI path, final ScheduleAsyncPersistencePOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public FileInStream openFile(AlluxioURI path, OpenFilePOptions options) { - return openFile(getStatus(path), options); - } - - @Override - public FileInStream openFile(URIStatus status, OpenFilePOptions options) { - return callWithReturn(() -> { - // TODO(lu) deal with other options e.g. maxUfsReadConcurrency - return new UfsFileInStream(offset -> { - try { - return mUfs.get().open(status.getPath(), OpenOptions.defaults().setOffset(offset)); - } catch (IOException e) { - throw AlluxioRuntimeException.from(e); - } - }, status.getLength()); - }); - } - - @Override - public void rename(AlluxioURI src, AlluxioURI dst, RenamePOptions options) { - call(() -> { - String srcPath = src.getPath(); - String dstPath = dst.getPath(); - boolean renamed; - if (mUfs.get().isFile(srcPath)) { - renamed = mUfs.get().renameFile(srcPath, dstPath); - } else { - renamed = mUfs.get().renameDirectory(srcPath, dstPath); - } - if (!renamed) { - throw new AlluxioRuntimeException( - Status.FAILED_PRECONDITION, - String.format("Failed to rename from %s to %s", srcPath, dstPath), - null, - ErrorType.External, - false - ); - } - }); - } - - @Override - public AlluxioURI reverseResolve(AlluxioURI ufsUri) { - throw new UnsupportedOperationException(); - } - - @Override - public void setAcl(AlluxioURI path, SetAclAction action, List entries, - SetAclPOptions options) { - call(() -> mUfs.get().setAclEntries(path.getPath(), entries)); - } - - @Override - public void setAttribute(AlluxioURI path, SetAttributePOptions options) { - call(() -> { - if (options.hasMode()) { - mUfs.get().setMode(path.getPath(), ModeUtils.protoToShort(options.getMode())); - } - if (options.hasOwner() && options.hasGroup()) { - mUfs.get().setOwner(path.getPath(), options.getOwner(), options.getGroup()); - } else if (options.hasOwner()) { - mUfs.get().setOwner(path.getPath(), options.getOwner(), null); - } else if (options.hasGroup()) { - mUfs.get().setOwner(path.getPath(), null, options.getOwner()); - } - if (options.hasPinned() || options.hasPersisted() || options.hasRecursive() - || options.hasReplicationMax() || options.hasReplicationMin() - || options.getXattrCount() != 0) { - LOG.error("UFS only supports setting mode, owner, and group. Does not support setting {}", - options); - throw new UnsupportedOperationException( - String.format("Cannot set attribute of %s", options)); - } - }); - } - - /** - * Starts the active syncing process on an Alluxio path. - * - * @param path the path to sync - */ - @Override - public void startSync(AlluxioURI path) { - throw new UnsupportedOperationException(); - } - - /** - * Stops the active syncing process on an Alluxio path. - * @param path the path to stop syncing - */ - @Override - public void stopSync(AlluxioURI path) { - throw new UnsupportedOperationException(); - } - - @Override - public void unmount(AlluxioURI path, UnmountPOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public void needsSync(AlluxioURI path) throws IOException, AlluxioException { - throw new UnsupportedOperationException(); - } - - /** - * Transform UFS file/directory status to client-side status. - * - * @param ufsStatus the UFS status to transform - * @return the client-side status - */ - private URIStatus transformStatus(UfsStatus ufsStatus) { - AlluxioURI ufsUri = new AlluxioURI(PathUtils.concatPath(mRootUFS, - CommonUtils.stripPrefixIfPresent(ufsStatus.getName(), mRootUFS.getPath()))); - FileInfo info = new FileInfo().setName(ufsUri.getName()) - .setPath(ufsStatus.getName()) - .setUfsPath(ufsUri.toString()) - .setFolder(ufsStatus.isDirectory()) - .setOwner(ufsStatus.getOwner()) - .setGroup(ufsStatus.getGroup()) - .setMode(ufsStatus.getMode()) - .setCompleted(true); - if (ufsStatus.getLastModifiedTime() != null) { - info.setLastModificationTimeMs(info.getLastModificationTimeMs()); - } - if (ufsStatus.getXAttr() != null) { - info.setXAttr(ufsStatus.getXAttr()); - } - if (ufsStatus instanceof UfsFileStatus) { - UfsFileStatus fileStatus = (UfsFileStatus) ufsStatus; - info.setLength(fileStatus.getContentLength()); - info.setBlockSizeBytes(fileStatus.getBlockSize()); - } else { - info.setLength(0); - } - return new URIStatus(info); - } - - private static void call(UfsCallable callable) { - try { - callable.call(); - } catch (IOException e) { - throw AlluxioRuntimeException.from(e); - } - } - - private static T callWithReturn(UfsCallableWithReturn callable) { - try { - return callable.call(); - } catch (IOException e) { - throw AlluxioRuntimeException.from(e); - } - } - - interface UfsCallable { - void call() throws IOException; - } - - interface UfsCallableWithReturn { - V call() throws IOException; - } -} diff --git a/core/client/fs/src/main/java/alluxio/client/file/ufs/UfsFileInStream.java b/core/client/fs/src/main/java/alluxio/client/file/ufs/UfsFileInStream.java deleted file mode 100644 index a59d0cda5b43..000000000000 --- a/core/client/fs/src/main/java/alluxio/client/file/ufs/UfsFileInStream.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.ufs; - -import alluxio.Seekable; -import alluxio.client.file.FileInStream; -import alluxio.exception.PreconditionMessage; - -import com.google.common.base.Preconditions; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.Optional; -import java.util.function.Function; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Supports reading from a ufs file directly. - */ -@NotThreadSafe -public class UfsFileInStream extends FileInStream { - private final long mLength; - private final Function mFileOpener; - private Optional mUfsInStream = Optional.empty(); - private long mPosition = 0L; - - /** - * Creates a new {@link UfsFileInStream}. - * - * @param fileOpener the file opener to open an ufs in stream with offset - * @param fileLength the file length - */ - public UfsFileInStream(Function fileOpener, long fileLength) { - mFileOpener = Preconditions.checkNotNull(fileOpener); - mLength = fileLength; - } - - @Override - public int read() throws IOException { - if (mPosition == mLength) { // at end of file - return -1; - } - updateStreamIfNeeded(); - int res = mUfsInStream.get().read(); - if (res == -1) { - return -1; - } - mPosition++; - return res; - } - - @Override - public int read(ByteBuffer byteBuffer, int off, int len) throws IOException { - byte[] byteArray = new byte[len]; - int totalBytesRead = read(byteArray, 0, len); - if (totalBytesRead <= 0) { - return totalBytesRead; - } - byteBuffer.position(off).limit(off + len); - byteBuffer.put(byteArray, 0, totalBytesRead); - return totalBytesRead; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - Preconditions.checkArgument(off >= 0 && len >= 0 && len + off <= b.length, - PreconditionMessage.ERR_BUFFER_STATE.toString(), b.length, off, len); - if (mPosition == mLength) { // at end of file - return -1; - } - updateStreamIfNeeded(); - int currentRead = 0; - int totalRead = 0; - while (totalRead < len) { - currentRead = mUfsInStream.get().read(b, off + totalRead, len - totalRead); - if (currentRead <= 0) { - break; - } - totalRead += currentRead; - } - mPosition += totalRead; - return totalRead == 0 ? currentRead : totalRead; - } - - @Override - public long skip(long n) throws IOException { - if (n <= 0) { - return 0; - } - long toBeSkipped = Math.min(n, mLength - mPosition); - if (!mUfsInStream.isPresent()) { - mPosition += toBeSkipped; - return toBeSkipped; - } - long res = mUfsInStream.get().skip(toBeSkipped); - if (res > 0) { - mPosition += res; - } - return res; - } - - @Override - public long remaining() { - return mLength - mPosition; - } - - @Override - public int positionedRead(long position, byte[] buffer, int offset, int length) - throws IOException { - seek(position); - return read(buffer, offset, length); - } - - @Override - public long getPos() throws IOException { - return mPosition; - } - - @Override - public void seek(long pos) throws IOException { - Preconditions.checkArgument(pos >= 0, "Seek position is negative: %s", pos); - Preconditions.checkArgument(pos <= mLength, - "Seek position (%s) exceeds the length of the file (%s)", pos, mLength); - if (mPosition == pos) { - return; - } - if (!mUfsInStream.isPresent()) { - mPosition = pos; - return; - } - if (mUfsInStream.get() instanceof Seekable) { - ((Seekable) mUfsInStream.get()).seek(pos); - } else if (mPosition < pos) { - while (mPosition < pos) { - mPosition += mUfsInStream.get().skip(pos - mPosition); - } - } else { - close(); - } - mPosition = pos; - } - - @Override - public void close() throws IOException { - if (mUfsInStream.isPresent()) { - mUfsInStream.get().close(); - mUfsInStream = Optional.empty(); - } - } - - private void updateStreamIfNeeded() { - if (mUfsInStream.isPresent()) { - return; - } - mUfsInStream = Optional.of(mFileOpener.apply(mPosition)); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/BlockStoreClientTest.java b/core/client/fs/src/test/java/alluxio/client/block/BlockStoreClientTest.java deleted file mode 100644 index 2aabc15b83e7..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/BlockStoreClientTest.java +++ /dev/null @@ -1,598 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import alluxio.ClientContext; -import alluxio.ConfigurationRule; -import alluxio.client.WriteType; -import alluxio.client.block.policy.BlockLocationPolicy; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.client.block.stream.BlockInStream; -import alluxio.client.block.stream.BlockOutStream; -import alluxio.client.block.stream.BlockWorkerClient; -import alluxio.client.block.stream.BlockWorkerDataReader; -import alluxio.client.block.stream.GrpcDataReader; -import alluxio.client.block.stream.NoopClosableResource; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.URIStatus; -import alluxio.client.file.options.InStreamOptions; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.PreconditionMessage; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.CreateLocalBlockResponse; -import alluxio.grpc.OpenFilePOptions; -import alluxio.grpc.OpenLocalBlockRequest; -import alluxio.grpc.OpenLocalBlockResponse; -import alluxio.network.TieredIdentityFactory; -import alluxio.resource.DummyCloseableResource; -import alluxio.util.FileSystemOptionsUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.BlockLocation; -import alluxio.wire.FileBlockInfo; -import alluxio.wire.FileInfo; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.BlockWorker; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import io.grpc.stub.ClientCallStreamObserver; -import io.grpc.stub.StreamObserver; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.Closeable; -import java.io.File; -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Tests for {@link BlockStoreClient}. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({FileSystemContext.class}) -public final class BlockStoreClientTest { - - private static final InstancedConfiguration S_CONF = Configuration.copyGlobal(); - - private static final long BLOCK_ID = 3L; - private static final long BLOCK_LENGTH = 100L; - private static final String WORKER_HOSTNAME_LOCAL = - NetworkAddressUtils.getLocalHostName((int) S_CONF - .getMs(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS)); - private static final String WORKER_HOSTNAME_REMOTE = "remote"; - private static final WorkerNetAddress WORKER_NET_ADDRESS_LOCAL = new WorkerNetAddress() - .setHost(WORKER_HOSTNAME_LOCAL); - private static final WorkerNetAddress WORKER_NET_ADDRESS_REMOTE = new WorkerNetAddress() - .setHost(WORKER_HOSTNAME_REMOTE); - private ClientCallStreamObserver mStreamObserver; - private StreamObserver mResponseObserver; - - /** - * A mock class used to return controlled result when selecting workers. - */ - @ThreadSafe - public static class MockBlockLocationPolicyTest implements BlockLocationPolicy { - private List mWorkerNetAddresses; - private int mIndex; - - /** - * Cosntructs this mock location policy with empty host list, - * needed for instantiation in {@link BlockLocationPolicy.Factory}. - * - * @param ignoredConf is unused - */ - public MockBlockLocationPolicyTest(AlluxioConfiguration ignoredConf) { - mIndex = 0; - mWorkerNetAddresses = Collections.emptyList(); - } - - /** - * Constructs this mock policy that returns the given result, once a time, in the input order. - * - * @param addresses list of addresses this mock policy will return - */ - public MockBlockLocationPolicyTest(List addresses) { - mWorkerNetAddresses = Lists.newArrayList(addresses); - mIndex = 0; - } - - public void setHosts(List addresses) { - mWorkerNetAddresses = Lists.newArrayList(addresses); - mIndex = 0; - } - - @Override - public Optional getWorker(GetWorkerOptions options) { - if (mWorkerNetAddresses.isEmpty()) { - return Optional.empty(); - } - return Optional.of(mWorkerNetAddresses.get(mIndex++)); - } - } - - private BlockMasterClient mMasterClient; - private BlockWorkerClient mWorkerClient; - private BlockStoreClient mBlockStore; - private FileSystemContext mContext; - private ClientContext mClientContext; - - @Before - public void before() throws Exception { - mMasterClient = PowerMockito.mock(BlockMasterClient.class); - mWorkerClient = PowerMockito.mock(BlockWorkerClient.class); - - mClientContext = ClientContext.create(S_CONF); - - mContext = PowerMockito.mock(FileSystemContext.class); - when(mContext.acquireBlockMasterClientResource()) - .thenReturn(new DummyCloseableResource<>(mMasterClient)); - when(mContext.getClientContext()).thenReturn(mClientContext); - when(mContext.getClusterConf()).thenReturn(S_CONF); - when(mContext.getReadBlockLocationPolicy(any(AlluxioConfiguration.class))) - .thenAnswer((Answer) invocation -> { - AlluxioConfiguration conf = - invocation.getArgument(0, AlluxioConfiguration.class); - return BlockLocationPolicy.Factory.create( - conf.getClass(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY), conf); - }); - - mBlockStore = new BlockStoreClient(mContext, - TieredIdentityFactory.fromString("node=" + WORKER_HOSTNAME_LOCAL, S_CONF)); - - when(mContext.acquireBlockWorkerClient(any(WorkerNetAddress.class))) - .thenReturn(new NoopClosableResource<>(mWorkerClient)); - mStreamObserver = PowerMockito.mock(ClientCallStreamObserver.class); - when(mWorkerClient.writeBlock(any(StreamObserver.class))) - .thenReturn(mStreamObserver); - when(mWorkerClient.openLocalBlock(any(StreamObserver.class))) - .thenReturn(mStreamObserver); - when(mStreamObserver.isReady()).thenReturn(true); - when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList( - new BlockWorkerInfo(new WorkerNetAddress(), -1, -1))); - } - - @Test - public void getOutStreamUsingLocationPolicy() { - OutStreamOptions options = - OutStreamOptions.defaults(mContext).setWriteType(WriteType.MUST_CACHE) - .setLocationPolicy((workerOptions) -> { - throw new RuntimeException("policy threw exception"); - }); - assertThrows(Exception.class, () -> mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options)); - } - - @Test - public void getOutStreamMissingLocationPolicy() { - OutStreamOptions options = - OutStreamOptions.defaults(mContext).setBlockSizeBytes(BLOCK_LENGTH) - .setWriteType(WriteType.MUST_CACHE).setLocationPolicy(null); - Exception e = assertThrows(NullPointerException.class, () -> - mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options)); - assertTrue(e.getMessage() - .contains(PreconditionMessage.BLOCK_WRITE_LOCATION_POLICY_UNSPECIFIED.toString())); - } - - @Test - public void getOutStreamNoWorker() { - OutStreamOptions options = - OutStreamOptions - .defaults(mContext) - .setBlockSizeBytes(BLOCK_LENGTH) - .setWriteType(WriteType.MUST_CACHE) - .setLocationPolicy( - new MockBlockLocationPolicyTest(Lists.newArrayList())); - Exception e = assertThrows(UnavailableException.class, () -> - mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options)); - assertTrue(e.getMessage() - .contains(ExceptionMessage.NO_SPACE_FOR_BLOCK_ON_WORKER.getMessage(BLOCK_LENGTH))); - } - - @Test - public void getOutStreamLocal() throws Exception { - File file = File.createTempFile("test", ".tmp"); - CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder() - .setPath(file.getAbsolutePath()).build(); - when(mWorkerClient.createLocalBlock(any(StreamObserver.class))) - .thenAnswer((Answer) invocation -> { - StreamObserver observer = - invocation.getArgument(0, StreamObserver.class); - observer.onNext(response); - return mStreamObserver; - }); - - OutStreamOptions options = OutStreamOptions.defaults(mContext) - .setBlockSizeBytes(BLOCK_LENGTH).setLocationPolicy( - new MockBlockLocationPolicyTest(Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL))) - .setWriteType(WriteType.MUST_CACHE); - BlockOutStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); - assertEquals(WORKER_NET_ADDRESS_LOCAL, stream.getAddress()); - } - - @Test - public void getOutStreamRemote() throws Exception { - WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); - WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); - OutStreamOptions options = - OutStreamOptions.defaults(mContext).setBlockSizeBytes(BLOCK_LENGTH) - .setLocationPolicy(new MockBlockLocationPolicyTest(Arrays.asList(worker1, worker2))) - .setWriteType(WriteType.MUST_CACHE); - BlockOutStream stream1 = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); - assertEquals(worker1, stream1.getAddress()); - BlockOutStream stream2 = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); - assertEquals(worker2, stream2.getAddress()); - } - - @Test - public void getOutStreamWithReplicated() throws Exception { - File file = File.createTempFile("test", ".tmp"); - CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder() - .setPath(file.getAbsolutePath()).build(); - when(mWorkerClient.createLocalBlock(any(StreamObserver.class))) - .thenAnswer((Answer) invocation -> { - StreamObserver observer = - invocation.getArgument(0, StreamObserver.class); - observer.onNext(response); - return mStreamObserver; - }); - - when(mContext.getCachedWorkers()).thenReturn(Lists - .newArrayList(new BlockWorkerInfo(WORKER_NET_ADDRESS_LOCAL, -1, -1), - new BlockWorkerInfo(WORKER_NET_ADDRESS_REMOTE, -1, -1))); - OutStreamOptions options = - OutStreamOptions.defaults(mContext).setBlockSizeBytes(BLOCK_LENGTH).setLocationPolicy( - new MockBlockLocationPolicyTest( - Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL, WORKER_NET_ADDRESS_REMOTE))) - .setWriteType(WriteType.MUST_CACHE).setReplicationMin(2); - BlockOutStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); - - assertEquals(alluxio.client.block.stream.BlockOutStream.class, stream.getClass()); - } - - @Test - public void getInStreamUfsMockLocaltion() throws Exception { - try (Closeable ignored = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY, - MockBlockLocationPolicyTest.class.getTypeName(), S_CONF).toResource()) { - WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); - WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); - BlockInfo info = new BlockInfo().setBlockId(0); - URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true) - .setBlockIds(Collections.singletonList(0L)) - .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); - OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build(); - InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, S_CONF, mContext); - ((MockBlockLocationPolicyTest) options.getUfsReadLocationPolicy()) - .setHosts(Arrays.asList(worker1, worker2)); - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); - when(mContext.getCachedWorkers()).thenReturn( - Lists.newArrayList(new BlockWorkerInfo(worker1, -1, -1), - new BlockWorkerInfo(worker2, -1, -1))); - - // Location policy chooses worker1 first. - assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); - // Location policy chooses worker2 second. - assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); - } - } - - @Test - public void getInStreamUfsLocalFirst() throws Exception { - WorkerNetAddress remote = new WorkerNetAddress().setHost("remote"); - WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); - BlockInfo info = new BlockInfo().setBlockId(0); - URIStatus dummyStatus = - new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) - .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); - OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build(); - InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, S_CONF, mContext); - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); - when(mContext.getCachedWorkers()).thenReturn( - Lists.newArrayList(new BlockWorkerInfo(remote, 100, 0), - new BlockWorkerInfo(local, 100, 0))); - - BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, options); - assertEquals(local, stream.getAddress()); - assertEquals(GrpcDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } - - @Test - public void getInStreamNoWorkers() throws Exception { - URIStatus dummyStatus = - new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L))); - InStreamOptions options = - new InStreamOptions(dummyStatus, FileSystemOptionsUtils.openFileDefaults(S_CONF), - S_CONF, mContext); - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); - when(mContext.getCachedWorkers()).thenReturn(Collections.emptyList()); - Exception e = assertThrows(UnavailableException.class, () -> - mBlockStore.getInStream(BLOCK_ID, options).getAddress()); - assertTrue(e.getMessage().contains(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage())); - } - - @Test - public void getInStreamMissingBlock() throws Exception { - URIStatus dummyStatus = new URIStatus( - new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L))); - InStreamOptions options = - new InStreamOptions(dummyStatus, FileSystemOptionsUtils.openFileDefaults(S_CONF), - S_CONF, mContext); - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); - Exception e = assertThrows(UnavailableException.class, () -> - mBlockStore.getInStream(BLOCK_ID, options).getAddress()); - assertTrue(e.getMessage().contains("unavailable in both Alluxio and UFS")); - } - - @Test - public void getInStreamLocal() throws Exception { - WorkerNetAddress remote = new WorkerNetAddress().setHost("remote"); - WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); - - // Mock away gRPC usage. - OpenLocalBlockResponse response = OpenLocalBlockResponse.newBuilder().setPath("/tmp").build(); - when(mWorkerClient.openLocalBlock(any(StreamObserver.class))).thenAnswer(invocation -> { - mResponseObserver = invocation.getArgument(0, StreamObserver.class); - return mStreamObserver; - }); - doAnswer(invocation -> { - mResponseObserver.onNext(response); - mResponseObserver.onCompleted(); - return null; - }).when(mStreamObserver).onNext(any(OpenLocalBlockRequest.class)); - - BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays - .asList(new BlockLocation().setWorkerAddress(remote), - new BlockLocation().setWorkerAddress(local))); - - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); - assertEquals(local, mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( - new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), - S_CONF, mContext)) - .getAddress()); - } - - @Test - public void getInStreamRemote() throws Exception { - WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1"); - WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2"); - - BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays - .asList(new BlockLocation().setWorkerAddress(remote1), - new BlockLocation().setWorkerAddress(remote2))); - - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); - // We should sometimes get remote1 and sometimes get remote2. - Set results = new HashSet<>(); - for (int i = 0; i < 40; i++) { - results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( - new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), - S_CONF, mContext)) - .getAddress()); - } - assertEquals(Sets.newHashSet(remote1, remote2), results); - } - - @Test - public void getInStreamProcessLocal() throws Exception { - WorkerNetAddress remote = new WorkerNetAddress().setHost("remote"); - WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); - BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays - .asList(new BlockLocation().setWorkerAddress(remote), - new BlockLocation().setWorkerAddress(local))); - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); - - when(mContext.hasProcessLocalWorker()).thenReturn(true); - Optional blockWorker = Optional.of(Mockito.mock(BlockWorker.class)); - when(mContext.getProcessLocalWorker()).thenReturn(blockWorker); - - BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( - new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), - S_CONF, mContext)); - assertEquals(local, stream.getAddress()); - assertEquals(BlockWorkerDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } - - @Test - public void getInStreamUfsProcessLocal() throws Exception { - WorkerNetAddress remote = new WorkerNetAddress().setHost("remote"); - WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); - BlockInfo info = new BlockInfo().setBlockId(0); - URIStatus dummyStatus = - new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) - .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); - OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build(); - InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, S_CONF, mContext); - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); - when(mContext.getCachedWorkers()).thenReturn( - Lists.newArrayList(new BlockWorkerInfo(remote, 100, 0), - new BlockWorkerInfo(local, 100, 0))); - - when(mContext.getNodeLocalWorker()).thenReturn(local); - when(mContext.hasProcessLocalWorker()).thenReturn(true); - Optional blockWorker = Optional.of(Mockito.mock(BlockWorker.class)); - when(mContext.getProcessLocalWorker()).thenReturn(blockWorker); - - BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, options); - assertEquals(local, stream.getAddress()); - assertEquals(BlockWorkerDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } - - @Test - public void getInStreamInAlluxioOnlyFallbackToAvailableWorker() throws Exception { - int workerCount = 4; - boolean persisted = false; - int[] blockLocations = new int[]{2, 3}; - Map failedWorkers = ImmutableMap.of( - 0, 3L, - 1, 1L, - 3, 2L); - int expectedWorker = 2; - testGetInStreamFallback(workerCount, persisted, blockLocations, failedWorkers, expectedWorker); - } - - @Test - public void getInStreamPersistedAndInAlluxioFallbackToUFS() throws Exception { - int workerCount = 3; - boolean persisted = true; - int[] blockLocations = new int[]{0, 2}; - Map failedWorkers = ImmutableMap.of( - 0, 5L, - 2, 2L); - int expectedWorker = 1; - testGetInStreamFallback(workerCount, persisted, blockLocations, failedWorkers, expectedWorker); - } - - @Test - public void getInStreamPersistedFallbackToLeastRecentlyFailed() throws Exception { - int workerCount = 3; - boolean persisted = true; - int[] blockLocations = new int[0]; - Map failedWorkers = ImmutableMap.of( - 0, 5L, - 1, 1L, - 2, 2L); - int expectedWorker = 1; - testGetInStreamFallback(workerCount, persisted, blockLocations, failedWorkers, expectedWorker); - } - - @Test - public void getInStreamInAlluxioOnlyFallbackToLeastRecentlyFailed() throws Exception { - int workerCount = 5; - boolean persisted = false; - int[] blockLocations = new int[]{1, 2, 3}; - Map failedWorkers = ImmutableMap.of( - 0, 5L, - 1, 3L, - 2, 2L, - 3, 4L, - 4, 1L); - int expectedWorker = 2; - testGetInStreamFallback(workerCount, persisted, blockLocations, failedWorkers, expectedWorker); - } - - @Test - public void getInStreamInAlluxioWhenCreateStreamIsFailed() throws Exception { - int workerCount = 5; - boolean persisted = false; - int[] blockLocations = new int[]{2, 3, 4}; - Map failedWorkers = ImmutableMap.of( - 0, 3L, - 1, 1L, - 3, 2L); - int expectedWorker = 2; - WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; - for (int i = 0; i < workers.length - 1; i++) { - workers[i] = new WorkerNetAddress().setHost(String.format("worker-%d", i)); - } - workers[workers.length - 1] = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); - when(mContext.acquireBlockWorkerClient(WORKER_NET_ADDRESS_LOCAL)) - .thenThrow(new UnavailableException("failed to connect to " - + WORKER_NET_ADDRESS_LOCAL.getHost())); - BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) - .setLocations(Arrays.stream(blockLocations).mapToObj(x -> - new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); - URIStatus dummyStatus = - new URIStatus(new FileInfo().setPersisted(persisted) - .setBlockIds(Collections.singletonList(BLOCK_ID)) - .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); - BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); - when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg - .getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() - .getNetAddress()); - InStreamOptions options = - new InStreamOptions(dummyStatus, FileSystemOptionsUtils.openFileDefaults(S_CONF), - S_CONF, mContext); - options.setUfsReadLocationPolicy(mockPolicy); - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); - when(mContext.getCachedWorkers()).thenReturn( - Arrays.stream(workers) - .map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList()))); - Map failedWorkerAddresses = failedWorkers.entrySet().stream() - .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - BlockInStream inStream = null; - int i = 2; - while (i-- > 0) { - try { - inStream = mBlockStore.getInStream(BLOCK_ID, options, - failedWorkerAddresses); - } catch (Exception e) { - //do nothing - } - } - Objects.requireNonNull(inStream); - assertEquals(workers[expectedWorker], inStream.getAddress()); - } - - private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, - Map failedWorkers, int expectedWorker) throws Exception { - WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; - Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); - BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) - .setLocations(Arrays.stream(blockLocations).mapToObj(x -> - new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); - URIStatus dummyStatus = - new URIStatus(new FileInfo().setPersisted(isPersisted) - .setBlockIds(Collections.singletonList(BLOCK_ID)) - .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); - BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); - when(mockPolicy.getWorker(any())).thenAnswer(arg -> Optional.ofNullable(arg - .getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() - .getNetAddress())); - InStreamOptions options = - new InStreamOptions(dummyStatus, FileSystemOptionsUtils.openFileDefaults(S_CONF), - S_CONF, mContext); - options.setUfsReadLocationPolicy(mockPolicy); - when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); - when(mContext.getCachedWorkers()).thenReturn(Arrays.stream(workers) - .map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList()))); - Map failedWorkerAddresses = failedWorkers.entrySet().stream() - .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); - - assertEquals(workers[expectedWorker], inStream.getAddress()); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/policy/CapacityBaseRandomPolicyTest.java b/core/client/fs/src/test/java/alluxio/client/block/policy/CapacityBaseRandomPolicyTest.java deleted file mode 100644 index 7748e42119ba..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/policy/CapacityBaseRandomPolicyTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.Configuration; -import alluxio.wire.WorkerNetAddress; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Optional; - -public class CapacityBaseRandomPolicyTest { - - @Test - public void getWorkerDifferentCapacity() { - GetWorkerOptions getWorkerOptions = GetWorkerOptions.defaults(); - ArrayList blockWorkerInfos = new ArrayList<>(); - WorkerNetAddress netAddress1 = new WorkerNetAddress().setHost("1"); - WorkerNetAddress netAddress2 = new WorkerNetAddress().setHost("2"); - WorkerNetAddress netAddress3 = new WorkerNetAddress().setHost("3"); - WorkerNetAddress netAddress4 = new WorkerNetAddress().setHost("4"); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress1, 10, 0)); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress2, 100, 0)); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress3, 0, 0)); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress4, 1000, 0)); - getWorkerOptions.setBlockWorkerInfos(blockWorkerInfos); - Assert.assertEquals(Optional.of(netAddress1), - buildPolicyWithTarget(0).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress1), - buildPolicyWithTarget(7).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress1), - buildPolicyWithTarget(9).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress2), - buildPolicyWithTarget(10).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress2), - buildPolicyWithTarget(70).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress2), - buildPolicyWithTarget(109).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress4), - buildPolicyWithTarget(110).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress4), - buildPolicyWithTarget(700).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress4), - buildPolicyWithTarget(1109).getWorker(getWorkerOptions)); - Optional address = buildPolicyWithTarget(1109).getWorker(getWorkerOptions); - Assert.assertTrue(address.isPresent()); - Assert.assertNotEquals(netAddress1, address.get()); - } - - @Test - public void getWorkerSameCapacity() { - GetWorkerOptions getWorkerOptions = GetWorkerOptions.defaults(); - ArrayList blockWorkerInfos = new ArrayList<>(); - WorkerNetAddress netAddress1 = new WorkerNetAddress().setHost("1"); - WorkerNetAddress netAddress2 = new WorkerNetAddress().setHost("2"); - WorkerNetAddress netAddress3 = new WorkerNetAddress().setHost("3"); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress1, 100, 0)); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress2, 100, 0)); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress3, 100, 0)); - getWorkerOptions.setBlockWorkerInfos(blockWorkerInfos); - Assert.assertEquals(Optional.of(netAddress1), - buildPolicyWithTarget(0).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress1), - buildPolicyWithTarget(7).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress1), - buildPolicyWithTarget(99).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress2), - buildPolicyWithTarget(100).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress2), - buildPolicyWithTarget(156).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress2), - buildPolicyWithTarget(199).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress3), - buildPolicyWithTarget(200).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress3), - buildPolicyWithTarget(211).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.of(netAddress3), - buildPolicyWithTarget(299).getWorker(getWorkerOptions)); - Optional address = buildPolicyWithTarget(299).getWorker(getWorkerOptions); - Assert.assertTrue(address.isPresent()); - Assert.assertNotEquals(netAddress1, address.get()); - } - - @Test - public void testNoMatchWorker() { - GetWorkerOptions getWorkerOptions = GetWorkerOptions.defaults(); - ArrayList blockWorkerInfos = new ArrayList<>(); - WorkerNetAddress netAddress1 = new WorkerNetAddress(); - WorkerNetAddress netAddress2 = new WorkerNetAddress(); - WorkerNetAddress netAddress3 = new WorkerNetAddress(); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress1, 0, 0)); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress2, 0, 0)); - blockWorkerInfos.add(new BlockWorkerInfo(netAddress3, 0, 0)); - getWorkerOptions.setBlockWorkerInfos(blockWorkerInfos); - Assert.assertEquals(Optional.empty(), buildPolicyWithTarget(0).getWorker(getWorkerOptions)); - Assert.assertEquals(Optional.empty(), buildPolicyWithTarget(1009).getWorker(getWorkerOptions)); - } - - /** - * @param targetValue must be in [0,totalCapacity) - */ - private CapacityBaseRandomPolicy buildPolicyWithTarget(final int targetValue) { - return new CapacityBaseRandomPolicy(Configuration.global()) { - @Override - protected long randomInCapacity(long totalCapacity) { - return targetValue; - } - }; - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/policy/DeterministicHashPolicyTest.java b/core/client/fs/src/test/java/alluxio/client/block/policy/DeterministicHashPolicyTest.java deleted file mode 100644 index 93b1fd6824df..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/policy/DeterministicHashPolicyTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.wire.BlockInfo; -import alluxio.wire.WorkerNetAddress; - -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Tests {@link DeterministicHashPolicy}. - */ -public final class DeterministicHashPolicyTest { - private static final int PORT = 1; - - private final List mWorkerInfos = new ArrayList<>(); - private static InstancedConfiguration sConf = Configuration.copyGlobal(); - - @Before - public void before() { - mWorkerInfos.clear(); - mWorkerInfos.add(new BlockWorkerInfo( - new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT) - .setWebPort(PORT), Constants.GB, 0)); - mWorkerInfos.add(new BlockWorkerInfo( - new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT) - .setWebPort(PORT), 2 * (long) Constants.GB, 0)); - mWorkerInfos.add(new BlockWorkerInfo( - new WorkerNetAddress().setHost("worker3").setRpcPort(PORT).setDataPort(PORT) - .setWebPort(PORT), 3 * (long) Constants.GB, 0)); - mWorkerInfos.add(new BlockWorkerInfo( - new WorkerNetAddress().setHost("worker4").setRpcPort(PORT).setDataPort(PORT) - .setWebPort(PORT), 3 * (long) Constants.GB, 0)); - } - - @Test - public void getWorkerDeterministically() { - DeterministicHashPolicy policy = (DeterministicHashPolicy) BlockLocationPolicy.Factory.create( - DeterministicHashPolicy.class, sConf); - String host = policy.getWorker(GetWorkerOptions.defaults().setBlockWorkerInfos(mWorkerInfos) - .setBlockInfo(new BlockInfo().setBlockId(1).setLength(2 * (long) Constants.GB))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost(); - for (int i = 0; i < 10; i++) { - DeterministicHashPolicy p = (DeterministicHashPolicy) BlockLocationPolicy.Factory.create( - DeterministicHashPolicy.class, - sConf); - // For the same block, always return the same worker. - assertEquals(host, p.getWorker( - GetWorkerOptions.defaults().setBlockWorkerInfos(mWorkerInfos) - .setBlockInfo(new BlockInfo().setBlockId(1).setLength(2 * (long) Constants.GB))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - assertEquals(host, p.getWorker( - GetWorkerOptions.defaults().setBlockWorkerInfos(mWorkerInfos) - .setBlockInfo(new BlockInfo().setBlockId(1).setLength(2 * (long) Constants.GB))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - } - } - - @Test - public void getWorkerEnoughCapacity() { - DeterministicHashPolicy policy = (DeterministicHashPolicy) BlockLocationPolicy.Factory.create( - DeterministicHashPolicy.class, sConf); - for (long blockId = 0; blockId < 100; blockId++) { - // worker1 does not have enough capacity. It should never be picked. - assertNotEquals("worker1", policy.getWorker( - GetWorkerOptions.defaults().setBlockWorkerInfos(mWorkerInfos) - .setBlockInfo(new BlockInfo().setBlockId(blockId) - .setLength(2 * (long) Constants.GB))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - } - } - - @Test - public void getWorkerMultipleShards() { - sConf.set(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY_DETERMINISTIC_HASH_SHARDS, 2); - DeterministicHashPolicy policy2 = (DeterministicHashPolicy) BlockLocationPolicy.Factory.create( - DeterministicHashPolicy.class, sConf); - Set addresses1 = new HashSet<>(); - Set addresses2 = new HashSet<>(); - for (int i = 0; i < 100; i++) { - addresses1.add(policy2.getWorker( - GetWorkerOptions.defaults().setBlockWorkerInfos(mWorkerInfos) - .setBlockInfo(new BlockInfo().setBlockId(1) - .setLength(2 * (long) Constants.GB))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - addresses2.add(policy2.getWorker( - GetWorkerOptions.defaults().setBlockWorkerInfos(mWorkerInfos) - .setBlockInfo(new BlockInfo().setBlockId(1) - .setLength(2 * (long) Constants.GB))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - } - // With sufficient traffic, 2 (= #shards) workers should be picked to serve the block. - assertEquals(2, addresses1.size()); - assertEquals(2, addresses2.size()); - assertEquals(addresses1, addresses2); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/policy/LocalFirstAvoidEvictionPolicyTest.java b/core/client/fs/src/test/java/alluxio/client/block/policy/LocalFirstAvoidEvictionPolicyTest.java deleted file mode 100644 index 0b6176c377cf..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/policy/LocalFirstAvoidEvictionPolicyTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import static alluxio.client.util.ClientTestUtils.worker; -import static org.junit.Assert.assertEquals; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.network.TieredIdentityFactory; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.testing.EqualsTester; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * Tests {@link LocalFirstAvoidEvictionPolicy}. The class delegates to {@link LocalFirstPolicy}, so - * most of its functionality is tested in {@link LocalFirstPolicyTest}. - */ -public class LocalFirstAvoidEvictionPolicyTest { - - private final AlluxioConfiguration mConf = Configuration.global(); - - @Test - public void chooseClosestTierAvoidEviction() throws Exception { - List workers = new ArrayList<>(); - workers.add(worker(Constants.GB, Constants.MB, "node2", "rack3")); - workers.add(worker(Constants.GB, 0, "node3", "rack2")); - workers.add(worker(Constants.GB, 0, "node4", "rack3")); - BlockLocationPolicy policy; - WorkerNetAddress chosen; - // local rack with enough availability - policy = new LocalFirstAvoidEvictionPolicy( - mConf.getBytes(PropertyKey.USER_BLOCK_AVOID_EVICTION_POLICY_RESERVED_BYTES), - TieredIdentityFactory.fromString("node=node2,rack=rack3", mConf), mConf); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.GB)); - chosen = policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")); - assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue()); - } - - /** - * Tests that another worker is picked in case the local host does not have enough availability. - */ - @Test - public void getOthersWhenNotEnoughAvailabilityOnLocal() { - String localhostName = NetworkAddressUtils.getLocalHostName(1000); - BlockLocationPolicy policy = new LocalFirstAvoidEvictionPolicy(mConf); - List workers = new ArrayList<>(); - workers.add(worker(Constants.GB, 0, "worker1", "")); - workers.add(worker(Constants.MB, Constants.MB, localhostName, "")); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.MB)); - assertEquals("worker1", policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")) - .getHost()); - } - - /** - * Tests that local host is picked if none of the workers has enough availability. - */ - @Test - public void getLocalWhenNoneHasAvailability() { - String localhostName = NetworkAddressUtils.getLocalHostName(1000); - BlockLocationPolicy policy = new LocalFirstAvoidEvictionPolicy(mConf); - List workers = new ArrayList<>(); - workers.add(worker(Constants.GB, Constants.MB, "worker1", "")); - workers.add(worker(Constants.GB, Constants.MB, localhostName, "")); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.GB)); - assertEquals(localhostName, policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")) - .getHost()); - } - - @Test - public void equalsTest() { - new EqualsTester() - .addEqualityGroup( - new LocalFirstAvoidEvictionPolicy(mConf), - new LocalFirstAvoidEvictionPolicy(mConf)) - .testEquals(); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/policy/LocalFirstPolicyTest.java b/core/client/fs/src/test/java/alluxio/client/block/policy/LocalFirstPolicyTest.java deleted file mode 100644 index 759f4bc6f389..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/policy/LocalFirstPolicyTest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import static alluxio.client.util.ClientTestUtils.worker; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.network.TieredIdentityFactory; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.testing.EqualsTester; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * Tests {@link LocalFirstPolicy}. - */ -public final class LocalFirstPolicyTest { - - private static final AlluxioConfiguration S_CONF = Configuration.global(); - private static final int S_RESOLUTION_TIMEOUT = - (int) Configuration.getMs(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS); - - /** - * Tests that the local host is returned first. - */ - @Test - public void getLocalFirst() { - String localhostName = - NetworkAddressUtils.getLocalHostName(S_RESOLUTION_TIMEOUT); - LocalFirstPolicy policy = new LocalFirstPolicy(S_CONF); - List workers = new ArrayList<>(); - workers.add(worker(Constants.GB, "worker1", "")); - workers.add(worker(Constants.GB, localhostName, "")); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.MB)); - assertEquals(localhostName, policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - } - - /** - * Tests that another worker is picked in case the local host does not have enough capacity. - */ - @Test - public void getOthersWhenNotEnoughCapacityOnLocal() { - String localhostName = NetworkAddressUtils.getLocalHostName(S_RESOLUTION_TIMEOUT); - LocalFirstPolicy policy = new LocalFirstPolicy(S_CONF); - List workers = new ArrayList<>(); - workers.add(worker(Constants.GB, "worker1", "")); - workers.add(worker(Constants.MB, localhostName, "")); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.GB)); - assertEquals("worker1", policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - } - - /** - * Tests that non-local workers are randomly selected. - */ - @Test - public void getOthersRandomly() { - LocalFirstPolicy policy = new LocalFirstPolicy(S_CONF); - List workers = new ArrayList<>(); - workers.add(worker(Constants.GB, "worker1", "")); - workers.add(worker(Constants.GB, "worker2", "")); - - boolean success = false; - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.MB)); - for (int i = 0; i < 100; i++) { - String host = policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost(); - if (!host.equals(policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost())) { - success = true; - break; - } - } - assertTrue(success); - } - - @Test - public void chooseClosestTier() throws Exception { - List workers = new ArrayList<>(); - workers.add(worker(Constants.GB, "node2", "rack3")); - workers.add(worker(Constants.GB, "node3", "rack2")); - workers.add(worker(Constants.GB, "node4", "rack3")); - LocalFirstPolicy policy; - WorkerNetAddress chosen; - // local rack - policy = new LocalFirstPolicy(TieredIdentityFactory.fromString("node=node1,rack=rack2", - S_CONF), S_CONF); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.GB)); - chosen = policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")); - assertEquals("rack2", chosen.getTieredIdentity().getTier(1).getValue()); - - // local node - policy = new LocalFirstPolicy(TieredIdentityFactory.fromString("node=node4,rack=rack3", - S_CONF), - S_CONF); - chosen = policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")); - assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue()); - } - - @Test - public void tieredLocalityEnoughSpace() throws Exception { - List workers = new ArrayList<>(); - // Local node doesn't have enough space - workers.add(worker(Constants.MB, "node2", "rack3")); - workers.add(worker(Constants.GB, "node3", "rack2")); - // Local rack has enough space - workers.add(worker(Constants.GB, "node4", "rack3")); - LocalFirstPolicy policy = new LocalFirstPolicy(TieredIdentityFactory - .fromString("node=node2,rack=rack3", S_CONF), S_CONF); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.GB)); - WorkerNetAddress chosen = policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")); - assertEquals(workers.get(2).getNetAddress(), chosen); - } - - @Test - public void equalsTest() throws Exception { - new EqualsTester() - .addEqualityGroup(new LocalFirstPolicy(TieredIdentityFactory.fromString("node=x,rack=y", - S_CONF), S_CONF)) - .addEqualityGroup(new LocalFirstPolicy(TieredIdentityFactory.fromString("node=x,rack=z", - S_CONF), S_CONF)) - .testEquals(); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/policy/MostAvailableFirstPolicyTest.java b/core/client/fs/src/test/java/alluxio/client/block/policy/MostAvailableFirstPolicyTest.java deleted file mode 100644 index e4bb60e97814..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/policy/MostAvailableFirstPolicyTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.test.util.CommonUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.WorkerNetAddress; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * Tests {@link MostAvailableFirstPolicy}. - */ -public final class MostAvailableFirstPolicyTest { - private static final int PORT = 1; - - /** - * Tests that the worker with the most available space is chosen. - */ - @Test - public void getMostAvailableWorker() { - List workerInfoList = new ArrayList<>(); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0)); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0)); - MostAvailableFirstPolicy policy = new MostAvailableFirstPolicy(null); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workerInfoList).setBlockInfo(new BlockInfo().setLength(Constants.MB)); - Assert.assertEquals("worker3", - policy.getWorker(options).orElseThrow( - () -> new IllegalStateException("Expected worker3")).getHost()); - } - - @Test - public void equalsTest() { - CommonUtils.testEquals(MostAvailableFirstPolicy.class, - new Class[]{AlluxioConfiguration.class}, - new Object[]{Configuration.global()}); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/policy/RoundRobinPolicyTest.java b/core/client/fs/src/test/java/alluxio/client/block/policy/RoundRobinPolicyTest.java deleted file mode 100644 index 0840355b2e67..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/policy/RoundRobinPolicyTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.test.util.CommonUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.WorkerNetAddress; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * Tests {@link RoundRobinPolicy}. - */ -public final class RoundRobinPolicyTest { - private static final int PORT = 1; - - /** - * Tests that the correct workers are chosen when round-robin is used. - */ - @Test - public void getWorker() { - List workerInfoList = new ArrayList<>(); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0)); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0)); - RoundRobinPolicy policy = new RoundRobinPolicy(Configuration.global()); - - GetWorkerOptions options = GetWorkerOptions.defaults().setBlockWorkerInfos(workerInfoList) - .setBlockInfo(new BlockInfo().setLength(2 * (long) Constants.GB)); - assertNotEquals( - policy.getWorker(options).orElseThrow( - () -> new IllegalStateException("Expected worker")).getHost(), - policy.getWorker(options.setBlockInfo(options.getBlockInfo().setBlockId(123))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - - assertEquals( - policy.getWorker(options.setBlockInfo(options.getBlockInfo().setBlockId(555))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost(), - policy.getWorker(options.setBlockInfo(options.getBlockInfo().setBlockId(555))) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - } - - /** - * Tests that no workers are returned when there are no eligible workers. - */ - @Test - public void getWorkerNoneEligible() { - RoundRobinPolicy policy = new RoundRobinPolicy(Configuration.global()); - GetWorkerOptions options = GetWorkerOptions.defaults().setBlockWorkerInfos(new ArrayList<>()) - .setBlockInfo(new BlockInfo().setLength(2 * (long) Constants.GB)); - assertFalse(policy.getWorker(options).isPresent()); - } - - /** - * Tests that no workers are returned when subsequent calls to the policy have no eligible - * workers. - */ - @Test - public void getWorkerNoneEligibleAfterCache() { - List workerInfoList = new ArrayList<>(); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); - - RoundRobinPolicy policy = new RoundRobinPolicy(Configuration.global()); - GetWorkerOptions options = GetWorkerOptions.defaults().setBlockWorkerInfos(workerInfoList) - .setBlockInfo(new BlockInfo().setLength(Constants.MB)); - assertTrue(policy.getWorker(options).isPresent()); - options.setBlockWorkerInfos(new ArrayList<>()); - assertFalse(policy.getWorker(options).isPresent()); - } - - @Test - public void equalsTest() { - AlluxioConfiguration conf = Configuration.global(); - CommonUtils.testEquals(RoundRobinPolicy.class, new Class[]{AlluxioConfiguration.class}, - new Object[]{conf}); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/policy/SpecificHostPolicyTest.java b/core/client/fs/src/test/java/alluxio/client/block/policy/SpecificHostPolicyTest.java deleted file mode 100644 index d63c4aa99fa4..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/policy/SpecificHostPolicyTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.options.GetWorkerOptions; -import alluxio.wire.BlockInfo; -import alluxio.wire.WorkerNetAddress; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * Tests {@link SpecificHostPolicy}. - */ -public final class SpecificHostPolicyTest { - private static final int PORT = 1; - - /** - * Tests that the correct worker is returned when using the policy. - */ - @Test - public void policy() { - SpecificHostPolicy policy = new SpecificHostPolicy("worker2"); - List workerInfoList = new ArrayList<>(); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); - GetWorkerOptions options = GetWorkerOptions.defaults() - .setBlockWorkerInfos(workerInfoList).setBlockInfo(new BlockInfo().setLength(Constants.MB)); - Assert.assertEquals("worker2", - policy.getWorker(options) - .orElseThrow(() -> new IllegalStateException("Expected worker")).getHost()); - } - - /** - * Tests that no worker is chosen when the worker specified in the policy is not part of the - * worker list. - */ - @Test - public void noMatchingHost() { - SpecificHostPolicy policy = new SpecificHostPolicy("worker3"); - List workerInfoList = new ArrayList<>(); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1F") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); - workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") - .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); - GetWorkerOptions options = GetWorkerOptions.defaults().setBlockWorkerInfos(workerInfoList) - .setBlockInfo(new BlockInfo().setLength(2 * (long) Constants.GB)); - Assert.assertFalse(policy.getWorker(options).isPresent()); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/policy/options/GetWorkerOptionsTest.java b/core/client/fs/src/test/java/alluxio/client/block/policy/options/GetWorkerOptionsTest.java deleted file mode 100644 index e26e582b01bb..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/policy/options/GetWorkerOptionsTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.policy.options; - -import static org.junit.Assert.assertEquals; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.test.util.CommonUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.WorkerNetAddress; - -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -/** - * Tests for the {@link GetWorkerOptions} class. - */ -public final class GetWorkerOptionsTest { - private static final int PORT = 1; - - private final List mWorkerInfos = new ArrayList<>(); - - /** - * Tests for defaults {@link GetWorkerOptions}. - */ - @Test - public void defaults() throws IOException { - GetWorkerOptions options = GetWorkerOptions.defaults().setBlockInfo(new BlockInfo()); - assertEquals(null, options.getBlockWorkerInfos()); - assertEquals(0, options.getBlockInfo().getBlockId()); - assertEquals(0, options.getBlockInfo().getLength()); - } - - /** - * Tests for setBlockWorkerInfo. - */ - @Test - public void setBlockWorkerInfoTest() { - mWorkerInfos.clear(); - mWorkerInfos.add(new BlockWorkerInfo( - new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT) - .setWebPort(PORT), Constants.GB, 0)); - mWorkerInfos.add(new BlockWorkerInfo( - new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT) - .setWebPort(PORT), 2 * (long) Constants.GB, 0)); - GetWorkerOptions options = GetWorkerOptions.defaults(); - options.setBlockWorkerInfos(mWorkerInfos); - assertEquals(mWorkerInfos, options.getBlockWorkerInfos()); - } - - /** - * Tests for setBlockId and setBlockSize. - */ - @Test - public void fields() { - Random rand = new Random(); - long blockId = rand.nextLong(); - long blockSize = rand.nextLong(); - GetWorkerOptions options = GetWorkerOptions.defaults(); - BlockInfo info = new BlockInfo().setBlockId(blockId).setLength(blockSize); - options.setBlockInfo(info); - assertEquals(info, options.getBlockInfo()); - } - - @Test - public void equalTest() throws Exception { - CommonUtils.testEquals(GetWorkerOptions.class); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/stream/BlockInStreamTest.java b/core/client/fs/src/test/java/alluxio/client/block/stream/BlockInStreamTest.java deleted file mode 100644 index 787b216cca2e..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/stream/BlockInStreamTest.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.when; - -import alluxio.ClientContext; -import alluxio.ConfigurationRule; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.URIStatus; -import alluxio.client.file.options.InStreamOptions; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.OpenLocalBlockRequest; -import alluxio.grpc.OpenLocalBlockResponse; -import alluxio.util.io.BufferUtils; -import alluxio.util.network.NettyUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.FileInfo; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.BlockWorker; - -import io.grpc.stub.ClientCallStreamObserver; -import io.grpc.stub.StreamObserver; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; -import org.mockito.Mockito; -import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.Closeable; -import java.util.Collections; -import java.util.Optional; - -/** - * Tests the {@link BlockInStream} class's static methods. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({NettyUtils.class}) -public class BlockInStreamTest { - private FileSystemContext mMockContext; - private BlockInfo mInfo; - private InStreamOptions mOptions; - private final InstancedConfiguration mConf = Configuration.copyGlobal(); - private StreamObserver mResponseObserver; - - @Before - public void before() throws Exception { - BlockWorkerClient workerClient = Mockito.mock(BlockWorkerClient.class); - ClientCallStreamObserver requestObserver = Mockito.mock(ClientCallStreamObserver.class); - when(requestObserver.isReady()).thenReturn(true); - when(workerClient.openLocalBlock(any(StreamObserver.class))) - .thenAnswer((Answer) invocation -> { - mResponseObserver = invocation.getArgument(0, StreamObserver.class); - return requestObserver; - }); - doAnswer(invocation -> { - mResponseObserver.onNext(OpenLocalBlockResponse.newBuilder().setPath("/tmp").build()); - mResponseObserver.onCompleted(); - return null; - }).when(requestObserver).onNext(any(OpenLocalBlockRequest.class)); - mMockContext = Mockito.mock(FileSystemContext.class); - when(mMockContext.acquireBlockWorkerClient(ArgumentMatchers.any(WorkerNetAddress.class))) - .thenReturn(new NoopClosableResource<>(workerClient)); - when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf)); - when(mMockContext.getClusterConf()).thenReturn(mConf); - mInfo = new BlockInfo().setBlockId(1); - mOptions = new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Collections - .singletonList(1L))), mConf, mMockContext); - } - - @Test - public void closeReaderAfterReadingAllData() throws Exception { - int chunkSize = 512; - TestDataReader.Factory factory = new TestDataReader.Factory( - chunkSize, BufferUtils.getIncreasingByteArray(2 * chunkSize)); - BlockInStream stream = new BlockInStream(factory, new WorkerNetAddress(), - BlockInStream.BlockInStreamSource.PROCESS_LOCAL, -1, 1024); - - byte[] res = new byte[chunkSize]; - int read; - read = stream.read(res, 0, chunkSize); - TestDataReader reader = factory.getDataReader(); - assertEquals(chunkSize, read); - assertNotNull(reader); - assertFalse(reader.isClosed()); - - // close data reader after reading all data - read = stream.read(res, 0, chunkSize); - assertEquals(chunkSize, read); - assertTrue(reader.isClosed()); - - read = stream.read(res, 0, chunkSize); - assertEquals(-1, read); - assertTrue(reader.isClosed()); - - stream.close(); - assertTrue(reader.isClosed()); - } - - @Test - public void createShortCircuit() throws Exception { - WorkerNetAddress dataSource = new WorkerNetAddress(); - BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.NODE_LOCAL; - BlockInStream stream = - BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); - assertEquals(LocalFileDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } - - @Test - public void createRemote() throws Exception { - WorkerNetAddress dataSource = new WorkerNetAddress(); - BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.REMOTE; - BlockInStream stream = - BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); - assertEquals(GrpcDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } - - @Test - public void createUfs() throws Exception { - WorkerNetAddress dataSource = new WorkerNetAddress(); - BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.UFS; - BlockInStream stream = - BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); - assertEquals(GrpcDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } - - @Test - public void createShortCircuitDisabled() throws Exception { - try (Closeable ignored = - new ConfigurationRule(PropertyKey.USER_SHORT_CIRCUIT_ENABLED, false, mConf) - .toResource()) { - WorkerNetAddress dataSource = new WorkerNetAddress(); - when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf)); - BlockInStream.BlockInStreamSource dataSourceType = - BlockInStream.BlockInStreamSource.NODE_LOCAL; - BlockInStream stream = - BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); - assertEquals(GrpcDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } - } - - @Test - public void createDomainSocketEnabled() throws Exception { - PowerMockito.mockStatic(NettyUtils.class); - PowerMockito.when( - NettyUtils.isDomainSocketAccessible(ArgumentMatchers.any(WorkerNetAddress.class), - ArgumentMatchers.any(InstancedConfiguration.class))) - .thenReturn(true); - PowerMockito.when( - NettyUtils.isDomainSocketSupported(ArgumentMatchers.any(WorkerNetAddress.class))) - .thenReturn(true); - WorkerNetAddress dataSource = new WorkerNetAddress(); - BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.NODE_LOCAL; - BlockInStream stream = BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, - mOptions); - assertEquals(GrpcDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } - - @Test - public void createProcessLocal() throws Exception { - WorkerNetAddress dataSource = new WorkerNetAddress(); - when(mMockContext.getNodeLocalWorker()).thenReturn(dataSource); - when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf)); - Optional blockWorker = Optional.of(Mockito.mock(BlockWorker.class)); - when(mMockContext.getProcessLocalWorker()).thenReturn(blockWorker); - BlockInStream.BlockInStreamSource dataSourceType = - BlockInStream.BlockInStreamSource.PROCESS_LOCAL; - BlockInStream stream = - BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); - assertEquals(BlockWorkerDataReader.Factory.class.getName(), - stream.getDataReaderFactory().getClass().getName()); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/stream/BlockOutStreamTest.java b/core/client/fs/src/test/java/alluxio/client/block/stream/BlockOutStreamTest.java deleted file mode 100644 index eb66afd1f636..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/stream/BlockOutStreamTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.wire.WorkerNetAddress; - -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * Tests {@link BlockOutStream}. - */ -public class BlockOutStreamTest { - private static final int CHUNK_SIZE = 128; - - @Test - public void packetWriteException() throws Exception { - DataWriter writer = new FailingTestDataWriter(ByteBuffer.allocate(CHUNK_SIZE)); - BlockOutStream bos = new BlockOutStream(writer, CHUNK_SIZE, new WorkerNetAddress()); - try { - bos.write(new byte[CHUNK_SIZE]); - Assert.fail("Expected write to throw an exception."); - } catch (IOException e) { - // Exception expected, continue. - } - // After an exception, we should still be able to cancel the stream. - // Test succeeds if we do not throw an exception in cancel. - bos.cancel(); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/stream/LocalFileDataWriterTest.java b/core/client/fs/src/test/java/alluxio/client/block/stream/LocalFileDataWriterTest.java deleted file mode 100644 index 0848036bc789..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/stream/LocalFileDataWriterTest.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import static org.mockito.Mockito.mock; - -import alluxio.AlluxioTestDirectory; -import alluxio.ClientContext; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.conf.Configuration; -import alluxio.grpc.CreateLocalBlockRequest; -import alluxio.grpc.CreateLocalBlockResponse; -import alluxio.util.IdUtils; -import alluxio.util.io.PathUtils; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.io.LocalFileBlockWriter; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; -import org.powermock.reflect.Whitebox; - -/** - * Tests {@link LocalFileDataWriterTest}. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({FileSystemContext.class, WorkerNetAddress.class, LocalFileDataWriter.class, - GrpcBlockingStream.class}) -public class LocalFileDataWriterTest { - private static final long BLOCK_ID = 1L; - - protected String mWorkDirectory; - - private WorkerNetAddress mAddress; - private BlockWorkerClient mClient; - private ClientContext mClientContext; - private FileSystemContext mContext; - private GrpcBlockingStream mStream; - - @Before - public void before() throws Exception { - mWorkDirectory = - AlluxioTestDirectory.createTemporaryDirectory("blocks").getAbsolutePath(); - - mClientContext = ClientContext.create(Configuration.global()); - - mContext = PowerMockito.mock(FileSystemContext.class); - mAddress = mock(WorkerNetAddress.class); - - mClient = mock(BlockWorkerClient.class); - PowerMockito.when(mContext.acquireBlockWorkerClient(mAddress)).thenReturn( - new NoopClosableResource<>(mClient)); - PowerMockito.when(mContext.getClientContext()).thenReturn(mClientContext); - PowerMockito.when(mContext.getClusterConf()).thenReturn(Configuration.global()); - - mStream = mock(GrpcBlockingStream.class); - PowerMockito.doNothing().when(mStream).send(ArgumentMatchers.any(), ArgumentMatchers.anyLong()); - PowerMockito.when(mStream.receive(ArgumentMatchers.anyLong())) - .thenReturn(CreateLocalBlockResponse.newBuilder() - .setPath(PathUtils.temporaryFileName(IdUtils.getRandomNonNegativeLong(), - PathUtils.concatPath(mWorkDirectory, BLOCK_ID))) - .build()); - PowerMockito.when(mStream.isCanceled()).thenReturn(false); - PowerMockito.when(mStream.isClosed()).thenReturn(false); - PowerMockito.when(mStream.isOpen()).thenReturn(true); - - PowerMockito.whenNew(GrpcBlockingStream.class).withAnyArguments().thenReturn(mStream); - } - - @After - public void after() throws Exception { - mClient.close(); - } - - @Test - public void streamCancelled() throws Exception { - LocalFileDataWriter writer = LocalFileDataWriter.create(mContext, mAddress, BLOCK_ID, - 128 /* unused */, OutStreamOptions.defaults(mContext)); - - // Cancel stream before cancelling the writer - PowerMockito.when(mStream.isCanceled()).thenReturn(true); - PowerMockito.when(mStream.isClosed()).thenReturn(true); - PowerMockito.when(mStream.isOpen()).thenReturn(true); - - writer.cancel(); - - // Verify there are no open files - LocalFileBlockWriter blockWriter = Whitebox.getInternalState(writer, "mWriter"); - Assert.assertTrue(Whitebox.getInternalState(blockWriter, "mClosed")); - } - - @Test - public void streamClosed() throws Exception { - LocalFileDataWriter writer = LocalFileDataWriter.create(mContext, mAddress, BLOCK_ID, - 128 /* unused */, OutStreamOptions.defaults(mContext)); - - // Close stream before closing the writer - PowerMockito.when(mStream.isCanceled()).thenReturn(true); - PowerMockito.when(mStream.isClosed()).thenReturn(true); - PowerMockito.when(mStream.isOpen()).thenReturn(true); - - writer.close(); - - // Verify there are no open files - LocalFileBlockWriter blockWriter = Whitebox.getInternalState(writer, "mWriter"); - Assert.assertTrue(Whitebox.getInternalState(blockWriter, "mClosed")); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/stream/TestBlockInStream.java b/core/client/fs/src/test/java/alluxio/client/block/stream/TestBlockInStream.java deleted file mode 100644 index 842f532ffca2..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/stream/TestBlockInStream.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.wire.WorkerNetAddress; - -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * A {@link BlockInStream} which reads from the given byte array. The stream is able to track how - * much bytes that have been read from the extended BlockInStream. - */ -public class TestBlockInStream extends BlockInStream { - /** A field tracks how much bytes read. */ - private int mBytesRead; - private boolean mClosed; - - public TestBlockInStream(byte[] data, long id, long length, - BlockInStreamSource source) { - super(new Factory(data), - new WorkerNetAddress(), source, id, length); - mBytesRead = 0; - } - - @Override - public int read(ByteBuffer byteBuffer, int off, int len) throws IOException { - int bytesRead = super.read(byteBuffer, off, len); - if (bytesRead <= 0) { - return bytesRead; - } - mBytesRead += bytesRead; - return bytesRead; - } - - @Override - public int positionedRead(long pos, byte[] b, int off, int len) throws IOException { - int bytesRead = super.positionedRead(pos, b, off, len); - if (bytesRead <= 0) { - return bytesRead; - } - mBytesRead += bytesRead; - return bytesRead; - } - - public boolean isClosed() { - return mClosed; - } - - @Override - public void close() throws IOException { - mClosed = true; - super.close(); - } - - /** - * @return how many bytes been read - */ - public int getBytesRead() { - return mBytesRead; - } - - /** - * Factory class to create {@link TestDataReader}s. - */ - public static class Factory implements DataReader.Factory { - private final byte[] mData; - - /** - * Creates an instance of {@link LocalFileDataReader.Factory}. - * - * @param data the data to serve - */ - public Factory(byte[] data) { - mData = data; - } - - @Override - public DataReader create(long offset, long len) { - return new TestDataReader(mData, 128, offset, len); - } - - @Override - public void close() {} - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/stream/TestBlockOutStream.java b/core/client/fs/src/test/java/alluxio/client/block/stream/TestBlockOutStream.java deleted file mode 100644 index 6f3b05249591..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/stream/TestBlockOutStream.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.wire.WorkerNetAddress; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; - -/** - * Test class for mocking {@link BlockOutStream} and exposing internal state. - */ -public class TestBlockOutStream extends BlockOutStream { - private final ByteBuffer mData; - private boolean mClosed; - private boolean mCanceled; - - /** - * Constructs a new {@link TestBlockOutStream} to be used in tests. - * - * @param data the data to test - * @param blockSize the block size - */ - public TestBlockOutStream(ByteBuffer data, long blockSize) { - super(new TestDataWriter(data), blockSize, new WorkerNetAddress()); - mData = data; - mClosed = false; - mCanceled = false; - } - - public byte[] getWrittenData() { - try { - super.flush(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return Arrays.copyOfRange(mData.array(), 0, mData.position()); - } - - public boolean isClosed() { - return mClosed; - } - - public boolean isCanceled() { - return mCanceled; - } - - @Override - public void close() throws IOException { - super.close(); - mClosed = true; - } - - @Override - public void cancel() throws IOException { - if (mClosed) { - return; - } - super.cancel(); - mCanceled = true; - mClosed = true; - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/stream/TestUnderFileSystemFileOutStream.java b/core/client/fs/src/test/java/alluxio/client/block/stream/TestUnderFileSystemFileOutStream.java deleted file mode 100644 index eb55e8623dd1..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/stream/TestUnderFileSystemFileOutStream.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.stream; - -import alluxio.wire.WorkerNetAddress; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; - -/** - * Test class for mocking {@link UnderFileSystemFileOutStream} and exposing internal state. - */ -public class TestUnderFileSystemFileOutStream extends UnderFileSystemFileOutStream { - private final ByteBuffer mData; - private boolean mClosed; - private boolean mCanceled; - - /** - * Constructs a new {@link TestUnderFileSystemFileOutStream} to be used in tests. - * - * @param data the data to test - */ - public TestUnderFileSystemFileOutStream(ByteBuffer data) { - super(new TestDataWriter(data), new WorkerNetAddress()); - mData = data; - mClosed = false; - mCanceled = false; - } - - public byte[] getWrittenData() { - try { - super.flush(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return Arrays.copyOfRange(mData.array(), 0, mData.position()); - } - - public boolean isClosed() { - return mClosed; - } - - public boolean isCanceled() { - return mCanceled; - } - - @Override - public void close() throws IOException { - super.close(); - mClosed = true; - } - - @Override - public void cancel() throws IOException { - if (mClosed) { - return; - } - super.cancel(); - mCanceled = true; - mClosed = true; - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/block/util/BlockLocationUtilsTest.java b/core/client/fs/src/test/java/alluxio/client/block/util/BlockLocationUtilsTest.java deleted file mode 100644 index 4d1f8d282343..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/block/util/BlockLocationUtilsTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.block.util; - -import static alluxio.client.util.ClientTestUtils.worker; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.collections.Pair; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.network.TieredIdentityFactory; -import alluxio.util.network.NettyUtils; -import alluxio.wire.WorkerNetAddress; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * Tests {@link BlockLocationUtils}. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest(NettyUtils.class) -public final class BlockLocationUtilsTest { - - @Test - public void chooseLocalAccessibleDomainSocket() throws Exception { - List workers = new ArrayList<>(); - workers.add(worker(Constants.GB, "node2", "rack2")); - // create worker info with domain socket path - BlockWorkerInfo workerWithDomainSocket = worker(Constants.GB, "node3", "rack3"); - String domainSocketPath = "/tmp/domain/uuid-node3"; - workerWithDomainSocket.getNetAddress().setDomainSocketPath(domainSocketPath); - workers.add(workerWithDomainSocket); - - // mock NettyUtils - PowerMockito.mockStatic(NettyUtils.class); - when(NettyUtils.isDomainSocketAccessible(eq(workerWithDomainSocket.getNetAddress()), - any(AlluxioConfiguration.class))).thenReturn(true); - - // choose worker with domain socket accessible ignoring rack - InstancedConfiguration conf = Configuration.copyGlobal(); - conf.set(PropertyKey.WORKER_DATA_SERVER_DOMAIN_SOCKET_AS_UUID, true); - List addresses = workers.stream() - .map(worker -> worker.getNetAddress()) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - Optional> chosen = BlockLocationUtils - .nearest(TieredIdentityFactory.fromString("node=node1,rack=rack2", conf), addresses, conf); - assertTrue(chosen.isPresent()); - assertTrue(chosen.get().getSecond()); - assertEquals(domainSocketPath, chosen.get().getFirst().getDomainSocketPath()); - assertEquals("node3", chosen.get().getFirst().getHost()); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/AlluxioFileInStreamTest.java b/core/client/fs/src/test/java/alluxio/client/file/AlluxioFileInStreamTest.java deleted file mode 100644 index b77f28d10ed6..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/AlluxioFileInStreamTest.java +++ /dev/null @@ -1,850 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyLong; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import alluxio.AlluxioURI; -import alluxio.ClientContext; -import alluxio.client.block.BlockStoreClient; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.stream.BlockInStream; -import alluxio.client.block.stream.BlockInStream.BlockInStreamSource; -import alluxio.client.block.stream.BlockWorkerClient; -import alluxio.client.block.stream.TestBlockInStream; -import alluxio.client.file.options.InStreamOptions; -import alluxio.client.util.ClientTestUtils; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.PreconditionMessage; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.OpenFilePOptions; -import alluxio.grpc.ReadPType; -import alluxio.resource.CloseableResource; -import alluxio.util.io.BufferUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.FileBlockInfo; -import alluxio.wire.FileInfo; -import alluxio.wire.WorkerNetAddress; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; -import org.powermock.modules.junit4.PowerMockRunnerDelegate; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -/** - * Tests for the {@link AlluxioFileInStream} class. - * - * It is a parameterized test that checks different caching behaviors when the blocks are located at - * different locations. - */ -@RunWith(PowerMockRunner.class) -@PowerMockRunnerDelegate(Parameterized.class) -@PrepareForTest({BlockStoreClient.class}) -public final class AlluxioFileInStreamTest { - private static final long BLOCK_LENGTH = 100L; - private final BlockInStreamSource mBlockSource; - private final long mFileSize; - private final long mNumBlocks; - private BlockStoreClient mBlockStore; - private FileSystemContext mContext; - private FileInfo mInfo; - private URIStatus mStatus; - - private final InstancedConfiguration mConf = Configuration.copyGlobal(); - - private List mInStreams; - - private AlluxioFileInStream mTestStream; - - /** - * @return a list of all sources of where the blocks reside and file size - */ - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList(new Object[][] { - {BlockInStreamSource.PROCESS_LOCAL, 350L}, - {BlockInStreamSource.NODE_LOCAL, 350L}, - {BlockInStreamSource.UFS, 350L}, - {BlockInStreamSource.REMOTE, 350L}, - {BlockInStreamSource.REMOTE, 800L} - }); - } - - /** - * @param blockSource the source of the block to read - * @param fileSize file size in bytes - */ - public AlluxioFileInStreamTest(BlockInStreamSource blockSource, long fileSize) { - mBlockSource = blockSource; - mFileSize = fileSize; - mNumBlocks = (mFileSize - 1) / BLOCK_LENGTH + 1; - } - - private long getBlockLength(int blockIndex) { - return Math.min(mFileSize, BLOCK_LENGTH * (blockIndex + 1)) - BLOCK_LENGTH * blockIndex; - } - - /** - * Sets up the context and streams before a test runs. - */ - @Before - public void before() throws Exception { - mInfo = new FileInfo().setBlockSizeBytes(BLOCK_LENGTH).setLength(mFileSize); - - ClientTestUtils.setSmallBufferSizes(mConf); - mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MIN, "1ms"); - mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MAX, "5ms"); - mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_MAX_DURATION, "1s"); - - BlockWorkerClient client = mock(BlockWorkerClient.class); - doNothing().when(client).cache(any()); - - FileSystemMasterClient masterClient = mock(FileSystemMasterClient.class); - - mContext = mock(FileSystemContext.class); - when(mContext.getClientContext()).thenReturn(ClientContext.create(mConf)); - when(mContext.getClusterConf()).thenReturn(mConf); - when(mContext.getPathConf(any(AlluxioURI.class))).thenReturn(mConf); - when(mContext.getNodeLocalWorker()).thenReturn(new WorkerNetAddress()); - when(mContext.getCachedWorkers()).thenReturn(new ArrayList<>()); - when(mContext.acquireBlockWorkerClient(any())) - .thenReturn(new CloseableResource(client) { - @Override - public void closeResource() {} - }); - when(mContext.acquireMasterClientResource()) - .thenReturn(new CloseableResource(masterClient) { - @Override - public void closeResource() {} - }); - mBlockStore = mock(BlockStoreClient.class); - PowerMockito.mockStatic(BlockStoreClient.class); - PowerMockito.when(BlockStoreClient.create(mContext)).thenReturn(mBlockStore); - - // Set up BufferedBlockInStreams and caching streams - mInStreams = new ArrayList<>(); - List blockIds = new ArrayList<>(); - List fileBlockInfos = new ArrayList<>(); - for (int i = 0; i < mNumBlocks; i++) { - blockIds.add((long) i); - FileBlockInfo fbInfo = new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(i)); - fileBlockInfos.add(fbInfo); - final byte[] input = BufferUtils - .getIncreasingByteArray((int) (i * BLOCK_LENGTH), (int) getBlockLength(i)); - mInStreams.add(new TestBlockInStream(input, i, input.length, mBlockSource)); - when(mContext.getCachedWorkers()) - .thenReturn(Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress(), 0, 0))); - when(mBlockStore.getInStream(eq((long) i), any(InStreamOptions.class), any())) - .thenAnswer(invocation -> { - long blockId = (Long) invocation.getArguments()[0]; - return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, - blockId, input.length, mBlockSource) : mInStreams.get((int) blockId); - }); - when(mBlockStore.getInStream(eq(new BlockInfo().setBlockId(i)), any(InStreamOptions.class), - any())).thenAnswer(invocation -> { - long blockId = ((BlockInfo) invocation.getArguments()[0]).getBlockId(); - return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, - blockId, input.length, mBlockSource) : mInStreams.get((int) blockId); - }); - } - mInfo.setBlockIds(blockIds); - mInfo.setFileBlockInfos(fileBlockInfos).setReplicationMax(1); - mStatus = new URIStatus(mInfo); - - OpenFilePOptions readOptions = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, readOptions, - mConf, mContext), mContext); - } - - @After - public void after() throws Exception { - mTestStream.close(); - ClientTestUtils.resetClient(mConf); - } - - /** - * Tests that reading through the file one byte at a time will yield the correct data. - */ - @Test - public void singleByteRead() throws Exception { - for (int i = 0; i < mFileSize; i++) { - assertEquals(i & 0xff, mTestStream.read()); - } - } - - /** - * Tests that reading half of a file works. - */ - @Test - public void readHalfFile() throws Exception { - testReadBuffer((int) (mFileSize / 2)); - } - - /** - * Tests that reading a part of a file works. - */ - @Test - public void readPartialBlock() throws Exception { - testReadBuffer((int) (BLOCK_LENGTH / 2)); - } - - /** - * Tests that reading the complete block works. - */ - @Test - public void readBlock() throws Exception { - testReadBuffer((int) BLOCK_LENGTH); - } - - /** - * Tests that reading the complete block works and the BlockInStream is closed. - */ - @Test - public void readBlockStreamCloseOnEnd() throws Exception { - int dataRead = (int) BLOCK_LENGTH; - byte[] buffer = new byte[dataRead]; - mTestStream.read(buffer); - assertEquals(true, mInStreams.get(0).isClosed()); - assertArrayEquals(BufferUtils.getIncreasingByteArray(dataRead), buffer); - } - - /** - * Tests that reading the complete file works. - */ - @Test - public void readFile() throws Exception { - testReadBuffer((int) mFileSize); - } - - /** - * Tests that reading the complete file works and all streams are closed when to the end of file. - */ - @Test - public void readFileStreamCloseOnEnd() throws Exception { - int dataRead = (int) mFileSize; - byte[] buffer = new byte[dataRead]; - mTestStream.read(buffer); - - for (int i = 0; i < mNumBlocks; i++) { - assertEquals(true, mInStreams.get(i).isClosed()); - } - assertArrayEquals(BufferUtils.getIncreasingByteArray(dataRead), buffer); - } - - /** - * Tests that reading a buffer at an offset writes the bytes to the correct places. - */ - @Test - public void readOffset() throws IOException { - int offset = (int) (BLOCK_LENGTH / 3); - int len = (int) BLOCK_LENGTH; - byte[] buffer = new byte[offset + len]; - // Create expectedBuffer containing `offset` 0's followed by `len` increasing bytes - byte[] expectedBuffer = new byte[offset + len]; - System.arraycopy(BufferUtils.getIncreasingByteArray(len), 0, expectedBuffer, offset, len); - mTestStream.read(buffer, offset, len); - assertArrayEquals(expectedBuffer, buffer); - } - - /** - * Read through the file in small chunks and verify each chunk. - */ - @Test - public void readManyChunks() throws IOException { - int chunksize = 10; - // chunksize must divide FILE_LENGTH evenly for this test to work - assertEquals(0, mFileSize % chunksize); - byte[] buffer = new byte[chunksize]; - int offset = 0; - for (int i = 0; i < mFileSize / chunksize; i++) { - mTestStream.read(buffer, 0, chunksize); - assertArrayEquals(BufferUtils.getIncreasingByteArray(offset, chunksize), buffer); - offset += chunksize; - } - } - - /** - * Tests that {@link FileInStream#remaining()} is correctly updated during reads, skips, and - * seeks. - */ - @Test - public void testRemaining() throws IOException { - assumeTrue(mFileSize > 310); - assertEquals(mFileSize, mTestStream.remaining()); - mTestStream.read(); - assertEquals(mFileSize - 1, mTestStream.remaining()); - mTestStream.read(new byte[150]); - assertEquals(mFileSize - 151, mTestStream.remaining()); - mTestStream.skip(140); - assertEquals(mFileSize - 291, mTestStream.remaining()); - mTestStream.seek(310); - assertEquals(mFileSize - 310, mTestStream.remaining()); - mTestStream.seek(130); - assertEquals(mFileSize - 130, mTestStream.remaining()); - } - - /** - * Tests seek, particularly that seeking over part of a block will cause us not to cache it, and - * cancels the existing cache stream. - */ - @Test - public void testSeek() throws IOException { - assumeTrue(mFileSize >= BLOCK_LENGTH * 3.1); - int seekAmount = (int) (BLOCK_LENGTH / 2); - int readAmount = (int) (BLOCK_LENGTH * 2); - byte[] buffer = new byte[readAmount]; - // Seek halfway into block 1 - mTestStream.seek(seekAmount); - // Read two blocks from 0.5 to 2.5 - mTestStream.read(buffer); - assertArrayEquals(BufferUtils.getIncreasingByteArray(seekAmount, readAmount), buffer); - - // second block is cached if the block is not local - byte[] expected = mBlockSource != BlockInStreamSource.REMOTE ? new byte[0] - : BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH, (int) BLOCK_LENGTH); - - // Seek to current position (does nothing) - mTestStream.seek(seekAmount + readAmount); - // Seek a short way past start of block 3 - mTestStream.seek((long) (BLOCK_LENGTH * 3.1)); - assertEquals(BufferUtils.byteToInt((byte) (BLOCK_LENGTH * 3.1)), mTestStream.read()); - mTestStream.seek(mFileSize); - } - - /** - * Tests seeking back to the beginning of a block after the block's remaining is 0. - */ - @Test - public void seekToBeginningAfterReadingWholeBlock() throws IOException { - // Read the whole block. - int blockSize = (int) BLOCK_LENGTH; - byte[] block = new byte[blockSize]; - mTestStream.read(block); - assertArrayEquals(BufferUtils.getIncreasingByteArray(0, blockSize), block); - - // Seek to the beginning of the current block, then read half of it. - mTestStream.seek(0); - int halfBlockSize = blockSize / 2; - byte[] halfBlock = new byte[halfBlockSize]; - mTestStream.read(halfBlock); - assertArrayEquals(BufferUtils.getIncreasingByteArray(0, halfBlockSize), halfBlock); - } - - /** - * Tests seeking to the beginning of the last block after reaching EOF. - */ - @Test - public void seekToLastBlockAfterReachingEOF() throws IOException { - mTestStream.read(new byte[(int) mFileSize]); - mTestStream.seek(mFileSize - BLOCK_LENGTH); - byte[] block = new byte[(int) BLOCK_LENGTH]; - mTestStream.read(block); - assertArrayEquals(BufferUtils.getIncreasingByteArray( - (int) (mFileSize - BLOCK_LENGTH), (int) BLOCK_LENGTH), block); - } - - /** - * Tests seeking to EOF, then seeking to position 0 and read the whole file. - */ - @Test - public void seekToEOFBeforeReadingFirstBlock() throws IOException { - mTestStream.seek(mFileSize); - mTestStream.seek(0); - byte[] block = new byte[(int) BLOCK_LENGTH]; - mTestStream.read(block); - assertArrayEquals( - BufferUtils.getIncreasingByteArray(0, (int) BLOCK_LENGTH), block); - } - - /** - * Tests seeking with incomplete block caching enabled. It seeks backward for more than a block. - */ - @Test - public void longSeekBackwardCachingPartiallyReadBlocks() throws IOException { - OpenFilePOptions options = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = - new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf, mContext), - mContext); - int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); - int readAmount = (int) (BLOCK_LENGTH * 3 - BLOCK_LENGTH / 2); - byte[] buffer = new byte[readAmount]; - mTestStream.read(buffer); - - // Seek backward. - mTestStream.seek(readAmount - seekAmount); - - // Block 2 is cached though it is not fully read. - validatePartialCaching(2, (int) BLOCK_LENGTH / 2); - } - - /** - * Tests reading and seeking with no local worker. Nothing should be cached. - */ - @Test - public void testSeekWithNoLocalWorker() throws IOException { - // Overrides the get local worker call - when(mContext.getNodeLocalWorker()).thenReturn(null); - OpenFilePOptions options = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = - new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf, mContext), - mContext); - int readAmount = (int) (BLOCK_LENGTH / 2); - byte[] buffer = new byte[readAmount]; - // read and seek several times - mTestStream.read(buffer); - assertEquals(readAmount, mInStreams.get(0).getBytesRead()); - mTestStream.seek(BLOCK_LENGTH + BLOCK_LENGTH / 2); - mTestStream.seek(0); - - // only reads the read amount, regardless of block source - assertEquals(readAmount, mInStreams.get(0).getBytesRead()); - assertEquals(0, mInStreams.get(1).getBytesRead()); - } - - @Test - public void seekAndClose() throws IOException { - OpenFilePOptions options = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = - new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf, mContext), - mContext); - int seekAmount = (int) (BLOCK_LENGTH / 2); - mTestStream.seek(seekAmount); - mTestStream.close(); - - // Block 0 is cached though it is not fully read. - validatePartialCaching(0, 0); - } - - /** - * Tests seeking with incomplete block caching enabled. It seeks backward within 1 block. - */ - @Test - public void shortSeekBackwardCachingPartiallyReadBlocks() throws IOException { - OpenFilePOptions options = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = - new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf, mContext), - mContext); - int seekAmount = (int) (BLOCK_LENGTH / 4); - int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2); - byte[] buffer = new byte[readAmount]; - mTestStream.read(buffer); - - // Seek backward. - mTestStream.seek(readAmount - seekAmount); - - // Block 1 is cached though it is not fully read. - validatePartialCaching(1, (int) BLOCK_LENGTH / 2); - - // Seek many times. It will cache block 1 only once. - for (int i = 0; i <= seekAmount; i++) { - mTestStream.seek(readAmount - seekAmount - i); - } - validatePartialCaching(1, (int) BLOCK_LENGTH / 2); - } - - /** - * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block. - */ - @Test - public void longSeekForwardCachingPartiallyReadBlocks() throws IOException { - OpenFilePOptions options = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf, - mContext), mContext); - int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); - int readAmount = (int) (BLOCK_LENGTH / 2); - byte[] buffer = new byte[readAmount]; - mTestStream.read(buffer); - - // Seek backward. - mTestStream.seek(readAmount + seekAmount); - - // Block 0 is cached though it is not fully read. - validatePartialCaching(0, readAmount); - - // Block 1 is being cached though its prefix it not read. - validatePartialCaching(1, 0); - mTestStream.close(); - validatePartialCaching(1, 0); - } - - /** - * Tests seeking with incomplete block caching enabled. It seeks forward within a block. - */ - @Test - public void shortSeekForwardCachingPartiallyReadBlocks() throws IOException { - OpenFilePOptions options = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = - new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf, mContext), - mContext); - int seekAmount = (int) (BLOCK_LENGTH / 4); - int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2); - byte[] buffer = new byte[readAmount]; - mTestStream.read(buffer); - - // Seek backward. - mTestStream.seek(readAmount + seekAmount); - - // Block 1 (till seek pos) is being cached. - validatePartialCaching(1, (int) BLOCK_LENGTH / 2); - - // Seek forward many times. The prefix is always cached. - for (int i = 0; i < seekAmount; i++) { - mTestStream.seek(readAmount + seekAmount + i); - validatePartialCaching(1, (int) BLOCK_LENGTH / 2); - } - } - - /** - * Tests skipping backwards when the seek buffer size is smaller than block size. - */ - @Test - public void seekBackwardSmallSeekBuffer() throws IOException { - OpenFilePOptions options = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = - new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf, mContext), - mContext); - int readAmount = (int) (BLOCK_LENGTH / 2); - byte[] buffer = new byte[readAmount]; - mTestStream.read(buffer); - - mTestStream.seek(readAmount - 1); - - validatePartialCaching(0, readAmount); - } - - /** - * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block - * and then seek to the file beginning. - */ - @Test - public void seekBackwardToFileBeginning() throws IOException { - OpenFilePOptions options = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = - new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf, mContext), - mContext); - int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); - - // Seek forward. - mTestStream.seek(seekAmount); - - // Block 1 is partially cached though it is not fully read. - validatePartialCaching(1, 0); - - // Seek backward. - mTestStream.seek(0); - - // Block 1 is fully cached though it is not fully read. - validatePartialCaching(1, 0); - - mTestStream.close(); - - // block 0 is cached - validatePartialCaching(0, 0); - } - - /** - * Tests skip, particularly that skipping the start of a block will cause us not to cache it, and - * cancels the existing cache stream. - */ - @Test - public void testSkip() throws IOException { - assumeTrue(mNumBlocks > 3); - int skipAmount = (int) (BLOCK_LENGTH / 2); - int readAmount = (int) (BLOCK_LENGTH * 2); - byte[] buffer = new byte[readAmount]; - // Skip halfway into block 1 - mTestStream.skip(skipAmount); - // Read two blocks from 0.5 to 2.5 - mTestStream.read(buffer); - assertArrayEquals(BufferUtils.getIncreasingByteArray(skipAmount, readAmount), buffer); - - assertEquals(0, mTestStream.skip(0)); - // Skip the next half block, bringing us to block 3 - assertEquals(BLOCK_LENGTH / 2, mTestStream.skip(BLOCK_LENGTH / 2)); - assertEquals(BufferUtils.byteToInt((byte) (BLOCK_LENGTH * 3)), mTestStream.read()); - } - - /** - * Tests that {@link IOException}s thrown by the {@link BlockStoreClient} are properly - * propagated. - */ - @Test - public void failGetInStream() throws IOException { - when(mBlockStore.getInStream(any(BlockInfo.class), any(InStreamOptions.class), any())) - .thenThrow(new UnavailableException("test exception")); - try { - mTestStream.read(); - fail("block store should throw exception"); - } catch (IOException e) { - assertEquals("test exception", e.getMessage()); - } - } - - /** - * Tests that reading out of bounds properly returns -1. - */ - @Test - public void readOutOfBounds() throws IOException { - mTestStream.read(new byte[(int) mFileSize]); - assertEquals(-1, mTestStream.read()); - assertEquals(-1, mTestStream.read(new byte[10])); - } - - /** - * Tests that specifying an invalid offset/length for a buffer read throws the right exception. - */ - @Test - public void readBadBuffer() throws IOException { - try { - mTestStream.read(new byte[10], 5, 6); - fail("the buffer read of invalid offset/length should fail"); - } catch (IllegalArgumentException e) { - assertEquals(String.format(PreconditionMessage.ERR_BUFFER_STATE.toString(), 10, 5, 6), - e.getMessage()); - } - } - - /** - * Tests that seeking to a negative position will throw the right exception. - */ - @Test - public void seekNegative() throws IOException { - try { - mTestStream.seek(-1); - fail("seeking negative position should fail"); - } catch (IllegalArgumentException e) { - assertEquals(String.format(PreconditionMessage.ERR_SEEK_NEGATIVE.toString(), -1), - e.getMessage()); - } - } - - /** - * Tests that seeking past the end of the stream will throw the right exception. - */ - @Test - public void seekPastEnd() throws IOException { - try { - mTestStream.seek(mFileSize + 1); - fail("seeking past the end of the stream should fail"); - } catch (IllegalArgumentException e) { - assertEquals(String.format(PreconditionMessage.ERR_SEEK_PAST_END_OF_FILE.toString(), - mFileSize + 1), e.getMessage()); - } - } - - /** - * Tests that skipping a negative amount correctly reports that 0 bytes were skipped. - */ - @Test - public void skipNegative() throws IOException { - assertEquals(0, mTestStream.skip(-10)); - } - - @Test - public void positionedRead() throws IOException { - byte[] b = new byte[(int) BLOCK_LENGTH]; - mTestStream.positionedRead(BLOCK_LENGTH, b, 0, b.length); - assertArrayEquals(BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH, (int) - BLOCK_LENGTH), b); - } - - /** - * Tests the BlockInStream is closed when reading to the end of the block. - */ - @Test - public void positionedReadStreamCloseOnEnd() throws IOException { - byte[] b = new byte[(int) BLOCK_LENGTH]; - mTestStream.positionedRead(0, b, 0, b.length); - assertEquals(true, mInStreams.get(0).isClosed()); - assertArrayEquals(BufferUtils.getIncreasingByteArray((int) 0, (int) - BLOCK_LENGTH), b); - } - - @Test - public void multiBlockPositionedRead() throws IOException { - byte[] b = new byte[(int) BLOCK_LENGTH * 2]; - mTestStream.positionedRead(BLOCK_LENGTH / 2, b, 0, b.length); - assertArrayEquals(BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH / 2, (int) - BLOCK_LENGTH * 2), b); - } - - @Test - public void readOneRetry() throws Exception { - long offset = 37; - // Setups a broken stream for the first block to throw an exception. - TestBlockInStream workingStream = mInStreams.get(0); - TestBlockInStream brokenStream = mock(TestBlockInStream.class); - when(mBlockStore - .getInStream(any(BlockInfo.class), any(InStreamOptions.class), any())) - .thenReturn(brokenStream).thenReturn(workingStream); - when(brokenStream.read()).thenThrow(new UnavailableException("test exception")); - when(brokenStream.getPos()).thenReturn(offset); - - mTestStream.seek(offset); - int b = mTestStream.read(); - - doReturn(0).when(brokenStream).read(); - verify(brokenStream, times(1)).read(); - assertEquals(offset, b); - } - - @Test - public void readBufferRetry() throws Exception { - TestBlockInStream workingStream = mInStreams.get(0); - TestBlockInStream brokenStream = mock(TestBlockInStream.class); - when(mBlockStore - .getInStream(any(BlockInfo.class), any(InStreamOptions.class), any())) - .thenReturn(brokenStream).thenReturn(workingStream); - when(brokenStream.read(any(ByteBuffer.class), anyInt(), anyInt())) - .thenThrow(new UnavailableException("test exception")); - when(brokenStream.getPos()).thenReturn(BLOCK_LENGTH / 2); - - mTestStream.seek(BLOCK_LENGTH / 2); - byte[] b = new byte[(int) BLOCK_LENGTH * 2]; - mTestStream.read(b, 0, b.length); - - doReturn(0).when(brokenStream).read(any(ByteBuffer.class), anyInt(), anyInt()); - verify(brokenStream, times(1)) - .read(any(ByteBuffer.class), anyInt(), anyInt()); - assertArrayEquals(BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH / 2, (int) - BLOCK_LENGTH * 2), b); - } - - @Test - public void positionedReadRetry() throws Exception { - TestBlockInStream workingStream = mInStreams.get(0); - TestBlockInStream brokenStream = mock(TestBlockInStream.class); - when(mBlockStore - .getInStream(eq(0L), any(InStreamOptions.class), any())) - .thenReturn(brokenStream).thenReturn(workingStream); - when(brokenStream.positionedRead(anyLong(), any(byte[].class), anyInt(), anyInt())) - .thenThrow(new UnavailableException("test exception")); - - byte[] b = new byte[(int) BLOCK_LENGTH * 2]; - mTestStream.positionedRead(BLOCK_LENGTH / 2, b, 0, b.length); - - doReturn(0) - .when(brokenStream).positionedRead(anyLong(), any(byte[].class), anyInt(), anyInt()); - verify(brokenStream, times(1)) - .positionedRead(anyLong(), any(byte[].class), anyInt(), anyInt()); - assertArrayEquals(BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH / 2, (int) - BLOCK_LENGTH * 2), b); - } - - /** - * Tests that when the underlying blocks are inconsistent with the metadata in terms of block - * length, an exception is thrown rather than client hanging indefinitely. This case may happen if - * the file in Alluxio and UFS is out of sync. - */ - @Test - public void blockInStreamOutOfSync() throws Exception { - when(mBlockStore.getInStream(any(BlockInfo.class), any(InStreamOptions.class), any())) - .thenAnswer(new Answer() { - @Override - public BlockInStream answer(InvocationOnMock invocation) throws Throwable { - return new TestBlockInStream(new byte[1], 0, BLOCK_LENGTH, mBlockSource); - } - }); - byte[] buffer = new byte[(int) BLOCK_LENGTH]; - try { - mTestStream.read(buffer, 0, (int) BLOCK_LENGTH); - fail("BlockInStream is inconsistent, an Exception is expected"); - } catch (IllegalStateException e) { - // expect an exception to throw - } - } - - @Test - public void getPos() throws Exception { - assertEquals(0, mTestStream.getPos()); - mTestStream.read(); - assertEquals(1, mTestStream.getPos()); - mTestStream.read(new byte[(int) mFileSize], 0, (int) mFileSize); - assertEquals(mFileSize, mTestStream.getPos()); - } - - // See https://github.com/Alluxio/alluxio/issues/13828 - @Test - public void triggerAsyncOnClose() throws Exception { - assumeTrue(mBlockSource == BlockInStreamSource.UFS); - mInfo.setReplicationMax(1); - mStatus = new URIStatus(mInfo); - OpenFilePOptions readOptions = - OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); - mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, readOptions, - mConf, mContext), mContext); - mTestStream.read(new byte[(int) mFileSize], 0, (int) mFileSize); - assertEquals(mFileSize, mTestStream.getPos()); - assertTrue(mTestStream.triggerAsyncCaching(mInStreams.get(mInStreams.size() - 1))); - } - - /** - * Tests that reading dataRead bytes into a buffer will properly write those bytes to the cache - * streams and that the correct bytes are read from the {@link FileInStream}. - * - * @param dataRead the bytes to read - */ - private void testReadBuffer(int dataRead) throws Exception { - byte[] buffer = new byte[dataRead]; - mTestStream.read(buffer); - assertArrayEquals(BufferUtils.getIncreasingByteArray(dataRead), buffer); - } - - /** - * Validates the partial caching behavior. This function - * verifies the block at the given index is read for the given sizes. - */ - // TODO(binfan): with better netty RPC mocking, verify that async cache request for the target - // block is sent to the netty channel - private void validatePartialCaching(int index, int readSize) { - assertEquals(readSize, mInStreams.get(index).getBytesRead()); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/BaseFileSystemTest.java b/core/client/fs/src/test/java/alluxio/client/file/BaseFileSystemTest.java deleted file mode 100644 index 44e3a51a3e66..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/BaseFileSystemTest.java +++ /dev/null @@ -1,662 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import alluxio.AlluxioURI; -import alluxio.ClientContext; -import alluxio.TestLoggerRule; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.Bits; -import alluxio.grpc.CreateDirectoryPOptions; -import alluxio.grpc.CreateFilePOptions; -import alluxio.grpc.DeletePOptions; -import alluxio.grpc.FreePOptions; -import alluxio.grpc.GetStatusPOptions; -import alluxio.grpc.ListStatusPOptions; -import alluxio.grpc.MountPOptions; -import alluxio.grpc.OpenFilePOptions; -import alluxio.grpc.RenamePOptions; -import alluxio.grpc.SetAttributePOptions; -import alluxio.grpc.UnmountPOptions; -import alluxio.resource.CloseableResource; -import alluxio.util.FileSystemOptionsUtils; -import alluxio.wire.FileInfo; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.ArrayList; -import java.util.List; - -/** -* Unit test for functionality in {@link BaseFileSystem}. -*/ -@RunWith(PowerMockRunner.class) -@PrepareForTest({FileSystemContext.class, FileSystemMasterClient.class}) -public final class BaseFileSystemTest { - - private static final RuntimeException EXCEPTION = new RuntimeException("test exception"); - private static final String SHOULD_HAVE_PROPAGATED_MESSAGE = - "Exception should have been propagated"; - - private InstancedConfiguration mConf = Configuration.copyGlobal(); - - @Rule - private TestLoggerRule mTestLogger = new TestLoggerRule(); - - private FileSystem mFileSystem; - private FileSystemContext mFileContext; - private ClientContext mClientContext; - private FileSystemMasterClient mFileSystemMasterClient; - - private class DummyAlluxioFileSystem extends BaseFileSystem { - public DummyAlluxioFileSystem(FileSystemContext fsContext) { - super(fsContext); - } - } - - /** - * Sets up the file system and the context before a test runs. - */ - @Before - public void before() { - mConf.set(PropertyKey.USER_FILE_INCLUDE_OPERATION_ID, false); - mClientContext = ClientContext.create(mConf); - mFileContext = PowerMockito.mock(FileSystemContext.class); - mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class); - when(mFileContext.acquireMasterClientResource()).thenReturn( - new CloseableResource(mFileSystemMasterClient) { - @Override - public void closeResource() { - // Noop. - } - }); - when(mFileContext.getClientContext()).thenReturn(mClientContext); - when(mFileContext.getClusterConf()).thenReturn(mConf); - when(mFileContext.getPathConf(any())).thenReturn(mConf); - when(mFileContext.getUriValidationEnabled()).thenReturn(true); - mFileSystem = new DummyAlluxioFileSystem(mFileContext); - } - - @After - public void after() { - mConf = Configuration.copyGlobal(); - } - - /** - * Verifies and releases the master client after a test with a filesystem operation. - */ - public void verifyFilesystemContextAcquiredAndReleased() { - verify(mFileContext).acquireMasterClientResource(); - } - - /** - * Tests the creation of a file via the - * {@link BaseFileSystem#createFile(AlluxioURI, CreateFilePOptions)} method. - */ - @Test - public void createFile() throws Exception { - URIStatus status = new URIStatus(new FileInfo()); - AlluxioURI file = new AlluxioURI("/file"); - when(mFileSystemMasterClient.createFile(any(AlluxioURI.class), any(CreateFilePOptions.class))) - .thenReturn(status); - mFileSystem.createFile(file, CreateFilePOptions.getDefaultInstance()); - verify(mFileSystemMasterClient).createFile(file, - FileSystemOptionsUtils.createFileDefaults(mConf) - .toBuilder().mergeFrom(CreateFilePOptions.getDefaultInstance()).build()); - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Ensures that an exception is propagated correctly when creating a file system. - */ - @Test - public void createException() throws Exception { - doThrow(EXCEPTION).when(mFileSystemMasterClient) - .createFile(any(AlluxioURI.class), any(CreateFilePOptions.class)); - try { - mFileSystem.createFile(new AlluxioURI("/"), CreateFilePOptions.getDefaultInstance()); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Tests for the {@link BaseFileSystem#delete(AlluxioURI, DeletePOptions)} method. - */ - @Test - public void delete() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - DeletePOptions deleteOptions = DeletePOptions.newBuilder().setRecursive(true).build(); - mFileSystem.delete(file, deleteOptions); - verify(mFileSystemMasterClient).delete(file, - FileSystemOptionsUtils.deleteDefaults(mConf).toBuilder().mergeFrom(deleteOptions).build()); - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Ensures that an exception is propagated correctly when deleting a file. - */ - @Test - public void deleteException() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - DeletePOptions deleteOptions = DeletePOptions.newBuilder().setRecursive(true).build(); - doThrow(EXCEPTION).when(mFileSystemMasterClient).delete(file, - FileSystemOptionsUtils.deleteDefaults(mConf) - .toBuilder().mergeFrom(deleteOptions).build()); - try { - mFileSystem.delete(file, deleteOptions); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Tests for the {@link BaseFileSystem#free(AlluxioURI, FreePOptions)} method. - */ - @Test - public void free() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - FreePOptions freeOptions = FreePOptions.newBuilder().setRecursive(true).build(); - mFileSystem.free(file, freeOptions); - verify(mFileSystemMasterClient).free(file, FileSystemOptionsUtils.freeDefaults(mConf) - .toBuilder().mergeFrom(freeOptions).build()); - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Ensures that an exception is propagated correctly when freeing a file. - */ - @Test - public void freeException() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - FreePOptions freeOptions = FreePOptions.newBuilder().setRecursive(true).build(); - doThrow(EXCEPTION).when(mFileSystemMasterClient).free(file, - FileSystemOptionsUtils.freeDefaults(mConf).toBuilder().mergeFrom(freeOptions).build()); - try { - mFileSystem.free(file, freeOptions); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Tests for the {@link BaseFileSystem#getStatus(AlluxioURI, GetStatusPOptions)} method. - */ - @Test - public void getStatus() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - URIStatus status = new URIStatus(new FileInfo()); - GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance(); - when(mFileSystemMasterClient.getStatus(file, FileSystemOptionsUtils.getStatusDefaults(mConf) - .toBuilder().mergeFrom(getStatusOptions).build())).thenReturn(status); - assertSame(status, mFileSystem.getStatus(file, getStatusOptions)); - verify(mFileSystemMasterClient).getStatus(file, FileSystemOptionsUtils.getStatusDefaults(mConf) - .toBuilder().mergeFrom(getStatusOptions).build()); - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Ensures that an exception is propagated correctly when retrieving information. - */ - @Test - public void getStatusException() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance(); - when(mFileSystemMasterClient.getStatus(file, FileSystemOptionsUtils.getStatusDefaults(mConf) - .toBuilder().mergeFrom(getStatusOptions).build())).thenThrow(EXCEPTION); - try { - mFileSystem.getStatus(file, getStatusOptions); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Tests for the {@link BaseFileSystem#listStatus(AlluxioURI, ListStatusPOptions)} method. - */ - @Test - public void listStatus() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - List infos = new ArrayList<>(); - infos.add(new URIStatus(new FileInfo())); - ListStatusPOptions listStatusOptions = ListStatusPOptions.getDefaultInstance(); - when(mFileSystemMasterClient.listStatus(file, FileSystemOptionsUtils.listStatusDefaults(mConf) - .toBuilder().mergeFrom(listStatusOptions).build())).thenReturn(infos); - assertSame(infos, mFileSystem.listStatus(file, listStatusOptions)); - verify(mFileSystemMasterClient).listStatus(file, - FileSystemOptionsUtils.listStatusDefaults(mConf) - .toBuilder().mergeFrom(listStatusOptions).build()); - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Ensures that an exception is propagated correctly when listing the status. - */ - @Test - public void listStatusException() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - when(mFileSystemMasterClient.listStatus(file, FileSystemOptionsUtils.listStatusDefaults(mConf) - .toBuilder().mergeFrom(ListStatusPOptions.getDefaultInstance()).build())) - .thenThrow(EXCEPTION); - try { - mFileSystem.listStatus(file, ListStatusPOptions.getDefaultInstance()); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - - verifyFilesystemContextAcquiredAndReleased(); - } - -// /** -// * Tests for the {@link BaseFileSystem#loadMetadata(AlluxioURI, LoadMetadataOptions)} -// * method. -// */ -// @Test -// public void loadMetadata() throws Exception { -// AlluxioURI file = new AlluxioURI("/file"); -// LoadMetadataOptions loadMetadataOptions = LoadMetadataOptions.defaults().setRecursive(true); -// doNothing().when(mFileSystemMasterClient).loadMetadata(file, loadMetadataOptions); -// mFileSystem.loadMetadata(file, loadMetadataOptions); -// verify(mFileSystemMasterClient).loadMetadata(file, loadMetadataOptions); -// -// verifyFilesystemContextAcquiredAndReleased(); -// } -// -// /** -// * Ensures that an exception is propagated correctly when loading the metadata. -// */ -// @Test -// public void loadMetadataException() throws Exception { -// AlluxioURI file = new AlluxioURI("/file"); -// LoadMetadataOptions loadMetadataOptions = LoadMetadataOptions.defaults().setRecursive(true); -// doThrow(EXCEPTION).when(mFileSystemMasterClient) -// .loadMetadata(file, loadMetadataOptions); -// try { -// mFileSystem.loadMetadata(file, loadMetadataOptions); -// fail(SHOULD_HAVE_PROPAGATED_MESSAGE); -// } catch (Exception e) { -// assertSame(EXCEPTION, e); -// } -// -// verifyFilesystemContextAcquiredAndReleased(); -// } - - /** - * Tests for the {@link BaseFileSystem#createDirectory(AlluxioURI, CreateDirectoryPOptions)} - * method. - */ - @Test - public void createDirectory() throws Exception { - AlluxioURI dir = new AlluxioURI("/dir"); - CreateDirectoryPOptions createDirectoryOptions = CreateDirectoryPOptions.getDefaultInstance(); - doNothing().when(mFileSystemMasterClient).createDirectory(dir, - FileSystemOptionsUtils.createDirectoryDefaults(mConf) - .toBuilder().mergeFrom(createDirectoryOptions).build()); - mFileSystem.createDirectory(dir, createDirectoryOptions); - verify(mFileSystemMasterClient).createDirectory(dir, - FileSystemOptionsUtils.createDirectoryDefaults(mConf) - .toBuilder().mergeFrom(createDirectoryOptions).build()); - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Ensures that an exception is propagated correctly when creating a directory. - */ - @Test - public void createDirectoryException() throws Exception { - AlluxioURI dir = new AlluxioURI("/dir"); - CreateDirectoryPOptions createDirectoryOptions = CreateDirectoryPOptions.getDefaultInstance(); - doThrow(EXCEPTION).when(mFileSystemMasterClient).createDirectory(dir, - FileSystemOptionsUtils.createDirectoryDefaults(mConf) - .toBuilder().mergeFrom(createDirectoryOptions).build()); - try { - mFileSystem.createDirectory(dir, createDirectoryOptions); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Tests for the {@link BaseFileSystem#mount(AlluxioURI, AlluxioURI, MountPOptions)} method. - */ - @Test - public void mount() throws Exception { - AlluxioURI alluxioPath = new AlluxioURI("/t"); - AlluxioURI ufsPath = new AlluxioURI("/u"); - MountPOptions mountOptions = MountPOptions.getDefaultInstance(); - doNothing().when(mFileSystemMasterClient).mount(alluxioPath, ufsPath, - FileSystemOptionsUtils.mountDefaults(mConf).toBuilder().mergeFrom(mountOptions).build()); - mFileSystem.mount(alluxioPath, ufsPath, mountOptions); - verify(mFileSystemMasterClient).mount(alluxioPath, ufsPath, - FileSystemOptionsUtils.mountDefaults(mConf).toBuilder().mergeFrom(mountOptions).build()); - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Ensures that an exception is propagated correctly when mounting a path. - */ - @Test - public void mountException() throws Exception { - AlluxioURI alluxioPath = new AlluxioURI("/t"); - AlluxioURI ufsPath = new AlluxioURI("/u"); - MountPOptions mountOptions = MountPOptions.getDefaultInstance(); - doThrow(EXCEPTION).when(mFileSystemMasterClient) - .mount(alluxioPath, ufsPath, FileSystemOptionsUtils.mountDefaults(mConf) - .toBuilder().mergeFrom(mountOptions).build()); - try { - mFileSystem.mount(alluxioPath, ufsPath, mountOptions); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Tests for the {@link BaseFileSystem#openFile(AlluxioURI, OpenFilePOptions)} method to - * complete successfully. - */ - @Test - public void openFile() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - URIStatus status = new URIStatus(new FileInfo().setCompleted(true)); - GetStatusPOptions getStatusOptions = getOpenOptions(GetStatusPOptions.getDefaultInstance()); - when(mFileSystemMasterClient.getStatus(file, getStatusOptions)) - .thenReturn(status); - mFileSystem.openFile(file, OpenFilePOptions.getDefaultInstance()); - verify(mFileSystemMasterClient).getStatus(file, getStatusOptions); - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Ensures that an exception is propagated successfully when opening a file. - */ - @Test - public void openException() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - GetStatusPOptions getStatusOptions = getOpenOptions(GetStatusPOptions.getDefaultInstance()); - when(mFileSystemMasterClient.getStatus(file, getStatusOptions)) - .thenThrow(EXCEPTION); - try { - mFileSystem.openFile(file, OpenFilePOptions.getDefaultInstance()); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - - verifyFilesystemContextAcquiredAndReleased(); - } - - /** - * Tests for the {@link BaseFileSystem#rename(AlluxioURI, AlluxioURI, RenamePOptions)} - * method. - */ - @Test - public void rename() throws Exception { - AlluxioURI src = new AlluxioURI("/file"); - AlluxioURI dst = new AlluxioURI("/file2"); - RenamePOptions renameOptions = RenamePOptions.getDefaultInstance(); - doNothing().when(mFileSystemMasterClient).rename(src, dst, - FileSystemOptionsUtils.renameDefaults(mConf).toBuilder().mergeFrom(renameOptions).build()); - mFileSystem.rename(src, dst, renameOptions); - verify(mFileSystemMasterClient).rename(src, dst, - FileSystemOptionsUtils.renameDefaults(mConf).toBuilder().mergeFrom(renameOptions).build()); - } - - /** - * Ensures that an exception is propagated successfully when renaming a file. - */ - @Test - public void renameException() throws Exception { - AlluxioURI src = new AlluxioURI("/file"); - AlluxioURI dst = new AlluxioURI("/file2"); - RenamePOptions renameOptions = RenamePOptions.getDefaultInstance(); - doThrow(EXCEPTION).when(mFileSystemMasterClient).rename(src, dst, - FileSystemOptionsUtils.renameDefaults(mConf) - .toBuilder().mergeFrom(renameOptions).build()); - try { - mFileSystem.rename(src, dst, renameOptions); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - } - - /** - * Tests for the {@link BaseFileSystem#setAttribute(AlluxioURI, SetAttributePOptions)} method. - */ - @Test - public void setAttribute() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - SetAttributePOptions setAttributeOptions = - FileSystemOptionsUtils.setAttributeClientDefaults(mFileContext.getPathConf(file)); - mFileSystem.setAttribute(file, setAttributeOptions); - verify(mFileSystemMasterClient).setAttribute(file, setAttributeOptions); - } - - /** - * Tests that the metadata sync interval is included on setAttributePOptions by default. - */ - @Test - public void setAttributeSyncMetadataInterval() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - SetAttributePOptions opt = - FileSystemOptionsUtils.setAttributeClientDefaults(mFileContext.getPathConf(file)); - // Check that metadata sync interval from configuration is used when options are omitted - mFileSystem.setAttribute(file); - verify(mFileSystemMasterClient).setAttribute(file, opt); - } - - /** - * Ensures that an exception is propagated successfully when setting the state. - */ - @Test - public void setStateException() throws Exception { - AlluxioURI file = new AlluxioURI("/file"); - SetAttributePOptions setAttributeOptions = - FileSystemOptionsUtils.setAttributeClientDefaults(mFileContext.getPathConf(file)); - doThrow(EXCEPTION).when(mFileSystemMasterClient).setAttribute(file, setAttributeOptions); - try { - mFileSystem.setAttribute(file, setAttributeOptions); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - } - - /** - * Tests for the {@link BaseFileSystem#unmount(AlluxioURI, UnmountPOptions)} method. - */ - @Test - public void unmount() throws Exception { - AlluxioURI path = new AlluxioURI("/"); - UnmountPOptions unmountOptions = UnmountPOptions.getDefaultInstance(); - doNothing().when(mFileSystemMasterClient).unmount(path); - mFileSystem.unmount(path, unmountOptions); - verify(mFileSystemMasterClient).unmount(path); - } - - /** - * Ensures that an exception is propagated successfully when unmounting a path. - */ - @Test - public void unmountException() throws Exception { - AlluxioURI path = new AlluxioURI("/"); - UnmountPOptions unmountOptions = UnmountPOptions.getDefaultInstance(); - doThrow(EXCEPTION).when(mFileSystemMasterClient).unmount(path); - try { - mFileSystem.unmount(path, unmountOptions); - fail(SHOULD_HAVE_PROPAGATED_MESSAGE); - } catch (Exception e) { - assertSame(EXCEPTION, e); - } - } - - /** - * Ensures warnings are logged and an exception is thrown when an {@link AlluxioURI} with an - * invalid authority is passed. - */ - @Test - public void uriCheckBadAuthority() throws Exception { - mConf.set(PropertyKey.MASTER_HOSTNAME, "localhost"); - mConf.set(PropertyKey.MASTER_RPC_PORT, 19998); - - assertBadAuthority("localhost:1234", "Should fail on bad host and port"); - assertBadAuthority("zk@localhost:19998", "Should fail on zk authority"); - - assertTrue(loggedAuthorityWarning()); - assertTrue(loggedSchemeWarning()); - } - - /** - * Ensures an exception is thrown when an invalid scheme is passed. - */ - @Test - public void uriCheckBadScheme() throws Exception { - mConf.set(PropertyKey.MASTER_HOSTNAME, "localhost"); - mConf.set(PropertyKey.MASTER_RPC_PORT, 19998); - - AlluxioURI uri = new AlluxioURI("hdfs://localhost:19998/root"); - try { - mFileSystem.createDirectory(uri); - fail("Should have failed on bad host and port"); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("Scheme hdfs:// in AlluxioURI is invalid")); - } - } - - /** - * Ensures there is one warning when a URI with a valid scheme and authority is passed. - */ - @Test - public void uriCheckGoodSchemeAndAuthority() throws Exception { - mConf.set(PropertyKey.MASTER_HOSTNAME, "localhost"); - mConf.set(PropertyKey.MASTER_RPC_PORT, 19998); - before(); // Resets the filesystem and contexts to use proper configuration. - - useUriWithAuthority("localhost:19998"); - - assertTrue(loggedAuthorityWarning()); - assertTrue(loggedSchemeWarning()); - } - - /** - * Ensures there is no warnings or errors when an {@link AlluxioURI} without a scheme and - * authority is passed. - */ - @Test - public void uriCheckNoSchemeAuthority() throws Exception { - mConf.set(PropertyKey.MASTER_HOSTNAME, "localhost"); - mConf.set(PropertyKey.MASTER_RPC_PORT, 19998); - - AlluxioURI uri = new AlluxioURI("/root"); - mFileSystem.createDirectory(uri); - - assertFalse(loggedAuthorityWarning()); - assertFalse(loggedSchemeWarning()); - } - - @Test - public void uriCheckZkAuthorityMatch() throws Exception { - configureZk("a:0,b:0,c:0"); - useUriWithAuthority("zk@a:0,b:0,c:0"); // Same authority - useUriWithAuthority("zk@a:0;b:0+c:0"); // Same authority, but different delimiters - } - - @Test - public void uriCheckZkAuthorityMismatch() throws Exception { - configureZk("a:0,b:0,c:0"); - - assertBadAuthority("a:0,b:0,c:0", "Should fail on non-zk authority"); - assertBadAuthority("zk@a:0", "Should fail on zk authority with different addresses"); - assertBadAuthority("zk@a:0,b:0,c:1", "Should fail on zk authority with different addresses"); - } - - private GetStatusPOptions getOpenOptions(GetStatusPOptions getStatusOptions) { - return FileSystemOptionsUtils.getStatusDefaults(mConf) - .toBuilder().setAccessMode(Bits.READ).setUpdateTimestamps(true) - .mergeFrom(getStatusOptions).build(); - } - - private void assertBadAuthority(String authority, String failureMessage) throws Exception { - try { - useUriWithAuthority(authority); - fail(failureMessage); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("does not match")); - } - } - - private void useUriWithAuthority(String authority) throws Exception { - mFileSystem.createDirectory(new AlluxioURI(String.format("alluxio://%s/dir", authority))); - } - - private boolean loggedAuthorityWarning() { - return mTestLogger.wasLogged("The URI authority .* is ignored"); - } - - private boolean loggedSchemeWarning() { - return mTestLogger.wasLogged("The URI scheme .* is ignored"); - } - - private void configureZk(String addrs) { - mConf.set(PropertyKey.ZOOKEEPER_ENABLED, true); - mConf.set(PropertyKey.ZOOKEEPER_ADDRESS, addrs); - before(); // Resets the filesystem and contexts to use proper configuration - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/FileOutStreamTest.java b/core/client/fs/src/test/java/alluxio/client/file/FileOutStreamTest.java deleted file mode 100644 index a10b1fc3418c..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/FileOutStreamTest.java +++ /dev/null @@ -1,515 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import alluxio.AlluxioURI; -import alluxio.ClientContext; -import alluxio.Constants; -import alluxio.client.UnderStorageType; -import alluxio.client.WriteType; -import alluxio.client.block.BlockStoreClient; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.client.block.policy.BlockLocationPolicy; -import alluxio.client.block.stream.BlockOutStream; -import alluxio.client.block.stream.TestBlockOutStream; -import alluxio.client.block.stream.TestUnderFileSystemFileOutStream; -import alluxio.client.block.stream.UnderFileSystemFileOutStream; -import alluxio.client.file.options.OutStreamOptions; -import alluxio.client.util.ClientTestUtils; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.PreconditionMessage; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.CompleteFilePOptions; -import alluxio.grpc.CreateFilePOptions; -import alluxio.grpc.FileSystemMasterCommonPOptions; -import alluxio.grpc.GetStatusPOptions; -import alluxio.grpc.TtlAction; -import alluxio.grpc.WritePType; -import alluxio.network.TieredIdentityFactory; -import alluxio.resource.DummyCloseableResource; -import alluxio.security.GroupMappingServiceTestUtils; -import alluxio.util.io.BufferUtils; -import alluxio.wire.FileInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.collect.Lists; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Random; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Tests for the {@link FileOutStream} class. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({FileSystemContext.class, FileSystemMasterClient.class, BlockStoreClient.class, - UnderFileSystemFileOutStream.class}) -public class FileOutStreamTest { - - private static InstancedConfiguration sConf = Configuration.copyGlobal(); - - private static final long BLOCK_LENGTH = 100L; - private static final AlluxioURI FILE_NAME = new AlluxioURI("/file"); - - private FileSystemContext mFileSystemContext; - private BlockStoreClient mBlockStore; - private FileSystemMasterClient mFileSystemMasterClient; - - private Map mAlluxioOutStreamMap; - private TestUnderFileSystemFileOutStream mUnderStorageOutputStream; - private AtomicBoolean mUnderStorageFlushed; - - private AlluxioFileOutStream mTestStream; - private ClientContext mClientContext; - - /** - * Sets up the different contexts and clients before a test runs. - */ - @Before - public void before() throws Exception { - GroupMappingServiceTestUtils.resetCache(); - ClientTestUtils.setSmallBufferSizes(sConf); - - mClientContext = ClientContext.create(sConf); - - // PowerMock enums and final classes - mFileSystemContext = PowerMockito.mock(FileSystemContext.class); - when(mFileSystemContext.getClientContext()).thenReturn(mClientContext); - when(mFileSystemContext.getClusterConf()).thenReturn(sConf); - when(mFileSystemContext.getPathConf(any(AlluxioURI.class))).thenReturn(sConf); - when(mFileSystemContext.getWriteBlockLocationPolicy(any(AlluxioConfiguration.class))) - .thenAnswer((Answer) invocation -> { - AlluxioConfiguration conf = - invocation.getArgument(0, AlluxioConfiguration.class); - return BlockLocationPolicy.Factory.create( - conf.getClass(PropertyKey.USER_BLOCK_WRITE_LOCATION_POLICY), conf); - }); - mBlockStore = PowerMockito.mock(BlockStoreClient.class); - mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class); - - PowerMockito.mockStatic(BlockStoreClient.class); - PowerMockito.when(BlockStoreClient.create(mFileSystemContext)).thenReturn(mBlockStore); - - when(mFileSystemContext.acquireMasterClientResource()) - .thenReturn(new DummyCloseableResource<>(mFileSystemMasterClient)); - when(mFileSystemMasterClient.getStatus(any(AlluxioURI.class), any(GetStatusPOptions.class))) - .thenReturn(new URIStatus(new FileInfo())); - - // Return sequentially increasing numbers for new block ids - when(mFileSystemMasterClient.getNewBlockIdForFile(FILE_NAME)) - .thenAnswer(new Answer() { - private long mCount = 0; - - @Override - public Long answer(InvocationOnMock invocation) throws Throwable { - return mCount++; - } - }); - - // Set up out streams. When they are created, add them to outStreamMap - final Map outStreamMap = new HashMap<>(); - when(mBlockStore.getOutStream(anyLong(), eq(BLOCK_LENGTH), - any(OutStreamOptions.class))).thenAnswer(new Answer() { - @Override - public TestBlockOutStream answer(InvocationOnMock invocation) throws Throwable { - Long blockId = invocation.getArgument(0, Long.class); - if (!outStreamMap.containsKey(blockId)) { - TestBlockOutStream newStream = - new TestBlockOutStream(ByteBuffer.allocate(1000), BLOCK_LENGTH); - outStreamMap.put(blockId, newStream); - } - return outStreamMap.get(blockId); - } - }); - BlockWorkerInfo workerInfo = - new BlockWorkerInfo(new WorkerNetAddress().setHost("localhost") - .setTieredIdentity(TieredIdentityFactory.fromString("node=localhost", sConf)) - .setRpcPort(1).setDataPort(2).setWebPort(3), Constants.GB, 0); - when(mFileSystemContext.getCachedWorkers()).thenReturn(Lists.newArrayList(workerInfo)); - mAlluxioOutStreamMap = outStreamMap; - - // Create an under storage stream so that we can check whether it has been flushed - final AtomicBoolean underStorageFlushed = new AtomicBoolean(false); - mUnderStorageOutputStream = - new TestUnderFileSystemFileOutStream(ByteBuffer.allocate(5000)) { - @Override - public void flush() throws IOException { - super.flush(); - underStorageFlushed.set(true); - } - }; - mUnderStorageFlushed = underStorageFlushed; - - PowerMockito.mockStatic(UnderFileSystemFileOutStream.class); - PowerMockito.when( - UnderFileSystemFileOutStream.create(any(FileSystemContext.class), - any(WorkerNetAddress.class), any(OutStreamOptions.class))).thenReturn( - mUnderStorageOutputStream); - - OutStreamOptions options = - OutStreamOptions.defaults(mFileSystemContext).setBlockSizeBytes(BLOCK_LENGTH) - .setWriteType(WriteType.CACHE_THROUGH).setUfsPath(FILE_NAME.getPath()); - mTestStream = createTestStream(FILE_NAME, options); - } - - @After - public void after() { - ClientTestUtils.resetClient(sConf); - } - - /** - * Tests that a single byte is written to the out stream correctly. - */ - @Test - public void singleByteWrite() throws Exception { - mTestStream.write(5); - mTestStream.close(); - assertArrayEquals(new byte[] {5}, mAlluxioOutStreamMap.get(0L).getWrittenData()); - } - - /** - * Tests that many bytes, written one at a time, are written to the out streams correctly. - */ - @Test - public void manyBytesWrite() throws IOException { - int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); - for (int i = 0; i < bytesToWrite; i++) { - mTestStream.write(i); - } - mTestStream.close(); - verifyIncreasingBytesWritten(bytesToWrite); - } - - /** - * Tests that writing a buffer all at once will write bytes to the out streams correctly. - */ - @Test - public void writeBuffer() throws IOException { - int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); - mTestStream.write(BufferUtils.getIncreasingByteArray(bytesToWrite)); - mTestStream.close(); - verifyIncreasingBytesWritten(bytesToWrite); - } - - /** - * Tests writing a buffer at an offset. - */ - @Test - public void writeOffset() throws IOException { - int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); - int offset = (int) (BLOCK_LENGTH / 3); - mTestStream.write(BufferUtils.getIncreasingByteArray(bytesToWrite + offset), offset, - bytesToWrite); - mTestStream.close(); - verifyIncreasingBytesWritten(offset, bytesToWrite); - } - - /** - * Tests that {@link FileOutStream#close()} will close but not cancel the underlying out streams. - * Also checks that {@link FileOutStream#close()} persists and completes the file. - */ - @Test - public void close() throws Exception { - mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); - mTestStream.close(); - for (long streamIndex = 0; streamIndex < 2; streamIndex++) { - assertFalse(mAlluxioOutStreamMap.get(streamIndex).isCanceled()); - assertTrue(mAlluxioOutStreamMap.get(streamIndex).isClosed()); - } - verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class)); - } - - /** - * Tests that {@link FileOutStream#cancel()} will cancel and close the underlying out streams, and - * delete from the under file system when the delegation flag is set. Also makes sure that - * cancel() doesn't persist or complete the file. - */ - @Test - public void cancelWithDelegation() throws Exception { - mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); - mTestStream.cancel(); - for (long streamIndex = 0; streamIndex < 2; streamIndex++) { - assertTrue(mAlluxioOutStreamMap.get(streamIndex).isClosed()); - assertTrue(mAlluxioOutStreamMap.get(streamIndex).isCanceled()); - } - // Don't complete the file if the stream was canceled - verify(mFileSystemMasterClient, times(0)).completeFile(any(AlluxioURI.class), - any(CompleteFilePOptions.class)); - } - - /** - * Tests that {@link FileOutStream#flush()} will flush the under store stream. - */ - @Test - public void flush() throws IOException { - assertFalse(mUnderStorageFlushed.get()); - mTestStream.flush(); - assertTrue(mUnderStorageFlushed.get()); - } - - /** - * Tests that if an exception is thrown by the underlying out stream, and the user is using - * {@link UnderStorageType#NO_PERSIST} for their under storage type, the correct exception - * message will be thrown. - */ - @Test - public void cacheWriteExceptionNonSyncPersist() throws IOException { - OutStreamOptions options = - OutStreamOptions.defaults(mFileSystemContext).setBlockSizeBytes(BLOCK_LENGTH) - .setWriteType(WriteType.MUST_CACHE); - BlockOutStream stream = mock(BlockOutStream.class); - when(mBlockStore.getOutStream(anyLong(), anyLong(), any(OutStreamOptions.class))) - .thenReturn(stream); - mTestStream = createTestStream(FILE_NAME, options); - - when(stream.remaining()).thenReturn(BLOCK_LENGTH); - doThrow(new IOException("test error")).when(stream).write(7); - - try { - mTestStream.write(7); - fail("the test should fail"); - } catch (IOException e) { - assertEquals(ExceptionMessage.FAILED_CACHE.getMessage("test error"), e.getMessage()); - } - } - - /** - * Tests that if an exception is thrown by the underlying out stream, and the user is using - * {@link UnderStorageType#SYNC_PERSIST} for their under storage type, the error is recovered - * from by writing the data to the under storage out stream. - */ - @Test - public void cacheWriteExceptionSyncPersist() throws IOException { - BlockOutStream stream = mock(BlockOutStream.class); - when(mBlockStore.getOutStream(anyLong(), anyLong(), any(OutStreamOptions.class))) - .thenReturn(stream); - - when(stream.remaining()).thenReturn(BLOCK_LENGTH); - doThrow(new IOException("test error")).when(stream).write((byte) 7); - mTestStream.write(7); - mTestStream.write(8); - assertArrayEquals(new byte[] {7, 8}, mUnderStorageOutputStream.getWrittenData()); - // The cache stream is written to only once - the FileInStream gives up on it after it throws - // the first exception. - verify(stream, times(1)).write(anyInt()); - } - - /** - * Tests that write only writes a byte. - */ - @Test - public void truncateWrite() throws IOException { - // Only writes the lowest byte - mTestStream.write(0x1fffff00); - mTestStream.write(0x1fffff01); - mTestStream.close(); - verifyIncreasingBytesWritten(2); - } - - /** - * Tests that the correct exception is thrown when a buffer is written with invalid offset/length. - */ - @Test - public void writeBadBufferOffset() throws IOException { - try { - mTestStream.write(new byte[10], 5, 6); - fail("buffer write with invalid offset/length should fail"); - } catch (IllegalArgumentException e) { - assertEquals(String.format(PreconditionMessage.ERR_BUFFER_STATE.toString(), 10, 5, 6), - e.getMessage()); - } - } - - /** - * Tests that writing a null buffer throws the correct exception. - */ - @Test - public void writeNullBuffer() throws IOException { - try { - mTestStream.write(null); - fail("writing null should fail"); - } catch (IllegalArgumentException e) { - assertEquals(PreconditionMessage.ERR_WRITE_BUFFER_NULL.toString(), e.getMessage()); - } - } - - /** - * Tests that writing a null buffer with offset/length information throws the correct exception. - */ - @Test - public void writeNullBufferOffset() throws IOException { - try { - mTestStream.write(null, 0, 0); - fail("writing null should fail"); - } catch (IllegalArgumentException e) { - assertEquals(PreconditionMessage.ERR_WRITE_BUFFER_NULL.toString(), e.getMessage()); - } - } - - /** - * Tests that the async write invokes the expected client APIs. - */ - @Test - public void asyncWrite() throws Exception { - OutStreamOptions options = - OutStreamOptions.defaults(mFileSystemContext).setBlockSizeBytes(BLOCK_LENGTH) - .setWriteType(WriteType.ASYNC_THROUGH); - mTestStream = createTestStream(FILE_NAME, options); - - mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); - mTestStream.close(); - - // Verify that async persist request is sent with complete file request. - ArgumentCaptor parameterCaptor = - ArgumentCaptor.forClass(CompleteFilePOptions.class); - verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), parameterCaptor.capture()); - Assert.assertTrue(parameterCaptor.getValue().hasAsyncPersistOptions()); - } - - /** - * Tests that common options are propagated to async write request. - */ - @Test - public void asyncWriteOptionPropagation() throws Exception { - Random rand = new Random(); - FileSystemMasterCommonPOptions commonOptions = - FileSystemMasterCommonPOptions.newBuilder().setTtl(rand.nextLong()) - .setTtlAction(TtlAction.values()[rand.nextInt(TtlAction.values().length)]) - .setSyncIntervalMs(rand.nextLong()).build(); - - OutStreamOptions options = - new OutStreamOptions(CreateFilePOptions.newBuilder().setWriteType(WritePType.ASYNC_THROUGH) - .setBlockSizeBytes(BLOCK_LENGTH).setCommonOptions(commonOptions).build(), - mFileSystemContext, sConf); - - // Verify that OutStreamOptions have captured the common options properly. - assertEquals(options.getCommonOptions(), commonOptions); - - mTestStream = createTestStream(FILE_NAME, options); - mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); - mTestStream.close(); - verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class)); - - // Verify that common options for OutStreamOptions are propagated to ScheduleAsyncPersistence. - ArgumentCaptor parameterCaptor = - ArgumentCaptor.forClass(CompleteFilePOptions.class); - - verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), parameterCaptor.capture()); - assertEquals(parameterCaptor.getValue().getAsyncPersistOptions().getCommonOptions(), - options.getCommonOptions()); - } - - /** - * Tests that the number of bytes written is correct when the stream is created with different - * under storage types. - */ - @Test - public void getBytesWrittenWithDifferentUnderStorageType() throws IOException { - for (WriteType type : WriteType.values()) { - OutStreamOptions options = - OutStreamOptions.defaults(mFileSystemContext).setBlockSizeBytes(BLOCK_LENGTH) - .setWriteType(type).setUfsPath(FILE_NAME.getPath()); - mTestStream = createTestStream(FILE_NAME, options); - mTestStream.write(BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH)); - mTestStream.flush(); - assertEquals(BLOCK_LENGTH, mTestStream.getBytesWritten()); - } - } - - @Test - public void createWithNoWorker() { - // The default 2 minutes is too long. - sConf.set(PropertyKey.USER_FILE_WRITE_INIT_MAX_DURATION, "10sec"); - mClientContext = ClientContext.create(sConf); - OutStreamOptions options = OutStreamOptions.defaults(mFileSystemContext) - .setLocationPolicy((getWorkerOptions) -> Optional.empty()) - .setWriteType(WriteType.CACHE_THROUGH); - Exception e = assertThrows(UnavailableException.class, - () -> mTestStream = createTestStream(FILE_NAME, options)); - assertTrue(e.getMessage().contains(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage())); - } - - private void verifyIncreasingBytesWritten(int len) { - verifyIncreasingBytesWritten(0, len); - } - - /** - * Verifies that the out streams have had exactly `len` increasing bytes written to them, with the - * first byte starting at `start`. Also verifies that the same bytes have been written to the - * under storage file stream. - */ - private void verifyIncreasingBytesWritten(int start, int len) { - long filledStreams = len / BLOCK_LENGTH; - for (long streamIndex = 0; streamIndex < filledStreams; streamIndex++) { - assertTrue("stream " + streamIndex + " was never written", - mAlluxioOutStreamMap.containsKey(streamIndex)); - assertArrayEquals(BufferUtils - .getIncreasingByteArray((int) (streamIndex * BLOCK_LENGTH + start), (int) BLOCK_LENGTH), - mAlluxioOutStreamMap.get(streamIndex).getWrittenData()); - } - long lastStreamBytes = len - filledStreams * BLOCK_LENGTH; - assertArrayEquals( - BufferUtils.getIncreasingByteArray((int) (filledStreams * BLOCK_LENGTH + start), - (int) lastStreamBytes), - mAlluxioOutStreamMap.get(filledStreams).getWrittenData()); - - assertArrayEquals(BufferUtils.getIncreasingByteArray(start, len), - mUnderStorageOutputStream.getWrittenData()); - } - - /** - * Creates a {@link FileOutStream} for test. - * - * @param path the file path - * @param options the set of options specific to this operation - * @return a {@link FileOutStream} - */ - private AlluxioFileOutStream createTestStream(AlluxioURI path, OutStreamOptions options) - throws IOException { - return new AlluxioFileOutStream(path, options, mFileSystemContext); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/MetadataCachingFileSystemTest.java b/core/client/fs/src/test/java/alluxio/client/file/MetadataCachingFileSystemTest.java deleted file mode 100644 index 6c2f5a0d56a7..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/MetadataCachingFileSystemTest.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file; - -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - -import alluxio.AlluxioURI; -import alluxio.ClientContext; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.FileDoesNotExistException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.NotFoundException; -import alluxio.grpc.CreateDirectoryPOptions; -import alluxio.grpc.CreateFilePOptions; -import alluxio.grpc.DeletePOptions; -import alluxio.grpc.GetStatusPOptions; -import alluxio.grpc.ListStatusPOptions; -import alluxio.grpc.RenamePOptions; -import alluxio.resource.CloseableResource; -import alluxio.wire.FileInfo; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({FileSystemContext.class}) -public class MetadataCachingFileSystemTest { - private static final AlluxioURI DIR = new AlluxioURI("/dir"); - private static final AlluxioURI FILE = new AlluxioURI("/dir/file"); - private static final AlluxioURI NOT_EXIST_FILE = new AlluxioURI("/dir/not_exist_file"); - private static final ListStatusPOptions LIST_STATUS_OPTIONS = - ListStatusPOptions.getDefaultInstance(); - private static final URIStatus FILE_STATUS = - new URIStatus(new FileInfo().setPath(FILE.getPath()).setCompleted(true)); - - private InstancedConfiguration mConf = Configuration.copyGlobal(); - private FileSystemContext mFileContext; - private ClientContext mClientContext; - private RpcCountingFileSystemMasterClient mFileSystemMasterClient; - private Map mFileStatusMap; - private MetadataCachingFileSystem mFs; - - @Before - public void before() throws Exception { - // Avoid async update file access time to call getStatus to mess up the test results - mConf.set(PropertyKey.USER_UPDATE_FILE_ACCESSTIME_DISABLED, true); - mClientContext = ClientContext.create(mConf); - mFileContext = PowerMockito.mock(FileSystemContext.class); - mFileSystemMasterClient = new RpcCountingFileSystemMasterClient(); - when(mFileContext.acquireMasterClientResource()) - .thenReturn(new CloseableResource(mFileSystemMasterClient) { - @Override - public void closeResource() { - // Noop. - } - }); - when(mFileContext.getClientContext()).thenReturn(mClientContext); - when(mFileContext.getClusterConf()).thenReturn(mConf); - when(mFileContext.getPathConf(any())).thenReturn(mConf); - when(mFileContext.getUriValidationEnabled()).thenReturn(true); - mFs = new MetadataCachingFileSystem(new BaseFileSystem(mFileContext), mFileContext); - mFileStatusMap = new HashMap<>(); - } - - @After - public void after() { - mConf = Configuration.copyGlobal(); - } - - @Test - public void getStatus() throws Exception { - mFs.getStatus(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - // The following getStatus gets from cache, so no RPC will be made. - mFs.getStatus(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - } - - @Test - public void iterateStatus() throws Exception { - List expectedStatuses = new ArrayList<>(); - mFs.iterateStatus(DIR, expectedStatuses::add); - assertEquals(1, mFileSystemMasterClient.listStatusRpcCount(DIR)); - // List status has cached the file status, so no RPC will be made. - mFs.getStatus(FILE); - assertEquals(0, mFileSystemMasterClient.getStatusRpcCount(FILE)); - List gotStatuses = new ArrayList<>(); - mFs.iterateStatus(DIR, gotStatuses::add); - // List status results have been cached, so listStatus RPC was only called once - // at the beginning of the method. - assertEquals(1, mFileSystemMasterClient.listStatusRpcCount(DIR)); - assertEquals(expectedStatuses, gotStatuses); - } - - @Test - public void iterateStatusRecursive() throws Exception { - mFs.iterateStatus(DIR, LIST_STATUS_OPTIONS.toBuilder().setRecursive(true).build(), ignored -> { - }); - assertEquals(1, mFileSystemMasterClient.listStatusRpcCount(DIR)); - mFs.iterateStatus(DIR, LIST_STATUS_OPTIONS.toBuilder().setRecursive(true).build(), ignored -> { - }); - assertEquals(2, mFileSystemMasterClient.listStatusRpcCount(DIR)); - } - - @Test - public void listStatus() throws Exception { - List expectedStatuses = mFs.listStatus(DIR); - assertEquals(1, mFileSystemMasterClient.listStatusRpcCount(DIR)); - // List status has cached the file status, so no RPC will be made. - mFs.getStatus(FILE); - assertEquals(0, mFileSystemMasterClient.getStatusRpcCount(FILE)); - List gotStatuses = mFs.listStatus(DIR); - // List status results have been cached, so listStatus RPC was only called once - // at the beginning of the method. - assertEquals(1, mFileSystemMasterClient.listStatusRpcCount(DIR)); - assertEquals(expectedStatuses, gotStatuses); - } - - @Test - public void listStatusRecursive() throws Exception { - mFs.listStatus(DIR, LIST_STATUS_OPTIONS.toBuilder().setRecursive(true).build()); - assertEquals(1, mFileSystemMasterClient.listStatusRpcCount(DIR)); - mFs.listStatus(DIR, LIST_STATUS_OPTIONS.toBuilder().setRecursive(true).build()); - assertEquals(2, mFileSystemMasterClient.listStatusRpcCount(DIR)); - } - - @Test - public void openFile() throws Exception { - mFs.openFile(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - mFs.openFile(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - } - - @Test - public void getBlockLocations() throws Exception { - mFs.getBlockLocations(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - mFs.getBlockLocations(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - } - - @Test - public void getNoneExistStatus() throws Exception { - try { - mFs.getStatus(NOT_EXIST_FILE); - Assert.fail("Failed while getStatus for a non-exist path."); - } catch (FileDoesNotExistException e) { - // expected exception thrown. test passes - } - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(NOT_EXIST_FILE)); - // The following getStatus gets from cache, so no RPC will be made. - try { - mFs.getStatus(NOT_EXIST_FILE); - Assert.fail("Failed while getStatus for a non-exist path."); - } catch (FileDoesNotExistException e) { - // expected exception thrown. test passes - } - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(NOT_EXIST_FILE)); - } - - @Test - public void createNoneExistFile() throws Exception { - try { - mFs.getStatus(NOT_EXIST_FILE); - Assert.fail("Failed while getStatus for a non-exist path."); - } catch (FileDoesNotExistException e) { - // expected exception thrown. test passes - } - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(NOT_EXIST_FILE)); - // The following getStatus gets from cache, so no RPC will be made. - mFs.createFile(NOT_EXIST_FILE); - mFs.getStatus(NOT_EXIST_FILE); - assertEquals(2, mFileSystemMasterClient.getStatusRpcCount(NOT_EXIST_FILE)); - } - - @Test - public void createNoneExistDirectory() throws Exception { - try { - mFs.getStatus(NOT_EXIST_FILE); - Assert.fail("Failed while getStatus for a non-exist path."); - } catch (FileDoesNotExistException e) { - // expected exception thrown. test passes - } - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(NOT_EXIST_FILE)); - // The following getStatus gets from cache, so no RPC will be made. - mFs.createDirectory(NOT_EXIST_FILE); - mFs.getStatus(NOT_EXIST_FILE); - assertEquals(2, mFileSystemMasterClient.getStatusRpcCount(NOT_EXIST_FILE)); - } - - @Test - public void createAndDelete() throws Exception { - mFs.createFile(NOT_EXIST_FILE); - mFs.getStatus(NOT_EXIST_FILE); - mFs.delete(NOT_EXIST_FILE); - try { - mFs.getStatus(NOT_EXIST_FILE); - Assert.fail("Failed while getStatus for a non-exist path."); - } catch (FileDoesNotExistException e) { - // expected exception thrown. test passes - } - assertEquals(2, mFileSystemMasterClient.getStatusRpcCount(NOT_EXIST_FILE)); - } - - @Test - public void createAndRename() throws Exception { - mFs.createFile(NOT_EXIST_FILE); - mFs.getStatus(NOT_EXIST_FILE); - mFs.rename(NOT_EXIST_FILE, new AlluxioURI(NOT_EXIST_FILE.getPath() + ".rename")); - try { - mFs.getStatus(NOT_EXIST_FILE); - Assert.fail("Failed while getStatus for a non-exist path."); - } catch (FileDoesNotExistException e) { - // expected exception thrown. test passes - } - assertEquals(2, mFileSystemMasterClient.getStatusRpcCount(NOT_EXIST_FILE)); - } - - @Test - public void dropMetadataCacheFile() throws Exception { - mFs.getStatus(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - mFs.getStatus(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - mFs.dropMetadataCache(FILE); - mFs.getStatus(FILE); - assertEquals(2, mFileSystemMasterClient.getStatusRpcCount(FILE)); - } - - @Test - public void dropMetadataCacheDir() throws Exception { - mFs.listStatus(DIR); - assertEquals(1, mFileSystemMasterClient.listStatusRpcCount(DIR)); - mFs.dropMetadataCache(DIR); - mFs.listStatus(DIR); - assertEquals(2, mFileSystemMasterClient.listStatusRpcCount(DIR)); - } - - @Test - public void dropMetadataCacheDirFile() throws Exception { - mFs.listStatus(DIR); - mFs.getStatus(FILE); - assertEquals(1, mFileSystemMasterClient.listStatusRpcCount(DIR)); - assertEquals(0, mFileSystemMasterClient.getStatusRpcCount(FILE)); - mFs.dropMetadataCache(FILE); - mFs.getStatus(FILE); - assertEquals(1, mFileSystemMasterClient.getStatusRpcCount(FILE)); - mFs.listStatus(DIR); - assertEquals(2, mFileSystemMasterClient.listStatusRpcCount(DIR)); - } - - class RpcCountingFileSystemMasterClient extends MockFileSystemMasterClient { - RpcCountingFileSystemMasterClient() { - } - - private Map mGetStatusCount = new HashMap<>(); - private Map mListStatusCount = new HashMap<>(); - - int getStatusRpcCount(AlluxioURI uri) { - return mGetStatusCount.getOrDefault(uri, 0); - } - - int listStatusRpcCount(AlluxioURI uri) { - return mListStatusCount.getOrDefault(uri, 0); - } - - @Override - public URIStatus getStatus(AlluxioURI path, GetStatusPOptions options) - throws AlluxioStatusException { - mGetStatusCount.compute(path, (k, v) -> v == null ? 1 : v + 1); - if (path.toString().equals(FILE_STATUS.getPath())) { - return FILE_STATUS; - } - if (mFileStatusMap.containsKey(path)) { - return mFileStatusMap.get(path); - } - throw new NotFoundException("Path \"" + path.getPath() + "\" does not exist."); - } - - @Override - public void iterateStatus(AlluxioURI path, ListStatusPOptions options, - Consumer action) throws AlluxioStatusException { - mListStatusCount.compute(path, (k, v) -> v == null ? 1 : v + 1); - action.accept(FILE_STATUS); - } - - @Override - public List listStatus(AlluxioURI path, ListStatusPOptions options) - throws AlluxioStatusException { - mListStatusCount.compute(path, (k, v) -> v == null ? 1 : v + 1); - return Arrays.asList(FILE_STATUS); - } - - @Override - public URIStatus createFile(AlluxioURI path, CreateFilePOptions options) - throws AlluxioStatusException { - URIStatus status = new URIStatus( - new FileInfo() - .setPath(NOT_EXIST_FILE.getPath()) - .setCompleted(true)); - mFileStatusMap.put(path, status); - return status; - } - - @Override - public void createDirectory(AlluxioURI path, - CreateDirectoryPOptions options) throws AlluxioStatusException { - URIStatus status = new URIStatus( - new FileInfo() - .setPath(NOT_EXIST_FILE.getPath()) - .setCompleted(true)); - mFileStatusMap.put(path, status); - } - - @Override - public void delete(AlluxioURI path, DeletePOptions options) - throws AlluxioStatusException { - mFileStatusMap.remove(path); - } - - @Override - public void rename(AlluxioURI src, AlluxioURI dst, RenamePOptions options) - throws AlluxioStatusException { - mFileStatusMap.put(dst, mFileStatusMap.remove(src)); - } - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/DefaultMetaStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/DefaultMetaStoreTest.java deleted file mode 100644 index 301c85965fc4..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/DefaultMetaStoreTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache; - -import static org.junit.Assert.assertThrows; - -import alluxio.client.file.cache.evictor.CacheEvictorOptions; -import alluxio.client.file.cache.evictor.FIFOCacheEvictor; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.client.file.cache.store.PageStoreOptions; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.exception.PageNotFoundException; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; - -import com.codahale.metrics.Gauge; -import com.google.common.collect.ImmutableList; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.nio.file.Paths; - -/** - * Tests for the {@link DefaultPageMetaStore} class. - */ -public class DefaultMetaStoreTest { - protected final PageId mPage = new PageId("1L", 2L); - protected final AlluxioConfiguration mConf = Configuration.global(); - protected PageStoreDir mPageStoreDir; - protected PageInfo mPageInfo; - protected DefaultPageMetaStore mMetaStore; - protected Gauge mCachedPageGauge; - - @Rule - public TemporaryFolder mTempFolder = new TemporaryFolder(); - - /** - * Sets up the instances. - */ - @Before - public void before() { - MetricsSystem.clearAllMetrics(); - mPageStoreDir = - PageStoreDir.createPageStoreDir( - new CacheEvictorOptions().setEvictorClass(FIFOCacheEvictor.class), - new PageStoreOptions().setRootDir( - Paths.get(mTempFolder.getRoot().getAbsolutePath()))); - mPageInfo = new PageInfo(mPage, 1024, - mPageStoreDir); - mMetaStore = new DefaultPageMetaStore(ImmutableList.of(mPageStoreDir)); - mCachedPageGauge = - MetricsSystem.METRIC_REGISTRY.getGauges().get(MetricKey.CLIENT_CACHE_PAGES.getName()); - } - - @Test - public void addNew() { - mMetaStore.addPage(mPage, mPageInfo); - Assert.assertTrue(mMetaStore.hasPage(mPage)); - Assert.assertEquals(1, mCachedPageGauge.getValue()); - } - - @Test - public void addExist() { - mMetaStore.addPage(mPage, mPageInfo); - mMetaStore.addPage(mPage, mPageInfo); - Assert.assertTrue(mMetaStore.hasPage(mPage)); - Assert.assertEquals(1, mCachedPageGauge.getValue()); - } - - @Test - public void removeExist() throws Exception { - mMetaStore.addPage(mPage, mPageInfo); - Assert.assertTrue(mMetaStore.hasPage(mPage)); - mMetaStore.removePage(mPage); - Assert.assertFalse(mMetaStore.hasPage(mPage)); - Assert.assertEquals(0, mCachedPageGauge.getValue()); - } - - @Test - public void removeNotExist() throws Exception { - assertThrows(PageNotFoundException.class, () -> { - Assert.assertEquals(mPageInfo, mMetaStore.removePage(mPage)); - }); - Assert.assertEquals(0, mCachedPageGauge.getValue()); - } - - @Test - public void hasPage() { - Assert.assertFalse(mMetaStore.hasPage(mPage)); - mMetaStore.addPage(mPage, mPageInfo); - Assert.assertTrue(mMetaStore.hasPage(mPage)); - } - - @Test - public void getPageInfo() throws Exception { - mMetaStore.addPage(mPage, mPageInfo); - Assert.assertEquals(mPageInfo, mMetaStore.getPageInfo(mPage)); - } - - @Test - public void getPageInfoNotExist() throws Exception { - assertThrows(PageNotFoundException.class, () -> mMetaStore.getPageInfo(mPage)); - } - - @Test - public void evict() throws Exception { - mMetaStore.addPage(mPage, mPageInfo); - Assert.assertEquals(mPageInfo, mMetaStore.evict(mPageStoreDir)); - mMetaStore.removePage(mPageInfo.getPageId()); - Assert.assertNull(mMetaStore.evict(mPageStoreDir)); - Assert.assertEquals(0, mCachedPageGauge.getValue()); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/QuotaPageMetaStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/QuotaPageMetaStoreTest.java deleted file mode 100644 index 01d620dfe292..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/QuotaPageMetaStoreTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import alluxio.client.file.cache.evictor.CacheEvictorOptions; -import alluxio.client.file.cache.evictor.FIFOCacheEvictor; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.client.file.cache.store.PageStoreOptions; -import alluxio.client.quota.CacheScope; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; - -import com.google.common.collect.ImmutableList; -import org.junit.Before; -import org.junit.Test; - -import java.nio.file.Paths; - -/** - * Tests for the {@link QuotaPageMetaStore} class. - */ -public class QuotaPageMetaStoreTest extends DefaultMetaStoreTest { - private final AlluxioConfiguration mConf = Configuration.global(); - private final CacheScope mPartitionScope = CacheScope.create("schema.table.partition"); - private final CacheScope mTableScope = CacheScope.create("schema.table"); - private final CacheScope mSchemaScope = CacheScope.create("schema"); - private final long mPageSize = 8765; - - private QuotaPageMetaStore mQuotaMetaStore; - - @Before - public void before() { - MetricsSystem.clearAllMetrics(); - CacheEvictorOptions evictorOptions = - new CacheEvictorOptions().setEvictorClass(FIFOCacheEvictor.class); - mPageStoreDir = - PageStoreDir.createPageStoreDir(evictorOptions, - new PageStoreOptions().setRootDir( - Paths.get(mTempFolder.getRoot().getAbsolutePath()))); - mPageInfo = new PageInfo(mPage, 1024, - mPageStoreDir); - mMetaStore = new QuotaPageMetaStore(evictorOptions, ImmutableList.of(mPageStoreDir)); - mQuotaMetaStore = (QuotaPageMetaStore) mMetaStore; - mCachedPageGauge = - MetricsSystem.METRIC_REGISTRY.getGauges().get(MetricKey.CLIENT_CACHE_PAGES.getName()); - } - - @Test - public void evictInScope() throws Exception { - assertNull(mQuotaMetaStore.evict(CacheScope.GLOBAL, mPageStoreDir)); - PageInfo pageInfo = new PageInfo(mPage, mPageSize, mSchemaScope, mPageStoreDir); - mQuotaMetaStore.addPage(mPage, pageInfo); - assertNull(mQuotaMetaStore.evict(mPartitionScope, mPageStoreDir)); - assertNull(mQuotaMetaStore.evict(mTableScope, mPageStoreDir)); - assertEquals(pageInfo, mQuotaMetaStore.evict(mSchemaScope, mPageStoreDir)); - assertEquals(pageInfo, mQuotaMetaStore.evict(CacheScope.GLOBAL, mPageStoreDir)); - } - - @Test - public void evictInScope2() throws Exception { - CacheScope partitionScope1 = CacheScope.create("schema.table.partition1"); - CacheScope partitionScope2 = CacheScope.create("schema.table.partition2"); - PageId pageId1 = new PageId("1L", 2L); - PageId pageId2 = new PageId("3L", 4L); - PageInfo pageInfo1 = new PageInfo(pageId1, 1234, partitionScope1, mPageStoreDir); - PageInfo pageInfo2 = new PageInfo(pageId2, 5678, partitionScope2, mPageStoreDir); - mQuotaMetaStore.addPage(pageId1, pageInfo1); - mQuotaMetaStore.addPage(pageId2, pageInfo2); - assertEquals(pageInfo1, mQuotaMetaStore.evict(partitionScope1, mPageStoreDir)); - assertEquals(pageInfo2, mQuotaMetaStore.evict(partitionScope2, mPageStoreDir)); - PageInfo evicted = mQuotaMetaStore.evict(mTableScope, mPageStoreDir); - assertTrue(evicted == pageInfo1 || evicted == pageInfo2); - evicted = mQuotaMetaStore.evict(mSchemaScope, mPageStoreDir); - assertTrue(evicted == pageInfo1 || evicted == pageInfo2); - evicted = mQuotaMetaStore.evict(CacheScope.GLOBAL, mPageStoreDir); - assertTrue(evicted == pageInfo1 || evicted == pageInfo2); - mQuotaMetaStore.removePage(pageId1); - assertNull(mQuotaMetaStore.evict(partitionScope1, mPageStoreDir)); - assertEquals(pageInfo2, mQuotaMetaStore.evict(partitionScope2, mPageStoreDir)); - assertEquals(pageInfo2, mQuotaMetaStore.evict(mTableScope, mPageStoreDir)); - assertEquals(pageInfo2, mQuotaMetaStore.evict(mSchemaScope, mPageStoreDir)); - assertEquals(pageInfo2, mQuotaMetaStore.evict(CacheScope.GLOBAL, mPageStoreDir)); - } - - @Test - public void bytesInScope() throws Exception { - PageInfo pageInfo = new PageInfo(mPage, mPageSize, mPartitionScope, mPageStoreDir); - mQuotaMetaStore.addPage(mPage, pageInfo); - assertEquals(mPageSize, mQuotaMetaStore.bytes(mPartitionScope)); - assertEquals(mPageSize, mQuotaMetaStore.bytes(mTableScope)); - assertEquals(mPageSize, mQuotaMetaStore.bytes(mSchemaScope)); - assertEquals(mPageSize, mQuotaMetaStore.bytes(CacheScope.GLOBAL)); - mQuotaMetaStore.removePage(mPage); - assertEquals(0, mQuotaMetaStore.bytes(mPartitionScope)); - assertEquals(0, mQuotaMetaStore.bytes(mTableScope)); - assertEquals(0, mQuotaMetaStore.bytes(mSchemaScope)); - assertEquals(0, mQuotaMetaStore.bytes(CacheScope.GLOBAL)); - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java deleted file mode 100644 index 3c6e768f5903..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.cache.store; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import alluxio.ProjectConstants; -import alluxio.client.file.cache.CacheManagerOptions; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageInfo; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.util.io.BufferUtils; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; - -@RunWith(Parameterized.class) -public class PageStoreDirTest { - private final AlluxioConfiguration mConf = Configuration.global(); - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList(new Object[][] { - {PageStoreType.ROCKS}, - {PageStoreType.LOCAL}, - {PageStoreType.MEM} - }); - } - - @Parameterized.Parameter - public PageStoreType mPageStoreType; - - private PageStoreOptions mOptions; - - @Rule - public TemporaryFolder mTemp = new TemporaryFolder(); - - private PageStoreDir mPageStoreDir; - - @Before - public void before() throws Exception { - CacheManagerOptions cacheManagerOptions = CacheManagerOptions.create(mConf); - mOptions = cacheManagerOptions.getPageStoreOptions().get(0); - mOptions.setStoreType(mPageStoreType); - mOptions.setPageSize(1024); - mOptions.setCacheSize(65536); - mOptions.setAlluxioVersion(ProjectConstants.VERSION); - mOptions.setRootDir(Paths.get(mTemp.getRoot().getAbsolutePath())); - - mPageStoreDir = - PageStoreDir.createPageStoreDir(cacheManagerOptions.getCacheEvictorOptions(), mOptions); - } - - @After - public void after() throws Exception { - if (mPageStoreDir != null) { - mPageStoreDir.close(); - } - } - - @Test - public void getPages() throws Exception { - int len = 32; - int count = 16; - byte[] data = BufferUtils.getIncreasingByteArray(len); - Set pages = new HashSet<>(count); - for (int i = 0; i < count; i++) { - PageId id = new PageId("0", i); - mPageStoreDir.getPageStore().put(id, data); - pages.add(new PageInfo(id, data.length, mPageStoreDir)); - } - Set restored = new HashSet<>(); - mPageStoreDir.scanPages((pageInfo -> restored.add(pageInfo.get()))); - if (mOptions.getType().equals(PageStoreType.MEM)) { - assertTrue(restored.isEmpty()); - } else { - assertEquals(pages, restored); - } - } - - @Test - public void getPagesUUID() throws Exception { - int len = 32; - int count = 16; - byte[] data = BufferUtils.getIncreasingByteArray(len); - Set pages = new HashSet<>(count); - for (int i = 0; i < count; i++) { - PageId id = new PageId(UUID.randomUUID().toString(), i); - mPageStoreDir.getPageStore().put(id, data); - pages.add(new PageInfo(id, data.length, mPageStoreDir)); - } - Set restored = new HashSet<>(); - mPageStoreDir.scanPages((pageInfo -> restored.add(pageInfo.get()))); - if (mOptions.getType().equals(PageStoreType.MEM)) { - assertTrue(restored.isEmpty()); - } else { - assertEquals(pages, restored); - } - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/ufs/UfsFileInStreamTest.java b/core/client/fs/src/test/java/alluxio/client/file/ufs/UfsFileInStreamTest.java deleted file mode 100644 index eaafd5d76849..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/ufs/UfsFileInStreamTest.java +++ /dev/null @@ -1,502 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.ufs; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import alluxio.AlluxioTestDirectory; -import alluxio.conf.Configuration; -import alluxio.exception.AlluxioException; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.underfs.UfsFileStatus; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.UnderFileSystemFactoryRegistry; -import alluxio.underfs.local.LocalUnderFileSystemFactory; -import alluxio.underfs.options.DeleteOptions; -import alluxio.underfs.options.OpenOptions; -import alluxio.util.io.BufferUtils; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.file.Paths; -import java.util.Random; -import java.util.UUID; - -/** - * Add unit tests for {@link UfsFileInStream}. - */ -public class UfsFileInStreamTest { - private static final int CHUNK_SIZE = 100; - private String mRootUfs; - private UnderFileSystem mUfs; - - /** - * Sets up the file system and the context before a test runs. - */ - @Before - public void before() { - mRootUfs = AlluxioTestDirectory.createTemporaryDirectory("ufs").toString(); - UnderFileSystemFactoryRegistry.register(new LocalUnderFileSystemFactory()); - mUfs = UnderFileSystem.Factory.create(mRootUfs, Configuration.global()); - } - - @After - public void after() throws IOException, AlluxioException { - mUfs.deleteDirectory(mRootUfs, DeleteOptions.defaults().setRecursive(true)); - } - - @Test - public void readWithNullUfsStream() { - assertThrows(NullPointerException.class, - () -> new UfsFileInStream(null, 0L).close()); - } - - @Test - public void openClose() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, 0); - getStream(ufsPath).close(); - } - - @Test - public void singleByteRead() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, 1); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(0, inStream.read()); - } - } - - @Test - public void twoBytesRead() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, 2); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(0, inStream.read()); - assertEquals(1, inStream.read()); - } - } - - @Test - public void manyBytesRead() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - byte[] res = new byte[CHUNK_SIZE]; - assertEquals(CHUNK_SIZE, inStream.read(res)); - assertTrue(BufferUtils.equalIncreasingByteArray(CHUNK_SIZE, res)); - } - } - - @Test - public void manyBytesReadByteBuffer() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - ByteBuffer buffer = ByteBuffer.allocate(CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(CHUNK_SIZE, inStream.read(buffer)); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, CHUNK_SIZE, buffer)); - } - } - - @Test - public void readAll() throws IOException { - int len = CHUNK_SIZE * 5; - int start = 0; - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE * 5); - byte[] res = new byte[CHUNK_SIZE]; - try (UfsFileInStream inStream = getStream(ufsPath)) { - while (start < len) { - assertEquals(CHUNK_SIZE, inStream.read(res)); - assertTrue(BufferUtils.equalIncreasingByteArray(start, CHUNK_SIZE, res)); - start += CHUNK_SIZE; - } - } - } - - @Test - public void readAllByteBuffer() throws IOException { - int len = CHUNK_SIZE * 5; - int start = 0; - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE * 5); - ByteBuffer buffer = ByteBuffer.allocate(CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - while (start < len) { - assertEquals(CHUNK_SIZE, inStream.read(buffer)); - assertTrue(BufferUtils.equalIncreasingByteBuffer(start, CHUNK_SIZE, buffer)); - start += CHUNK_SIZE; - buffer.clear(); - } - } - } - - @Test - public void readOffset() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - int start = CHUNK_SIZE / 4; - int len = CHUNK_SIZE / 2; - try (UfsFileInStream inStream = getStream(ufsPath)) { - byte[] res = new byte[CHUNK_SIZE]; - assertEquals(CHUNK_SIZE / 2, inStream.read(res, start, len)); - for (int i = start; i < start + len; i++) { - assertEquals(i - start, res[i]); - } - } - } - - @Test - public void readOffsetByteBuffer() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - int start = CHUNK_SIZE / 4; - int len = CHUNK_SIZE / 2; - try (UfsFileInStream inStream = getStream(ufsPath)) { - ByteBuffer buffer = ByteBuffer.allocate(CHUNK_SIZE); - assertEquals(CHUNK_SIZE / 2, inStream.read(buffer, start, len)); - for (int i = start; i < start + len; i++) { - assertEquals(i - start, buffer.get(i)); - } - } - } - - @Test - public void readOutOfBound() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - byte[] res = new byte[CHUNK_SIZE * 2]; - assertEquals(CHUNK_SIZE, inStream.read(res)); - assertTrue(BufferUtils.matchIncreasingByteArray(0, CHUNK_SIZE, res)); - assertEquals(-1, inStream.read(res)); - } - } - - @Test - public void readOutOfBoundByteBuffer() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - ByteBuffer buffer = ByteBuffer.allocate(CHUNK_SIZE * 2); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(CHUNK_SIZE, inStream.read(buffer)); - assertTrue(BufferUtils.matchIncreasingByteBuffer(0, CHUNK_SIZE, buffer)); - assertEquals(-1, inStream.read(buffer)); - } - } - - @Test - public void readOverflowOffLen() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertThrows(IllegalArgumentException.class, - () -> inStream.read(new byte[CHUNK_SIZE], 0, CHUNK_SIZE * 2)); - } - } - - @Test - public void readOverflowOffLenByteBuffer() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertThrows(IllegalArgumentException.class, - () -> inStream.read(ByteBuffer.allocate(CHUNK_SIZE), 0, CHUNK_SIZE * 2)); - } - } - - @Test - public void readNullArray() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertThrows(NullPointerException.class, - () -> inStream.read((byte[]) null)); - } - } - - @Test - public void readNullBuffer() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertThrows(NullPointerException.class, - () -> inStream.read((ByteBuffer) null, 0, CHUNK_SIZE)); - } - } - - @Test - public void readNullArrayOffset() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertThrows(NullPointerException.class, - () -> inStream.read((byte[]) null, 0, CHUNK_SIZE)); - } - } - - @Test - public void readNullBufferOffset() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertThrows(NullPointerException.class, - () -> inStream.read((ByteBuffer) null, 0, CHUNK_SIZE)); - } - } - - @Test - public void positionedRead() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - byte[] res = new byte[CHUNK_SIZE / 2]; - assertEquals(CHUNK_SIZE / 2, - inStream.positionedRead(CHUNK_SIZE / 2, res, 0, CHUNK_SIZE / 2)); - assertTrue(BufferUtils.equalIncreasingByteArray(CHUNK_SIZE / 2, CHUNK_SIZE / 2, res)); - } - } - - @Test - public void positionedReadMulti() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - Random random = new Random(); - try (UfsFileInStream inStream = getStream(ufsPath)) { - for (int i = 0; i < 10; i++) { - int pos = random.nextInt(CHUNK_SIZE); - int len = CHUNK_SIZE - pos; - byte[] res = new byte[len]; - assertEquals(len, - inStream.positionedRead(pos, res, 0, len)); - assertTrue(BufferUtils.equalIncreasingByteArray(pos, len, res)); - } - } - } - - @Test - public void seekForward() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - Random random = new Random(); - int pos = 0; - try (UfsFileInStream inStream = getStream(ufsPath)) { - for (int i = 0; i < 10; i++) { - pos += random.nextInt(CHUNK_SIZE - pos); - inStream.seek(pos); - assertEquals(pos, inStream.getPos()); - int len = CHUNK_SIZE - pos; - byte[] res = new byte[len]; - assertEquals(len, - inStream.read(res, 0, len)); - assertTrue(BufferUtils.equalIncreasingByteArray(pos, len, res)); - if (CHUNK_SIZE == pos) { - break; - } - } - } - } - - @Test - public void seekBackward() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - Random random = new Random(); - int pos = CHUNK_SIZE - 1; - try (UfsFileInStream inStream = getStream(ufsPath)) { - for (int i = 0; i < 10; i++) { - pos -= random.nextInt(pos); - inStream.seek(pos); - assertEquals(pos, inStream.getPos()); - int len = CHUNK_SIZE - pos; - byte[] res = new byte[len]; - assertEquals(len, - inStream.read(res, 0, len)); - assertTrue(BufferUtils.equalIncreasingByteArray(pos, len, res)); - if (pos <= 0) { - break; - } - } - } - } - - @Test - public void seekToBeginning() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - byte[] res = new byte[CHUNK_SIZE]; - assertEquals(CHUNK_SIZE, inStream.read(res)); - assertTrue(BufferUtils.equalIncreasingByteArray(CHUNK_SIZE, res)); - inStream.seek(0); - assertEquals(0, inStream.getPos()); - assertEquals(CHUNK_SIZE, inStream.read(res)); - assertTrue(BufferUtils.equalIncreasingByteArray(CHUNK_SIZE, res)); - } - } - - @Test - public void seekForwardAndBackward() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - Random random = new Random(); - try (UfsFileInStream inStream = getStream(ufsPath)) { - for (int i = 0; i < 10; i++) { - int pos = random.nextInt(CHUNK_SIZE); - inStream.seek(pos); - assertEquals(pos, inStream.getPos()); - int len = CHUNK_SIZE - pos; - byte[] res = new byte[len]; - assertEquals(len, - inStream.read(res, 0, len)); - assertTrue(BufferUtils.equalIncreasingByteArray(pos, len, res)); - } - } - } - - @Test - public void seekPassEnd() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertThrows(IllegalArgumentException.class, () -> inStream.seek(CHUNK_SIZE + 1)); - } - } - - @Test - public void seekNegative() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertThrows(IllegalArgumentException.class, () -> inStream.seek(-1)); - } - } - - @Test - public void skip() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - Random random = new Random(); - try (UfsFileInStream inStream = getStream(ufsPath)) { - for (int i = 0; i < 10; i++) { - if (inStream.remaining() <= 0) { - break; - } - int skip = random.nextInt((int) inStream.remaining()); - assertEquals(skip, inStream.skip(skip)); - assertEquals(skip, inStream.getPos()); - int len = CHUNK_SIZE - skip; - byte[] res = new byte[len]; - assertEquals(len, - inStream.read(res, 0, len)); - assertTrue(BufferUtils.equalIncreasingByteArray(skip, len, res)); - } - } - } - - @Test - public void skipToEnd() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(CHUNK_SIZE, inStream.skip(CHUNK_SIZE)); - assertEquals(-1, inStream.read()); - } - } - - @Test - public void skipPassEnd() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(CHUNK_SIZE, inStream.skip(CHUNK_SIZE + 1)); - Assert.assertEquals(-1, inStream.read()); - } - } - - @Test - public void skipNegative() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(0, inStream.skip(-1)); - Assert.assertEquals(0, inStream.read()); - } - } - - @Test - public void getPosition() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(0, inStream.getPos()); - inStream.read(); - assertEquals(1, inStream.getPos()); - int len = CHUNK_SIZE / 2; - inStream.read(new byte[len], 0, len); - assertEquals(1 + len, inStream.getPos()); - len = CHUNK_SIZE / 4; - inStream.read(ByteBuffer.allocate(len), 0, len); - assertEquals(1 + CHUNK_SIZE / 4 * 3, inStream.getPos()); - } - } - - @Test - public void remaining() throws IOException { - String ufsPath = getUfsPath(); - createFile(ufsPath, CHUNK_SIZE); - try (UfsFileInStream inStream = getStream(ufsPath)) { - assertEquals(CHUNK_SIZE, inStream.remaining()); - inStream.read(); - assertEquals(CHUNK_SIZE - 1, inStream.remaining()); - int len = CHUNK_SIZE / 2; - inStream.read(new byte[len], 0, len); - assertEquals(CHUNK_SIZE - len - 1, inStream.remaining()); - len = CHUNK_SIZE / 4; - inStream.read(ByteBuffer.allocate(len), 0, len); - assertEquals(CHUNK_SIZE / 4 - 1, inStream.remaining()); - } - } - - private String getUfsPath() { - return Paths.get(mRootUfs, String.valueOf(UUID.randomUUID())).toString(); - } - - private UfsFileInStream getStream(String ufsPath) throws IOException { - return new UfsFileInStream(offset -> { - try { - return mUfs.open(ufsPath, OpenOptions.defaults().setOffset(offset)); - } catch (IOException e) { - throw AlluxioRuntimeException.from(e); - } - }, ((UfsFileStatus) mUfs.getStatus(ufsPath)).getContentLength()); - } - - private void createFile(String ufsPath, int len) throws IOException { - createFile(ufsPath, 0, len); - } - - private void createFile(String ufsPath, int start, int len) throws IOException { - try (OutputStream outStream = mUfs.create(ufsPath)) { - outStream.write(BufferUtils.getIncreasingByteArray(start, len)); - } - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/file/ufs/UfsFileOutStreamTest.java b/core/client/fs/src/test/java/alluxio/client/file/ufs/UfsFileOutStreamTest.java deleted file mode 100644 index f62ab0d77865..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/file/ufs/UfsFileOutStreamTest.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.file.ufs; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import alluxio.AlluxioTestDirectory; -import alluxio.conf.Configuration; -import alluxio.exception.AlluxioException; -import alluxio.underfs.UfsFileStatus; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.UnderFileSystemFactoryRegistry; -import alluxio.underfs.local.LocalUnderFileSystemFactory; -import alluxio.underfs.options.DeleteOptions; -import alluxio.util.io.BufferUtils; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Paths; -import java.util.UUID; - -/** - * Add unit tests for {@link UfsFileOutStream}. - */ -public class UfsFileOutStreamTest { - private static final int CHUNK_SIZE = 128; - private String mRootUfs; - private UnderFileSystem mUfs; - - /** - * Sets up the file system and the context before a test runs. - */ - @Before - public void before() { - mRootUfs = AlluxioTestDirectory.createTemporaryDirectory("ufs").toString(); - UnderFileSystemFactoryRegistry.register(new LocalUnderFileSystemFactory()); - mUfs = UnderFileSystem.Factory.create(mRootUfs, Configuration.global()); - } - - @After - public void after() throws IOException, AlluxioException { - mUfs.deleteDirectory(mRootUfs, DeleteOptions.defaults().setRecursive(true)); - } - - @Test - public void createClose() throws IOException { - String ufsPath = getUfsPath(); - new UfsFileOutStream(mUfs.create(ufsPath)).close(); - assertTrue(mUfs.getStatus(ufsPath).isFile()); - assertEquals(0L, ((UfsFileStatus) mUfs.getStatus(ufsPath)).getContentLength()); - } - - @Test - public void singleByteWrite() throws IOException { - byte byteToWrite = 5; - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - outStream.write(byteToWrite); - } - try (InputStream inputStream = mUfs.open(ufsPath)) { - assertEquals(byteToWrite, inputStream.read()); - } - } - - @Test - public void writeIncreasingBytes() throws IOException { - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - for (int i = 0; i < CHUNK_SIZE; i++) { - outStream.write(i); - } - } - verifyIncreasingBytesWritten(ufsPath, CHUNK_SIZE); - } - - @Test - public void writeIncreasingByteArray() throws IOException { - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - outStream.write(BufferUtils.getIncreasingByteArray(CHUNK_SIZE)); - } - verifyIncreasingBytesWritten(ufsPath, CHUNK_SIZE); - } - - @Test - public void writeIncreasingByteArrayOffsetLen() throws IOException { - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - outStream.write(BufferUtils.getIncreasingByteArray(CHUNK_SIZE), 0, CHUNK_SIZE); - } - verifyIncreasingBytesWritten(ufsPath, CHUNK_SIZE); - } - - @Test - public void writePartialIncreasingByteArray() throws IOException { - int offset = CHUNK_SIZE / 2; - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - outStream.write(BufferUtils.getIncreasingByteArray(CHUNK_SIZE), offset, CHUNK_SIZE / 2); - } - verifyIncreasingBytesWritten(ufsPath, offset, CHUNK_SIZE / 2); - } - - @Test - public void writeOffset() throws IOException { - int bytesToWrite = CHUNK_SIZE * 5 + CHUNK_SIZE / 2; - int offset = CHUNK_SIZE / 3; - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - byte[] array = BufferUtils.getIncreasingByteArray(bytesToWrite + offset); - outStream.write(array, offset, bytesToWrite); - } - verifyIncreasingBytesWritten(ufsPath, offset, bytesToWrite); - } - - @Test - public void writeWithNullUfsStream() { - Assert.assertThrows(NullPointerException.class, - () -> new UfsFileOutStream(null).close()); - } - - @Test - public void writeOverflowOffLen() throws IOException { - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - assertThrows(IllegalArgumentException.class, () -> - outStream.write(BufferUtils.getIncreasingByteArray(CHUNK_SIZE), 5, CHUNK_SIZE + 5)); - } - } - - @Test - public void writeNullByteArray() throws IOException { - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - assertThrows(IllegalArgumentException.class, () -> - outStream.write(null)); - } - } - - @Test - public void getBytesWrittenWhenWrite() throws IOException { - String ufsPath = getUfsPath(); - try (UfsFileOutStream outStream = new UfsFileOutStream(mUfs.create(ufsPath))) { - outStream.write(BufferUtils.getIncreasingByteArray(CHUNK_SIZE), 0, CHUNK_SIZE); - assertEquals(CHUNK_SIZE, outStream.getBytesWritten()); - outStream.write(BufferUtils.getIncreasingByteArray(CHUNK_SIZE, CHUNK_SIZE), 0, CHUNK_SIZE); - assertEquals(CHUNK_SIZE * 2, outStream.getBytesWritten()); - } - verifyIncreasingBytesWritten(ufsPath, CHUNK_SIZE * 2); - } - - private String getUfsPath() { - return Paths.get(mRootUfs, String.valueOf(UUID.randomUUID())).toString(); - } - - private void verifyIncreasingBytesWritten(String ufs, int len) throws IOException { - verifyIncreasingBytesWritten(ufs, 0, len); - } - - private void verifyIncreasingBytesWritten(String ufs, int start, int len) throws IOException { - int block = 128; - byte[] array = new byte[Math.min(block, len)]; - int curToRead; - try (InputStream inputStream = mUfs.open(ufs)) { - while (len > 0) { - curToRead = Math.min(len, block); - int read = inputStream.read(array, 0, curToRead); - Assert.assertTrue(read > 0); - Assert.assertTrue(BufferUtils.matchIncreasingByteArray(start, 0, read, array)); - len -= read; - start += read; - } - } - } -} diff --git a/core/client/fs/src/test/java/alluxio/client/util/ClientTestUtils.java b/core/client/fs/src/test/java/alluxio/client/util/ClientTestUtils.java deleted file mode 100644 index 8d033a90b10e..000000000000 --- a/core/client/fs/src/test/java/alluxio/client/util/ClientTestUtils.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.util; - -import alluxio.Constants; -import alluxio.client.block.BlockWorkerInfo; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.wire.TieredIdentity; -import alluxio.wire.TieredIdentity.LocalityTier; -import alluxio.wire.WorkerNetAddress; - -import java.util.ArrayList; -import java.util.List; - -/** - * Utility methods for the client tests. - */ -public final class ClientTestUtils { - - /** - * Sets small buffer sizes so that Alluxio does not run out of heap space. - */ - public static void setSmallBufferSizes(InstancedConfiguration conf) { - conf.set(PropertyKey.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES, "4KB"); - conf.set(PropertyKey.USER_FILE_BUFFER_BYTES, "4KB"); - } - - /** - * Resets the client to its initial state, re-initializing Alluxio contexts. - * - * This method should only be used as a cleanup mechanism between tests. - */ - public static void resetClient(InstancedConfiguration conf) { - try { - resetContexts(conf); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private static void resetContexts(InstancedConfiguration conf) { - conf.set(PropertyKey.USER_METRICS_COLLECTION_ENABLED, false); - } - - public static BlockWorkerInfo worker(long capacity, String node, String rack) { - return worker(capacity, 0, node, rack); - } - - public static BlockWorkerInfo worker(long capacity, long used, String node, String rack) { - WorkerNetAddress address = new WorkerNetAddress(); - List tiers = new ArrayList<>(); - if (node != null && !node.isEmpty()) { - address.setHost(node); - tiers.add(new LocalityTier(Constants.LOCALITY_NODE, node)); - } - if (rack != null && !rack.isEmpty()) { - tiers.add(new LocalityTier(Constants.LOCALITY_RACK, rack)); - } - address.setTieredIdentity(new TieredIdentity(tiers)); - return new BlockWorkerInfo(address, capacity, used); - } -} diff --git a/core/client/hdfs/pom.xml b/core/client/hdfs/pom.xml deleted file mode 100644 index 86ee0410727f..000000000000 --- a/core/client/hdfs/pom.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - 4.0.0 - - org.alluxio - alluxio-core-client - 2.10.0-SNAPSHOT - - alluxio-core-client-hdfs - jar - Alluxio Core - Client - HDFS - HDFS Client of Alluxio Core - - - - - ${project.parent.parent.parent.basedir}/build - false - - - - - - com.google.guava - guava - - - org.apache.hadoop - hadoop-client - - - - - org.alluxio - alluxio-core-client-fs - ${project.version} - - - org.alluxio - alluxio-core-common - ${project.version} - - - org.alluxio - alluxio-core-transport - ${project.version} - - - - - org.alluxio - alluxio-core-common - ${project.version} - test-jar - test - - - - - - hadoop-1 - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - **/AlluxioFileSystem.java - - - - - - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - - diff --git a/core/client/hdfs/src/main/java/alluxio/hadoop/AlluxioHdfsInputStream.java b/core/client/hdfs/src/main/java/alluxio/hadoop/AlluxioHdfsInputStream.java deleted file mode 100644 index 581ae7e8de95..000000000000 --- a/core/client/hdfs/src/main/java/alluxio/hadoop/AlluxioHdfsInputStream.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.hadoop; - -import alluxio.client.file.FileInStream; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.FSDataInputStream; - -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * A wrapper class to translate Hadoop FileSystem FSDataInputStream to Alluxio FileSystem - * FileInStream. - */ -public class AlluxioHdfsInputStream extends FileInStream { - private final FSDataInputStream mInput; - - /** - * @param input Hadoop FileSystem FSDataInputStream - */ - public AlluxioHdfsInputStream(FSDataInputStream input) { - mInput = Preconditions.checkNotNull(input, "null"); - } - - @Override - public int read(byte[] bytes) throws IOException { - return mInput.read(bytes); - } - - @Override - public int read(byte[] bytes, int offset, int length) throws IOException { - return mInput.read(bytes, offset, length); - } - - @Override - public int read() throws IOException { - return mInput.read(); - } - - @Override - public int read(ByteBuffer buf) throws IOException { - return mInput.read(buf); - } - - @Override - public long skip(long length) throws IOException { - return mInput.skip(length); - } - - @Override - public int available() throws IOException { - return mInput.available(); - } - - @Override - public void close() throws IOException { - mInput.close(); - } - - @Override - public synchronized void mark(int limit) { - mInput.mark(limit); - } - - @Override - public synchronized void reset() throws IOException { - mInput.reset(); - } - - @Override - public boolean markSupported() { - return mInput.markSupported(); - } - - @Override - public void seek(long position) throws IOException { - mInput.seek(position); - } - - @Override - public long getPos() throws IOException { - return mInput.getPos(); - } - - // TODO(binfan): implement this method - @Override - public long remaining() { - throw new UnsupportedOperationException("Remaining is not supported"); - } - - @Override - public int positionedRead(long position, byte[] buffer, int offset, int length) - throws IOException { - return mInput.read(position, buffer, offset, length); - } -} diff --git a/core/client/hdfs/src/main/java/alluxio/hadoop/FileSystem.java b/core/client/hdfs/src/main/java/alluxio/hadoop/FileSystem.java deleted file mode 100644 index 65cf0873e5b7..000000000000 --- a/core/client/hdfs/src/main/java/alluxio/hadoop/FileSystem.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.hadoop; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.client.file.URIStatus; -import alluxio.conf.PropertyKey; -import alluxio.uri.Authority; -import alluxio.uri.EmbeddedLogicalAuthority; -import alluxio.uri.MultiMasterAuthority; -import alluxio.uri.SingleMasterAuthority; -import alluxio.uri.UnknownAuthority; -import alluxio.uri.ZookeeperAuthority; -import alluxio.uri.ZookeeperLogicalAuthority; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; - -import java.io.IOException; -import java.net.URI; -import java.util.HashMap; -import java.util.Map; -import java.util.StringJoiner; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * An Alluxio client API compatible with Apache Hadoop {@link org.apache.hadoop.fs.FileSystem} - * interface. Any program working with Hadoop HDFS can work with Alluxio transparently. Note that - * the performance of using this API may not be as efficient as the performance of using the Alluxio - * native API defined in {@link alluxio.client.file.FileSystem}, which this API is built on top of. - */ -@NotThreadSafe -public class FileSystem extends AbstractFileSystem { - /** - * Constructs a new {@link FileSystem}. - */ - public FileSystem() { - super(); - } - - /** - * Constructs a new {@link FileSystem} instance with a - * specified {@link alluxio.client.file.FileSystem} handler for tests. - * - * @param fileSystem handler to file system - */ - public FileSystem(alluxio.client.file.FileSystem fileSystem) { - super(fileSystem); - } - - @Override - public String getScheme() { - return Constants.SCHEME; - } - - @Override - protected boolean isZookeeperMode() { - return mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED); - } - - @Override - protected Map getConfigurationFromUri(URI uri, Configuration conf) { - AlluxioURI alluxioUri = new AlluxioURI(uri.toString()); - Map alluxioConfProperties = new HashMap<>(); - - if (alluxioUri.getAuthority() instanceof ZookeeperAuthority) { - ZookeeperAuthority authority = (ZookeeperAuthority) alluxioUri.getAuthority(); - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), true); - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), - authority.getZookeeperAddress()); - } else if (alluxioUri.getAuthority() instanceof SingleMasterAuthority) { - SingleMasterAuthority authority = (SingleMasterAuthority) alluxioUri.getAuthority(); - alluxioConfProperties.put(PropertyKey.MASTER_HOSTNAME.getName(), authority.getHost()); - alluxioConfProperties.put(PropertyKey.MASTER_RPC_PORT.getName(), authority.getPort()); - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), false); - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), null); - // Unset the embedded journal related configuration - // to support alluxio URI has the highest priority - alluxioConfProperties.put(PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES.getName(), null); - alluxioConfProperties.put(PropertyKey.MASTER_RPC_ADDRESSES.getName(), null); - } else if (alluxioUri.getAuthority() instanceof MultiMasterAuthority) { - MultiMasterAuthority authority = (MultiMasterAuthority) alluxioUri.getAuthority(); - alluxioConfProperties.put(PropertyKey.MASTER_RPC_ADDRESSES.getName(), - authority.getMasterAddresses()); - // Unset the zookeeper configuration to support alluxio URI has the highest priority - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), false); - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), null); - } else if (alluxioUri.getAuthority() instanceof EmbeddedLogicalAuthority) { - EmbeddedLogicalAuthority authority = (EmbeddedLogicalAuthority) alluxioUri.getAuthority(); - String masterNamesConfKey = PropertyKey.Template.MASTER_LOGICAL_NAMESERVICES - .format(authority.getLogicalName()).getName(); - String[] masterNames = conf.getTrimmedStrings(masterNamesConfKey); - Preconditions.checkArgument(masterNames.length != 0, - "Invalid uri. You must set %s to use the logical name ", masterNamesConfKey); - - StringJoiner masterRpcAddress = new StringJoiner(","); - for (String masterName : masterNames) { - String name = PropertyKey.Template.MASTER_LOGICAL_RPC_ADDRESS - .format(authority.getLogicalName(), masterName).getName(); - String address = conf.get(name); - Preconditions.checkArgument(address != null, "You need to set %s", name); - masterRpcAddress.add(address); - } - - alluxioConfProperties.put(PropertyKey.MASTER_RPC_ADDRESSES.getName(), - masterRpcAddress.toString()); - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), false); - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), null); - } else if (alluxioUri.getAuthority() instanceof ZookeeperLogicalAuthority) { - ZookeeperLogicalAuthority authority = (ZookeeperLogicalAuthority) alluxioUri.getAuthority(); - String zkNodesConfKey = PropertyKey.Template.MASTER_LOGICAL_ZOOKEEPER_NAMESERVICES - .format(authority.getLogicalName()).getName(); - String[] zkNodeNames = conf.getTrimmedStrings(zkNodesConfKey); - Preconditions.checkArgument(zkNodeNames.length != 0, - "Invalid uri. You must set %s to use the logical name", zkNodesConfKey); - - StringJoiner zkAddress = new StringJoiner(","); - for (String zkName : zkNodeNames) { - String name = PropertyKey.Template.MASTER_LOGICAL_ZOOKEEPER_ADDRESS - .format(authority.getLogicalName(), zkName).getName(); - String address = conf.get(name); - Preconditions.checkArgument(address != null, "You need to set %s", name); - zkAddress.add(address); - } - - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), true); - alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), zkAddress.toString()); - } - return alluxioConfProperties; - } - - @Override - protected void validateFsUri(URI fsUri) throws IOException, IllegalArgumentException { - Preconditions.checkArgument(fsUri.getScheme().equals(getScheme()), - "URI scheme %s does not match the expected scheme %s", fsUri.getScheme(), getScheme()); - - Authority auth = Authority.fromString(fsUri.getAuthority()); - if (auth instanceof UnknownAuthority) { - throw new IOException(String.format("Authority \"%s\" is unknown. The client can not be " - + "configured with the authority from %s", auth, fsUri)); - } - } - - @Override - protected String getFsScheme(URI fsUri) { - return getScheme(); - } - - @Override - protected AlluxioURI getAlluxioPath(Path path) { - return new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); - } - - @Override - protected Path getFsPath(String fsUriHeader, URIStatus fileStatus) { - return new Path(fsUriHeader + fileStatus.getPath()); - } -} diff --git a/core/client/hdfs/src/main/java/alluxio/hadoop/HdfsFileInputStream.java b/core/client/hdfs/src/main/java/alluxio/hadoop/HdfsFileInputStream.java deleted file mode 100644 index 8ac3f7ab3285..000000000000 --- a/core/client/hdfs/src/main/java/alluxio/hadoop/HdfsFileInputStream.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.hadoop; - -import alluxio.AlluxioURI; -import alluxio.client.file.FileInStream; -import alluxio.client.file.FileSystem; -import alluxio.exception.AlluxioException; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.FileDoesNotExistException; - -import org.apache.hadoop.fs.ByteBufferReadable; -import org.apache.hadoop.fs.FileSystem.Statistics; -import org.apache.hadoop.fs.PositionedReadable; -import org.apache.hadoop.fs.Seekable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.EOFException; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * An input stream for reading a file from HDFS. This is just a wrapper around - * {@link FileInStream} with additional statistics gathering in a {@link Statistics} object. - */ -@NotThreadSafe -public class HdfsFileInputStream extends InputStream implements Seekable, PositionedReadable, - ByteBufferReadable { - private static final Logger LOG = LoggerFactory.getLogger(HdfsFileInputStream.class); - - private final Statistics mStatistics; - private final FileInStream mInputStream; - - private boolean mClosed = false; - - /** - * Constructs a new stream for reading a file from HDFS. - * - * @param fs the file system - * @param uri the Alluxio file URI - * @param stats filesystem statistics - */ - public HdfsFileInputStream(FileSystem fs, AlluxioURI uri, Statistics stats) - throws IOException { - LOG.debug("HdfsFileInputStream({}, {})", uri, stats); - - mStatistics = stats; - try { - mInputStream = fs.openFile(uri); - } catch (FileDoesNotExistException e) { - // Transform the Alluxio exception to a Java exception to satisfy the HDFS API contract. - throw new FileNotFoundException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(uri)); - } catch (AlluxioException e) { - throw new IOException(e); - } - } - - /** - * Constructs a new stream for reading a file from HDFS. - * - * @param inputStream the input stream - * @param stats filesystem statistics - */ - public HdfsFileInputStream(FileInStream inputStream, Statistics stats) { - mInputStream = inputStream; - mStatistics = stats; - } - - @Override - public int available() throws IOException { - if (mClosed) { - throw new IOException("Cannot query available bytes from a closed stream."); - } - return (int) mInputStream.remaining(); - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - mInputStream.close(); - mClosed = true; - } - - @Override - public long getPos() throws IOException { - return mInputStream.getPos(); - } - - @Override - public int read() throws IOException { - if (mClosed) { - throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage()); - } - - int data = mInputStream.read(); - if (data != -1 && mStatistics != null) { - mStatistics.incrementBytesRead(1); - } - return data; - } - - @Override - public int read(byte[] buffer) throws IOException { - return read(buffer, 0, buffer.length); - } - - @Override - public int read(byte[] buffer, int offset, int length) throws IOException { - if (mClosed) { - throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage()); - } - - int bytesRead = mInputStream.read(buffer, offset, length); - if (bytesRead != -1 && mStatistics != null) { - mStatistics.incrementBytesRead(bytesRead); - } - return bytesRead; - } - - @Override - public int read(ByteBuffer buf) throws IOException { - if (mClosed) { - throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage()); - } - int bytesRead = mInputStream.read(buf); - if (bytesRead != -1 && mStatistics != null) { - mStatistics.incrementBytesRead(bytesRead); - } - return bytesRead; - } - - @Override - public int read(long position, byte[] buffer, int offset, int length) throws IOException { - if (mClosed) { - throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage()); - } - - int bytesRead = mInputStream.positionedRead(position, buffer, offset, length); - if (bytesRead != -1 && mStatistics != null) { - mStatistics.incrementBytesRead(bytesRead); - } - return bytesRead; - } - - @Override - public void readFully(long position, byte[] buffer) throws IOException { - readFully(position, buffer, 0, buffer.length); - } - - @Override - public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { - int totalBytesRead = 0; - while (totalBytesRead < length) { - int bytesRead = - read(position + totalBytesRead, buffer, offset + totalBytesRead, length - totalBytesRead); - if (bytesRead == -1) { - throw new EOFException(); - } - totalBytesRead += bytesRead; - } - } - - @Override - public void seek(long pos) throws IOException { - try { - mInputStream.seek(pos); - } catch (IllegalArgumentException e) { // convert back to IOException - throw new IOException(e); - } - } - - /** - * This method is not supported in {@link HdfsFileInputStream}. - * - * @param targetPos N/A - * @return N/A - * @throws IOException always - */ - @Override - public boolean seekToNewSource(long targetPos) throws IOException { - throw new IOException("This method is not supported."); - } - - @Override - public long skip(long n) throws IOException { - if (mClosed) { - throw new IOException("Cannot skip bytes in a closed stream."); - } - return mInputStream.skip(n); - } -} diff --git a/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java b/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java deleted file mode 100644 index 68d841d6ce4d..000000000000 --- a/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.hadoop; - -import static com.google.common.hash.Hashing.md5; -import static java.nio.charset.StandardCharsets.UTF_8; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.client.file.CacheContext; -import alluxio.client.file.URIStatus; -import alluxio.client.file.cache.CacheManager; -import alluxio.client.file.cache.LocalCacheFileInStream; -import alluxio.client.file.cache.filter.CacheFilter; -import alluxio.conf.AlluxioConfiguration; -import alluxio.metrics.MetricsConfig; -import alluxio.metrics.MetricsSystem; -import alluxio.wire.FileInfo; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.Progressable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -/** - * An Alluxio client compatible with Apache Hadoop {@link org.apache.hadoop.fs.FileSystem} - * interface, using Alluxio local cache. This client will first consult the local cache before - * requesting the remote Hadoop FileSystem in case of cache misses. - */ -public class LocalCacheFileSystem extends org.apache.hadoop.fs.FileSystem { - private static final Logger LOG = LoggerFactory.getLogger(LocalCacheFileSystem.class); - private static final Set SUPPORTED_FS = new HashSet() { - { - add(Constants.SCHEME); - add("ws"); - } - }; - - /** The external Hadoop filesystem to query on cache miss. */ - private final org.apache.hadoop.fs.FileSystem mExternalFileSystem; - private final HadoopFileOpener mHadoopFileOpener; - private final LocalCacheFileInStream.FileInStreamOpener mAlluxioFileOpener; - private CacheManager mCacheManager; - private CacheFilter mCacheFilter; - private org.apache.hadoop.conf.Configuration mHadoopConf; - private AlluxioConfiguration mAlluxioConf; - - /** - * @param fileSystem File System instance - */ - public LocalCacheFileSystem(org.apache.hadoop.fs.FileSystem fileSystem) { - this(fileSystem, uriStatus -> fileSystem.open(new Path(uriStatus.getPath()))); - } - - /** - * @param fileSystem File System instance - * @param fileOpener File opener instance - */ - public LocalCacheFileSystem(org.apache.hadoop.fs.FileSystem fileSystem, - HadoopFileOpener fileOpener) { - mExternalFileSystem = Preconditions.checkNotNull(fileSystem, "filesystem"); - mHadoopFileOpener = Preconditions.checkNotNull(fileOpener, "fileOpener"); - mAlluxioFileOpener = status -> new AlluxioHdfsInputStream(mHadoopFileOpener.open(status)); - } - - @Override - public synchronized void initialize(URI uri, org.apache.hadoop.conf.Configuration conf) - throws IOException { - if (!SUPPORTED_FS.contains(uri.getScheme())) { - throw new UnsupportedOperationException( - uri.getScheme() + " is not supported as the external filesystem."); - } - super.initialize(uri, conf); - mHadoopConf = conf; - // Set statistics - setConf(conf); - mAlluxioConf = HadoopUtils.toAlluxioConf(mHadoopConf); - // Handle metrics - Properties metricsProperties = new Properties(); - for (Map.Entry entry : conf) { - metricsProperties.setProperty(entry.getKey(), entry.getValue()); - } - MetricsSystem.startSinksFromConfig(new MetricsConfig(metricsProperties)); - mCacheManager = CacheManager.Factory.get(mAlluxioConf); - LocalCacheFileInStream.registerMetrics(); - mCacheFilter = CacheFilter.create(mAlluxioConf); - } - - @Override - public void close() throws IOException { - // super.close should be called first before releasing the resources in this instance, as the - // super class may invoke other methods in this class. For example, - // org.apache.hadoop.fs.FileSystem.close may check the existence of certain temp files before - // closing - super.close(); - } - - /** - * @return scheme - */ - // @Override This doesn't exist in Hadoop 1.x, so cannot put {@literal @Override}. - public String getScheme() { - return mExternalFileSystem.getScheme(); - } - - @Override - public FSDataInputStream open(Path path, int bufferSize) throws IOException { - if (mCacheManager == null) { - return mExternalFileSystem.open(path, bufferSize); - } - FileStatus externalFileStatus = mExternalFileSystem.getFileStatus(path); - // Note that, we don't set have fileId here because fileId is Alluxio specific - FileInfo info = new FileInfo() - .setLength(externalFileStatus.getLen()) - .setPath(externalFileStatus.getPath().toString()) - .setFolder(externalFileStatus.isDirectory()) - .setBlockSizeBytes(externalFileStatus.getBlockSize()) - .setLastModificationTimeMs(externalFileStatus.getModificationTime()) - .setLastAccessTimeMs(externalFileStatus.getAccessTime()) - .setOwner(externalFileStatus.getOwner()) - .setGroup(externalFileStatus.getGroup()); - // FilePath is a unique identifier for a file, however it can be a long string - // hence using md5 hash of the file path as the identifier in the cache. - CacheContext context = CacheContext.defaults().setCacheIdentifier( - md5().hashString(externalFileStatus.getPath().toString(), UTF_8).toString()); - URIStatus status = new URIStatus(info, context); - return open(status, bufferSize); - } - - /** - * Attempts to open the specified file for reading. - * - * @param status the status of the file to open - * @param bufferSize stream buffer size in bytes, currently unused - * @return an {@link FSDataInputStream} at the indicated path of a file - */ - public FSDataInputStream open(URIStatus status, int bufferSize) throws IOException { - if (mCacheManager == null || !mCacheFilter.needsCache(status)) { - return mExternalFileSystem.open(HadoopUtils.toPath(new AlluxioURI(status.getPath())), - bufferSize); - } - return new FSDataInputStream(new HdfsFileInputStream( - new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf), - statistics)); - } - - @Override - public URI getUri() { - return mExternalFileSystem.getUri(); - } - - @Override - public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, - int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { - return mExternalFileSystem.create(f, permission, overwrite, bufferSize, replication, blockSize, - progress); - } - - @Override - public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) - throws IOException { - return mExternalFileSystem.append(f, bufferSize, progress); - } - - @Override - public boolean rename(Path src, Path dst) throws IOException { - return mExternalFileSystem.rename(src, dst); - } - - @Override - public boolean delete(Path f, boolean recursive) throws IOException { - return mExternalFileSystem.delete(f, recursive); - } - - @Override - public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException { - return mExternalFileSystem.listStatus(f); - } - - @Override - public void setWorkingDirectory(Path new_dir) { - mExternalFileSystem.setWorkingDirectory(new_dir); - } - - @Override - public Path getWorkingDirectory() { - return mExternalFileSystem.getWorkingDirectory(); - } - - @Override - public boolean mkdirs(Path f, FsPermission permission) throws IOException { - return mExternalFileSystem.mkdirs(f, permission); - } - - @Override - public FileStatus getFileStatus(Path f) throws IOException { - return mExternalFileSystem.getFileStatus(f); - } -} diff --git a/core/client/pom.xml b/core/client/pom.xml deleted file mode 100644 index 338db707b66b..000000000000 --- a/core/client/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - 4.0.0 - - alluxio-core - org.alluxio - 2.10.0-SNAPSHOT - - alluxio-core-client - pom - Alluxio Core - Client - Alluxio Core Clients - - - fs - hdfs - - - - - ${project.parent.parent.basedir}/build - - - diff --git a/core/common/pom.xml b/core/common/pom.xml deleted file mode 100644 index b8adf2b56652..000000000000 --- a/core/common/pom.xml +++ /dev/null @@ -1,229 +0,0 @@ - - - 4.0.0 - - org.alluxio - alluxio-core - 2.10.0-SNAPSHOT - - alluxio-core-common - jar - Alluxio Core - Common Utilities - Common utilities shared in Alluxio core modules - - - - - ${project.parent.parent.basedir}/build - false - - - - - - com.amazonaws - aws-java-sdk-core - - - com.fasterxml.jackson.core - jackson-databind - - - commons-io - commons-io - - - commons-cli - commons-cli - - - org.apache.httpcomponents - httpclient - - - com.google.code.gson - gson - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - io.dropwizard.metrics - metrics-core - - - io.dropwizard.metrics - metrics-graphite - - - io.dropwizard.metrics - metrics-jmx - - - io.dropwizard.metrics - metrics-jvm - - - io.grpc - grpc-core - - - io.grpc - grpc-netty - - - io.grpc - grpc-stub - - - io.swagger - swagger-annotations - - - org.apache.commons - commons-lang3 - - - org.apache.curator - curator-client - - - org.apache.curator - curator-framework - - - org.apache.zookeeper - zookeeper - - - org.reflections - reflections - - - org.rocksdb - rocksdbjni - - - - io.netty - netty-tcnative-boringssl-static - 2.0.26.Final - - - - - org.alluxio - alluxio-core-transport - ${project.version} - - - - - com.google.guava - guava-testlib - test - - - org.apache.curator - curator-test - test - - - io.netty - netty-buffer - ${netty.version} - test-jar - test - - - - - - - src/main/resources - true - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.codehaus.mojo - buildnumber-maven-plugin - - - initialize - - create-metadata - - - - - ${project.build.directory}/generated/build-metadata - build.properties - git.revision - - - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - read-project-properties - - - - ${project.build.directory}/generated/build-metadata/build.properties - - - - - - - org.codehaus.mojo - templating-maven-plugin - - - filter-src - - filter-sources - - - ${basedir}/src/main/java-templates - ${project.build.directory}/generated-sources/java-templates - - - - - - - - diff --git a/core/common/src/main/java/alluxio/AlluxioRemoteLogFilter.java b/core/common/src/main/java/alluxio/AlluxioRemoteLogFilter.java deleted file mode 100644 index da5e16a54432..000000000000 --- a/core/common/src/main/java/alluxio/AlluxioRemoteLogFilter.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio; - -import org.apache.log4j.MDC; -import org.apache.log4j.spi.Filter; -import org.apache.log4j.spi.LoggingEvent; - -/** - * Thin filter to add MDC information to {@link LoggingEvent}. Remote log server can read - * {@link org.apache.log4j.spi.LoggingEvent} off {@link java.net.Socket} and retrieve the - * MDC information. In this case the MDC information is the name of the log4j appender. - * Remote log server then uses this appender to log messages. The implementation of this - * class is similar to that of {@link org.apache.log4j.varia.StringMatchFilter}. - */ -public class AlluxioRemoteLogFilter extends Filter { - /** Name (Key) of the MDC info. */ - public static final String REMOTE_LOG_MDC_PROCESS_TYPE_KEY = "ProcessType"; - - /** - * @deprecated Option name to configure this {@link AlluxioRemoteLogFilter} - * in log4j.properties. - */ - @Deprecated - public static final String PROCESS_TYPE_OPTION = "ProcessType"; - - /** Type of the process generating this log message. */ - private String mProcessType; - - /** - * @deprecated Gets the option strings. - * @return option strings as an array - */ - @Deprecated - public String[] getOptionStrings() { - return new String[] {PROCESS_TYPE_OPTION}; - } - - /** - * @deprecated Sets option value use key=value format. The log4j.properties file uses this - * to set options. See the log4j.properties for more details. - * - * @param key key (name) of the option - * @param value value of the option - */ - @Deprecated - public void setOption(String key, String value) { - if (key.equalsIgnoreCase(PROCESS_TYPE_OPTION)) { - mProcessType = value; - } - } - - /** - * Sets {@code mProcessType} to be the type of the process generating this log message. - * - * Log4j parses log4j.properties, extracting the Java class that corresponds to the filter. - * In this case, the Java class is {@link AlluxioRemoteLogFilter}. - * Log4j also extracts option information as a key-value pair, e.g. - * "ProcessType" : "MASTER". Then log4j invokes the {@link #setProcessType(String)} - * method to set the value of {@link #mProcessType}. - * - * @param processType name of the log appender - */ - public void setProcessType(String processType) { - mProcessType = processType; - } - - /** - * Retrieves the string representation of process type. - * - * This method can potentially be called by log4j reflectively. Even if currently log4j - * does not call it, it is possible to be called in the future. Therefore, to be - * consistent with {@link org.apache.log4j.varia.StringMatchFilter}, we should prevent - * this method from been removed. - * - * @return the name of the log appender - */ - public String getProcessType() { - return mProcessType; - } - - @Override - public int decide(LoggingEvent event) { - MDC.put(REMOTE_LOG_MDC_PROCESS_TYPE_KEY, mProcessType); - return ACCEPT; - } -} diff --git a/core/common/src/main/java/alluxio/RuntimeConstants.java b/core/common/src/main/java/alluxio/RuntimeConstants.java deleted file mode 100644 index 19e1fccf2ae3..000000000000 --- a/core/common/src/main/java/alluxio/RuntimeConstants.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * System constants that are determined during runtime. - */ -// Note: PropertyKey depends on this class, so this class shouldn't depend on any classes that have -// a dependency on PropertyKey. -@ThreadSafe -public final class RuntimeConstants { - /** The version of this Alluxio instance. */ - public static final String VERSION = ProjectConstants.VERSION; - - static { - if (VERSION.endsWith("SNAPSHOT")) { - ALLUXIO_DOCS_URL = "https://docs.alluxio.io/os/user/edge"; - ALLUXIO_JAVADOC_URL = "https://docs.alluxio.io/os/javadoc/edge"; - } else { - String[] majorMinor = VERSION.split("\\."); - ALLUXIO_DOCS_URL = - String.format("https://docs.alluxio.io/os/user/%s.%s", majorMinor[0], majorMinor[1]); - ALLUXIO_JAVADOC_URL = - String.format("https://docs.alluxio.io/os/javadoc/%s.%s", majorMinor[0], majorMinor[1]); - } - } - - /** The relative path to the Alluxio target jar. */ - public static final String ALLUXIO_JAR = "target/alluxio-" + VERSION - + "-jar-with-dependencies.jar"; - - /** The URL of Alluxio documentation for this version on project web site. */ - public static final String ALLUXIO_DOCS_URL; - - /** The URL of Alluxio javadoc documentation. */ - public static final String ALLUXIO_JAVADOC_URL; - - /** The URL of Alluxio debugging documentation. */ - public static final String ALLUXIO_DEBUG_DOCS_URL = ALLUXIO_DOCS_URL - + "/en/operation/Troubleshooting.html"; - - /** The URL of Alluxio 1.x to 2.x upgrade documentation. */ - public static final String ALLUXIO_2X_UPGRADE_DOC_URL = ALLUXIO_DOCS_URL - + "/en/operation/2.x-Upgrade.html"; - - /** The URL of Alluxio security documentation. */ - public static final String ALLUXIO_SECURITY_DOCS_URL = ALLUXIO_DOCS_URL - + "/en/operation/Security.html"; - - private RuntimeConstants() {} // prevent instantiation -} diff --git a/core/common/src/main/java/alluxio/SyncInfo.java b/core/common/src/main/java/alluxio/SyncInfo.java deleted file mode 100644 index 51faa910b225..000000000000 --- a/core/common/src/main/java/alluxio/SyncInfo.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio; - -import java.util.Collections; -import java.util.Map; -import java.util.Set; - -/** - * This class is used to represent what the active syncing process should sync. - */ -public class SyncInfo { - public static final long INVALID_TXID = -1; - private static final SyncInfo EMPTY_INFO = new SyncInfo(Collections.emptyMap(), - false, INVALID_TXID); - - // A map mapping syncpoints to files changed in those sync points - private final Map> mChangedFilesMap; - - // Ignore the changed files and simply sync the entire directory - private final boolean mForceSync; - - // transaction id that we can restart from once this sync is complete - private final long mTxId; - - /** - * Constructs a SyncInfo. - * - * @param changedFiles a map mapping syncpoint to changed files - * @param forceSync force sync the entire directory - * @param txId the transaction id that is synced in this sync - */ - public SyncInfo(Map> changedFiles, boolean forceSync, long txId) { - mChangedFilesMap = changedFiles; - mForceSync = forceSync; - mTxId = txId; - } - - /** - * Returns an empty SyncInfo object. - * - * @return emptyInfo object - */ - public static SyncInfo emptyInfo() { - return EMPTY_INFO; - } - - /** - * Returns a list of sync points. - * - * @return a list of sync points - */ - public Set getSyncPoints() { - return mChangedFilesMap.keySet(); - } - - /** - * REturns a set of changed files. - * - * @param syncPoint the syncPoint that we are monitoring - * @return a set of sync points - */ - public Set getChangedFiles(AlluxioURI syncPoint) { - return mChangedFilesMap.get(syncPoint); - } - - /** - * returns true if this sync should happen on the entire directory. - * - * @return true if this sync should happen on the entire dir - */ - public boolean isForceSync() { - return mForceSync; - } - - /** - * returns the transaction id that is synced in this sync period. - * - * @return a transaction id that is synced in this sync period - */ - public long getTxId() { - return mTxId; - } -} diff --git a/core/common/src/main/java/alluxio/check/UpdateCheck.java b/core/common/src/main/java/alluxio/check/UpdateCheck.java deleted file mode 100644 index ba64aee2853d..000000000000 --- a/core/common/src/main/java/alluxio/check/UpdateCheck.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.check; - -import alluxio.ProjectConstants; -import alluxio.exception.runtime.FailedPreconditionRuntimeException; -import alluxio.util.EnvironmentUtils; -import alluxio.util.FeatureUtils; -import alluxio.util.OSUtils; - -import com.amazonaws.util.EC2MetadataUtils; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Check for updates. - */ -@ThreadSafe -public final class UpdateCheck { - public static final String USER_AGENT_SEPARATOR = ";"; - - static final String PRODUCT_CODE_FORMAT = "ProductCode:%s"; - static final String OS_FORMAT = "OS:%s"; - - // Launch environment - static final String CFT_KEY = "cft"; - static final String DOCKER_KEY = "docker"; - static final String EC2_KEY = "ec2"; - static final String EMBEDDED_KEY = "embedded"; - static final String EMR_KEY = "emr"; - static final String GCE_KEY = "gce"; - static final String KUBERNETES_KEY = "kubernetes"; - - // Feature - static final String BACKUP_DELEGATION_KEY = "backupDelegation"; - static final String DAILY_BACKUP_KEY = "dailyBackup"; - static final String MASTER_AUDIT_LOG_KEY = "masterAuditLog"; - static final String PERSIST_BLACK_LIST_KEY = "persistBlackList"; - static final String PAGE_STORE_KEY = "pageStore"; - static final String INODE_METASTORE_ROCKS_KEY = "inodeRocks"; - static final String BLOCK_METASTORE_ROCKS_KEY = "blockRocks"; - static final String UNSAFE_PERSIST_KEY = "unsafePersist"; - static final String ZOOKEEPER_KEY = "zookeeper"; - - /** - * @param id the id of the current Alluxio identity (e.g. cluster id, instance id) - * @param additionalInfo additional information to send - * @param connectionRequestTimeout the connection request timeout for the HTTP request in ms - * @param connectTimeout the connection timeout for the HTTP request in ms - * @param socketTimeout the socket timeout for the HTTP request in ms - * @return the latest Alluxio version string - */ - public static String getLatestVersion(String id, List additionalInfo, - long connectionRequestTimeout, long connectTimeout, long socketTimeout) - throws IOException { - Preconditions.checkState(id != null && !id.isEmpty(), "id should not be null or empty"); - Preconditions.checkNotNull(additionalInfo); - // Create the GET request. - Joiner joiner = Joiner.on("/"); - String path = joiner.join("v0", "version"); - String url = new URL(new URL(ProjectConstants.UPDATE_CHECK_HOST), path).toString(); - - HttpGet post = new HttpGet(url); - post.setHeader("User-Agent", getUserAgentString(id, additionalInfo)); - post.setHeader("Authorization", "Basic " + ProjectConstants.UPDATE_CHECK_MAGIC_NUMBER); - - // Fire off the version check request. - HttpClient client = HttpClientBuilder.create() - .setDefaultRequestConfig( - RequestConfig.custom() - .setConnectionRequestTimeout((int) connectionRequestTimeout) - .setConnectTimeout((int) connectTimeout) - .setSocketTimeout((int) socketTimeout) - .build()) - .build(); - HttpResponse response = client.execute(post); - - // Check the response code. - int responseCode = response.getStatusLine().getStatusCode(); - if (responseCode != HttpURLConnection.HTTP_OK) { - throw new FailedPreconditionRuntimeException( - "Update check request failed with code: " + responseCode); - } - - return EntityUtils.toString(response.getEntity(), "UTF-8"); - } - - /** - * @param id the id of the current Alluxio identity (e.g. cluster id, instance id) - * @param additionalInfo additional information to add to result string - * @return a string representation of the user's environment in the format - * "Alluxio/{ALLUXIO_VERSION} (valueA; valueB)" - */ - @VisibleForTesting - public static String getUserAgentString(String id, List additionalInfo) { - List info = new ArrayList<>(); - info.add(id); - addUserAgentEnvironments(info); - addUserAgentFeatures(info); - info.addAll(additionalInfo); - return String.format("Alluxio/%s (%s)", ProjectConstants.VERSION, - Joiner.on(USER_AGENT_SEPARATOR + " ").skipNulls().join(info)); - } - - /** - * Adds the information of user environment to given list. - * - * @param info the list to add info to - */ - @VisibleForTesting - public static void addUserAgentEnvironments(List info) { - info.add(String.format(OS_FORMAT, OSUtils.OS_NAME)); - if (EnvironmentUtils.isDocker()) { - info.add(DOCKER_KEY); - } - if (EnvironmentUtils.isKubernetes()) { - info.add(KUBERNETES_KEY); - } - if (EnvironmentUtils.isGoogleComputeEngine()) { - info.add(GCE_KEY); - } else { - addEC2Info(info); - } - } - - /** - * Get the feature's information. - * - * @param info the list to add info to - */ - @VisibleForTesting - public static void addUserAgentFeatures(List info) { - addIfTrue(FeatureUtils.isEmbeddedJournal(), info, EMBEDDED_KEY); - addIfTrue(FeatureUtils.isInodeStoreRocks(), info, INODE_METASTORE_ROCKS_KEY); - addIfTrue(FeatureUtils.isBlockStoreRocks(), info, BLOCK_METASTORE_ROCKS_KEY); - addIfTrue(FeatureUtils.isZookeeperEnabled(), info, ZOOKEEPER_KEY); - addIfTrue(FeatureUtils.isBackupDelegationEnabled(), info, BACKUP_DELEGATION_KEY); - addIfTrue(FeatureUtils.isDailyBackupEnabled(), info, DAILY_BACKUP_KEY); - addIfTrue(!FeatureUtils.isPersistenceBlacklistEmpty(), info, PERSIST_BLACK_LIST_KEY); - addIfTrue(FeatureUtils.isUnsafeDirectPersistEnabled(), info, UNSAFE_PERSIST_KEY); - addIfTrue(FeatureUtils.isMasterAuditLoggingEnabled(), info, MASTER_AUDIT_LOG_KEY); - addIfTrue(FeatureUtils.isPageStoreEnabled(), info, PAGE_STORE_KEY); - } - - /** - * Add feature name if condition is true. - * - * @param valid true, if condition is valid - * @param features feature list - * @param featureName feature name - */ - public static void addIfTrue(boolean valid, List features, String featureName) { - if (valid) { - features.add(featureName); - } - } - - /** - * Adds the information of EC2 environment to given list. - * - * @param info the list to add info to - */ - private static void addEC2Info(List info) { - boolean isEC2 = false; - String productCode = EnvironmentUtils.getEC2ProductCode(); - if (!productCode.isEmpty()) { - info.add(String.format(PRODUCT_CODE_FORMAT, productCode)); - isEC2 = true; - } - - String userData = ""; - try { - userData = EC2MetadataUtils.getUserData(); - } catch (Throwable t) { - // Exceptions are expected if instance is not EC2 instance - // or get metadata operation is not allowed - } - if (userData != null && !userData.isEmpty()) { - isEC2 = true; - if (EnvironmentUtils.isCFT(userData)) { - info.add(CFT_KEY); - } else if (EnvironmentUtils.isEMR(userData)) { - info.add(EMR_KEY); - } - } else if (!isEC2 && EnvironmentUtils.isEC2()) { - isEC2 = true; - } - - if (isEC2) { - info.add(EC2_KEY); - } - } - - private UpdateCheck() {} // prevent instantiation -} diff --git a/core/common/src/main/java/alluxio/conf/Configuration.java b/core/common/src/main/java/alluxio/conf/Configuration.java deleted file mode 100644 index 09d250f2a222..000000000000 --- a/core/common/src/main/java/alluxio/conf/Configuration.java +++ /dev/null @@ -1,644 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf; - -import alluxio.Constants; -import alluxio.RuntimeConstants; -import alluxio.conf.path.PathConfiguration; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.UnauthenticatedException; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.ConfigProperty; -import alluxio.grpc.GetConfigurationPOptions; -import alluxio.grpc.GetConfigurationPResponse; -import alluxio.grpc.GrpcChannel; -import alluxio.grpc.GrpcChannelBuilder; -import alluxio.grpc.GrpcServerAddress; -import alluxio.grpc.GrpcUtils; -import alluxio.grpc.MetaMasterConfigurationServiceGrpc; -import alluxio.grpc.Scope; -import alluxio.util.ConfigurationUtils; -import alluxio.util.io.PathUtils; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetSocketAddress; -import java.net.URL; -import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiFunction; - -/** - *

- * Global configuration properties of Alluxio. This class works like a dictionary and serves each - * Alluxio configuration property as a key-value pair. - * - *

- * Alluxio configuration properties are loaded into this class in the following order with - * decreasing priority: - *

    - *
  1. Java system properties;
    2. - *
  2. Environment variables via {@code alluxio-env.sh} or from OS settings;
    3. - *
  3. Site specific properties via {@code alluxio-site.properties} file;
    4. - *
- * - *

- * The default properties are defined in the {@link PropertyKey} class in the codebase. Alluxio - * users can override values of these default properties by creating {@code alluxio-site.properties} - * and putting it under java {@code CLASSPATH} when running Alluxio (e.g., ${ALLUXIO_HOME}/conf/) - * - *

- * This class defines many convenient static methods which delegate to an internal - * {@link InstancedConfiguration}. To use this global configuration in a method that takes - * {@link AlluxioConfiguration} as an argument, pass {@link Configuration#global()}. - */ -public final class Configuration -{ - private static final Logger LOG = LoggerFactory.getLogger(Configuration.class); - - private static final AtomicReference SERVER_CONFIG_REFERENCE = - new AtomicReference<>(); - - static { - reloadProperties(); - } - - /** - * Create and return a copy of all properties. - * - * @return a copy of properties - */ - public static AlluxioProperties copyProperties() { - return SERVER_CONFIG_REFERENCE.get().copyProperties(); - } - - /** - * Merges the current configuration properties with new properties. If a property exists - * both in the new and current configuration, the one from the new configuration wins if - * its priority is higher or equal than the existing one. - * - * @param properties the source {@link Properties} to be merged - * @param source the source of the the properties (e.g., system property, default and etc) - */ - public static void merge(Map properties, Source source) { - SERVER_CONFIG_REFERENCE.get().merge(properties, source); - } - - // Public accessor methods - /** - * Sets the value for the appropriate key in the {@link Properties}. - * - * @param key the key to set - * @param value the value for the key - */ - public static void set(PropertyKey key, Object value) { - set(key, value, Source.RUNTIME); - } - - /** - * Sets the value for the appropriate key in the {@link Properties} by source. - * - * @param key the key to set - * @param value the value for the key - * @param source the source of the the properties (e.g., system property, default and etc) - */ - public static void set(PropertyKey key, Object value, Source source) { - if (key.getType() == PropertyKey.PropertyType.STRING) { - value = String.valueOf(value); - } - SERVER_CONFIG_REFERENCE.get().set(key, value, source); - } - - /** - * Unsets the value for the appropriate key in the {@link Properties}. - * - * @param key the key to unset - */ - public static void unset(PropertyKey key) { - SERVER_CONFIG_REFERENCE.get().unset(key); - } - - /** - * Gets the value for the given key in the {@link Properties}; if this key is not found, a - * RuntimeException is thrown. - * - * @param key the key to get the value for - * @return the value for the given key - */ - public static Object get(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().get(key); - } - - /** - * Gets the value for the given key in the {@link Properties}; if this key is not found, a - * RuntimeException is thrown. - * - * @param key the key to get the value for - * @param options options for getting configuration value - * @return the value for the given key - */ - public static Object get(PropertyKey key, ConfigurationValueOptions options) { - return SERVER_CONFIG_REFERENCE.get().get(key, options); - } - - /** - * @param key the key to get the value for - * @param defaultValue the value to return if no value is set for the specified key - * @param the type of default value - * @return the value - */ - public static T getOrDefault(PropertyKey key, T defaultValue) { - return SERVER_CONFIG_REFERENCE.get().getOrDefault(key, defaultValue); - } - - /** - * @param key the key to get the value for - * @param defaultValue the value to return if no value is set for the specified key - * @param options options for getting configuration value - * @return the value - */ - public static Object getOrDefault(PropertyKey key, String defaultValue, - ConfigurationValueOptions options) { - return SERVER_CONFIG_REFERENCE.get().getOrDefault(key, defaultValue, options); - } - - /** - * Checks if the configuration contains a value for the given key. - * - * @param key the key to check - * @return true if there is value for the key, false otherwise - */ - public static boolean isSet(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().isSet(key); - } - - /** - * Checks if the configuration contains a value for the given key that is set by a user. - * - * @param key the key to check - * @return true if there is value for the key by a user, false otherwise - */ - public static boolean isSetByUser(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().isSetByUser(key); - } - - /** - * @return the keys configured by the configuration - */ - public static Set keySet() { - return SERVER_CONFIG_REFERENCE.get().keySet(); - } - - /** - * Gets the String value for the given key. - * - * @param key the key to get the value for - * @return the value for the given key as an {@code String} - */ - public static String getString(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getString(key); - } - - /** - * Gets the integer representation of the value for the given key. - * - * @param key the key to get the value for - * @return the value for the given key as an {@code int} - */ - public static int getInt(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getInt(key); - } - - /** - * Gets the double representation of the value for the given key. - * - * @param key the key to get the value for - * @return the value for the given key as a {@code double} - */ - public static double getDouble(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getDouble(key); - } - - /** - * Gets the long integer representation of the value for the given key. - * - * @param key the key to get the value for - * @return the value for the given key as a {@code long} - */ - public static long getLong(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getLong(key); - } - - /** - * Gets the boolean representation of the value for the given key. - * - * @param key the key to get the value for - * @return the value for the given key as a {@code boolean} - */ - public static boolean getBoolean(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getBoolean(key); - } - - /** - * Gets the value for the given key as a list. - * - * @param key the key to get the value for - * @return the list of values for the given key - */ - public static List getList(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getList(key); - } - - /** - * Gets the value for the given key as an enum value. - * - * @param key the key to get the value for - * @param enumType the type of the enum - * @param the type of the enum - * @return the value for the given key as an enum value - */ - public static > T getEnum(PropertyKey key, Class enumType) { - return SERVER_CONFIG_REFERENCE.get().getEnum(key, enumType); - } - - /** - * Gets the bytes of the value for the given key. - * - * @param key the key to get the value for - * @return the bytes of the value for the given key - */ - public static long getBytes(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getBytes(key); - } - - /** - * Gets the time of key in millisecond unit. - * - * @param key the key to get the value for - * @return the time of key in millisecond unit - */ - public static long getMs(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getMs(key); - } - - /** - * Gets the time of the key as a duration. - * - * @param key the key to get the value for - * @return the value of the key represented as a duration - */ - public static Duration getDuration(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getDuration(key); - } - - /** - * Gets the value for the given key as a class. - * - * @param key the key to get the value for - * @param the type of the class - * @return the value for the given key as a class - */ - public static Class getClass(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getClass(key); - } - - /** - * Gets a set of properties that share a given common prefix key as a map. E.g., if A.B=V1 and - * A.C=V2, calling this method with prefixKey=A returns a map of {B=V1, C=V2}, where B and C are - * also valid properties. If no property shares the prefix, an empty map is returned. - * - * @param prefixKey the prefix key - * @return a map from nested properties aggregated by the prefix - */ - public static Map getNestedProperties(PropertyKey prefixKey) { - return SERVER_CONFIG_REFERENCE.get().getNestedProperties(prefixKey); - } - - /** - * @param key the property key - * @return the source for the given key - */ - public static Source getSource(PropertyKey key) { - return SERVER_CONFIG_REFERENCE.get().getSource(key); - } - - /** - * @return a map from all configuration property names to their values; values may potentially be - * null - */ - public static Map toMap() { - return SERVER_CONFIG_REFERENCE.get().toMap(); - } - - /** - * @param opts options for formatting the configuration values - * @return a map from all configuration property names to their values; values may potentially be - * null - */ - public static Map toMap(ConfigurationValueOptions opts) { - return SERVER_CONFIG_REFERENCE.get().toMap(opts); - } - - /** - * @return the global configuration through {@link AlluxioConfiguration} API, - * which is a read-only API - */ - public static AlluxioConfiguration global() { - return SERVER_CONFIG_REFERENCE.get(); - } - - /** - * @return the global configuration instance that is modifiable - */ - public static InstancedConfiguration modifiableGlobal() { - return SERVER_CONFIG_REFERENCE.get(); - } - - /** - * @return a copy of {@link InstancedConfiguration} object based on the global configuration - */ - public static InstancedConfiguration copyGlobal() { - InstancedConfiguration configuration = SERVER_CONFIG_REFERENCE.get(); - return new InstancedConfiguration( - configuration.copyProperties(), configuration.clusterDefaultsLoaded()); - } - - /** - * Gets all configuration properties filtered by the specified scope. - * - * @param scope the scope to filter by - * @return the properties - */ - public static List getConfiguration(Scope scope) { - AlluxioConfiguration conf = global(); - ConfigurationValueOptions useRawDisplayValue = - ConfigurationValueOptions.defaults().useDisplayValue(true); - - return conf.keySet().stream() - .filter(key -> GrpcUtils.contains(key.getScope(), scope)) - .map(key -> { - ConfigProperty.Builder configProp = ConfigProperty.newBuilder().setName(key.getName()) - .setSource(conf.getSource(key).toString()); - if (conf.isSet(key)) { - configProp.setValue(String.valueOf(conf.get(key, useRawDisplayValue))); - } - return configProp.build(); - }) - .collect(ImmutableList.toImmutableList()); - } - - /** - * Loads cluster default values for workers from the meta master if it's not loaded yet. - * - * @param address the master address - * @param scope the property scope - */ - public static void loadClusterDefaults(InetSocketAddress address, Scope scope) - throws AlluxioStatusException { - InstancedConfiguration conf = SERVER_CONFIG_REFERENCE.get(); - InstancedConfiguration newConf; - if (conf.getBoolean(PropertyKey.USER_CONF_CLUSTER_DEFAULT_ENABLED) - && !conf.clusterDefaultsLoaded()) { - do { - conf = SERVER_CONFIG_REFERENCE.get(); - GetConfigurationPResponse response = loadConfiguration(address, conf, false, true); - newConf = getClusterConf(response, conf, scope); - } while (!SERVER_CONFIG_REFERENCE.compareAndSet(conf, newConf)); - } - } - - /** - * Loads configuration from meta master in one RPC. - * - * @param address the meta master address - * @param conf the existing configuration - * @param ignoreClusterConf do not load cluster configuration related information - * @param ignorePathConf do not load path configuration related information - * @return the RPC response - */ - public static GetConfigurationPResponse loadConfiguration(InetSocketAddress address, - AlluxioConfiguration conf, boolean ignoreClusterConf, boolean ignorePathConf) - throws AlluxioStatusException { - GrpcChannel channel = null; - try { - LOG.debug("Alluxio client (version {}) is trying to load configuration from meta master {}", - RuntimeConstants.VERSION, address); - channel = GrpcChannelBuilder.newBuilder(GrpcServerAddress.create(address), conf) - .disableAuthentication().build(); - MetaMasterConfigurationServiceGrpc.MetaMasterConfigurationServiceBlockingStub client = - MetaMasterConfigurationServiceGrpc.newBlockingStub(channel); - GetConfigurationPResponse response = client.getConfiguration( - GetConfigurationPOptions.newBuilder().setRawValue(true) - .setIgnoreClusterConf(ignoreClusterConf).setIgnorePathConf(ignorePathConf).build()); - LOG.debug("Alluxio client has loaded configuration from meta master {}", address); - return response; - } catch (io.grpc.StatusRuntimeException e) { - throw new UnavailableException(String.format( - "Failed to handshake with master %s to load cluster default configuration values: %s", - address, e.getMessage()), e); - } catch (UnauthenticatedException e) { - throw new RuntimeException(String.format( - "Received authentication exception during boot-strap connect with host:%s", address), - e); - } finally { - if (channel != null) { - channel.shutdown(); - } - } - } - - /** - * Loads the cluster level configuration from the get configuration response, - * filters out the configuration for certain scope, and merges it with the existing configuration. - * - * @param response the get configuration RPC response - * @param conf the existing configuration - * @param scope the target scope - * @return the merged configuration - */ - public static InstancedConfiguration getClusterConf(GetConfigurationPResponse response, - AlluxioConfiguration conf, Scope scope) { - String clientVersion = conf.getString(PropertyKey.VERSION); - LOG.debug("Alluxio {} (version {}) is trying to load cluster level configurations", - scope, clientVersion); - List clusterConfig = response.getClusterConfigsList(); - Properties clusterProps = filterAndLoadProperties(clusterConfig, scope, (key, value) -> - String.format("Loading property: %s (%s) -> %s", key, key.getScope(), value)); - // Check version. - String clusterVersion = clusterProps.get(PropertyKey.VERSION).toString(); - if (!clientVersion.equals(clusterVersion)) { - LOG.warn("Alluxio {} version ({}) does not match Alluxio cluster version ({})", - scope, clientVersion, clusterVersion); - clusterProps.remove(PropertyKey.VERSION); - } - // Merge conf returned by master as the cluster default into conf object - AlluxioProperties props = conf.copyProperties(); - props.merge(clusterProps, Source.CLUSTER_DEFAULT); - // Use the constructor to set cluster defaults as being loaded. - InstancedConfiguration updatedConf = new InstancedConfiguration(props, true); - updatedConf.validate(); - LOG.debug("Alluxio {} has loaded cluster level configurations", scope); - return updatedConf; - } - - /** - * Loads the path level configuration from the get configuration response. - * - * Only client scope properties will be loaded. - * - * @param response the get configuration RPC response - * @param clusterConf cluster level configuration - * @return the loaded path level configuration - */ - public static PathConfiguration getPathConf(GetConfigurationPResponse response, - AlluxioConfiguration clusterConf) { - String clientVersion = clusterConf.getString(PropertyKey.VERSION); - LOG.debug("Alluxio client (version {}) is trying to load path level configurations", - clientVersion); - Map pathConfs = new HashMap<>(); - response.getPathConfigsMap().forEach((path, conf) -> { - Properties props = filterAndLoadProperties(conf.getPropertiesList(), Scope.CLIENT, - (key, value) -> String.format("Loading property: %s (%s) -> %s for path %s", - key, key.getScope(), value, path)); - AlluxioProperties properties = new AlluxioProperties(); - properties.merge(props, Source.PATH_DEFAULT); - pathConfs.put(path, new InstancedConfiguration(properties, true)); - }); - LOG.debug("Alluxio client has loaded path level configurations"); - return PathConfiguration.create(pathConfs); - } - - /** - * Filters and loads properties with a certain scope from the property list returned by grpc. - * The given scope should only be {@link Scope#WORKER} or {@link Scope#CLIENT}. - * - * @param properties the property list returned by grpc - * @param scope the scope to filter the received property list - * @param logMessage a function with key and value as parameter and returns debug log message - * @return the loaded properties - */ - private static Properties filterAndLoadProperties(List properties, - Scope scope, BiFunction logMessage) { - Properties props = new Properties(); - for (ConfigProperty property : properties) { - String name = property.getName(); - // TODO(binfan): support propagating unsetting properties from master - if (PropertyKey.isValid(name) && property.hasValue()) { - PropertyKey key = PropertyKey.fromString(name); - if (!GrpcUtils.contains(key.getScope(), scope)) { - // Only propagate properties contains the target scope - continue; - } - String value = property.getValue(); - props.put(key, value); - LOG.debug(logMessage.apply(key, value)); - } - } - return props; - } - - /** - * @return hash of properties - */ - public static String hash() { - return SERVER_CONFIG_REFERENCE.get().hash(); - } - - private Configuration() {} // prevent instantiation - - /** - * Reloads site properties from disk. - */ - public static void reloadProperties() { - // Bootstrap the configuration. This is necessary because we need to resolve alluxio.home - // (likely to be in system properties) to locate the conf dir to search for the site - // property file. - AlluxioProperties alluxioProperties = new AlluxioProperties(); - // Can't directly pass System.getProperties() because it is not thread-safe - // This can cause a ConcurrentModificationException when merging. - alluxioProperties.merge(System.getProperties().entrySet().stream() - .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)), - Source.SYSTEM_PROPERTY); - InstancedConfiguration conf = new InstancedConfiguration(alluxioProperties); - // Load site specific properties file if not in test mode. Note that we decide - // whether in test mode by default properties and system properties (via getBoolean). - if (!conf.getBoolean(PropertyKey.TEST_MODE)) { - // We are not in test mode, load site properties - // First try loading from config file - for (String path : conf.getList(PropertyKey.SITE_CONF_DIR)) { - String file = PathUtils.concatPath(path, Constants.SITE_PROPERTIES); - try (FileInputStream fileInputStream = new FileInputStream(file)) { - Optional properties = loadProperties(fileInputStream); - if (properties.isPresent()) { - alluxioProperties.merge(properties.get(), Source.siteProperty(file)); - conf = new InstancedConfiguration(alluxioProperties); - conf.validate(); - SERVER_CONFIG_REFERENCE.set(conf); - // If a site conf is successfully loaded, stop trying different paths. - return; - } - } catch (FileNotFoundException e) { - // skip - } catch (IOException e) { - LOG.warn("Failed to close property input stream from {}: {}", file, e.toString()); - } - } - - // Try to load from resource - URL resource = - ConfigurationUtils.class.getClassLoader().getResource(Constants.SITE_PROPERTIES); - if (resource != null) { - try (InputStream stream = resource.openStream()) { - Optional properties = loadProperties(stream); - if (properties.isPresent()) { - alluxioProperties.merge(properties.get(), Source.siteProperty(resource.getPath())); - conf = new InstancedConfiguration(alluxioProperties); - conf.validate(); - SERVER_CONFIG_REFERENCE.set(conf); - } - } catch (IOException e) { - LOG.warn("Failed to read properties from {}: {}", resource, e.toString()); - } - } - } - conf.validate(); - SERVER_CONFIG_REFERENCE.set(conf); - } - - /** - * @param stream the stream to read properties from - * @return a properties object populated from the stream - */ - private static Optional loadProperties(InputStream stream) { - Properties properties = new Properties(); - try { - properties.load(stream); - } catch (IOException e) { - LOG.warn("Unable to load properties: {}", e.toString()); - return Optional.empty(); - } - return Optional.of(properties); - } -} diff --git a/core/common/src/main/java/alluxio/conf/InstancedConfiguration.java b/core/common/src/main/java/alluxio/conf/InstancedConfiguration.java deleted file mode 100644 index 3bd384c01644..000000000000 --- a/core/common/src/main/java/alluxio/conf/InstancedConfiguration.java +++ /dev/null @@ -1,689 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf; - -import static alluxio.conf.PropertyKey.CONF_REGEX; -import static alluxio.conf.PropertyKey.REGEX_STRING; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static java.lang.String.format; - -import alluxio.conf.PropertyKey.Template; -import alluxio.exception.ExceptionMessage; -import alluxio.util.ConfigurationUtils; -import alluxio.util.FormatUtils; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.regex.Matcher; -import javax.annotation.Nonnull; - -/** - * Alluxio configuration. - - * WARNING: This API is not intended to be used outside of internal Alluxio code and may be - * changed or removed in a future minor release. - * - * Application code should use APIs {@link Configuration}. - * - */ -public class InstancedConfiguration implements AlluxioConfiguration { - private static final Logger LOG = LoggerFactory.getLogger(InstancedConfiguration.class); - /** Source of the truth of all property values (default or customized). */ - protected final AlluxioProperties mProperties; - - private final boolean mClusterDefaultsLoaded; - - /** - * Creates a new instance of {@link InstancedConfiguration}. - * - * WARNING: This API is not intended to be used outside of internal Alluxio code and may be - * changed or removed in a future minor release. - * - * Application code should use {@link Configuration#global()}. - * - * @param properties alluxio properties underlying this configuration - */ - public InstancedConfiguration(AlluxioProperties properties) { - this(properties, false); - } - - /** - * Creates a new instance of {@link InstancedConfiguration}. - * - * WARNING: This API is not intended to be used outside of internal Alluxio code and may be - * changed or removed in a future minor release. - * - * Application code should use {@link Configuration#global()}. - * - * @param properties alluxio properties underlying this configuration - * @param clusterDefaultsLoaded Whether or not the properties represent the cluster defaults - */ - public InstancedConfiguration(AlluxioProperties properties, boolean clusterDefaultsLoaded) { - mProperties = properties; - mClusterDefaultsLoaded = clusterDefaultsLoaded; - } - - /** - * Return reference to mProperties. - * @return mProperties - */ - public AlluxioProperties getProperties() { - return mProperties; - } - - @Override - public AlluxioProperties copyProperties() { - return mProperties.copy(); - } - - @Override - public Object get(PropertyKey key) { - return get(key, ConfigurationValueOptions.defaults()); - } - - @Override - public Object get(PropertyKey key, ConfigurationValueOptions options) { - Object value = mProperties.get(key); - if (value == null) { - // if value or default value is not set in configuration for the given key - throw new RuntimeException(ExceptionMessage.UNDEFINED_CONFIGURATION_KEY.getMessage(key)); - } - - if (!(value instanceof String)) { - return value; - } - - if (!options.shouldUseRawValue()) { - try { - value = lookup(key, (String) value); - } catch (UnresolvablePropertyException e) { - throw new RuntimeException("Could not resolve key \"" - + key.getName() + "\": " + e.getMessage(), e); - } - } - if (options.shouldUseDisplayValue()) { - PropertyKey.DisplayType displayType = key.getDisplayType(); - switch (displayType) { - case DEFAULT: - break; - case CREDENTIALS: - value = "******"; - break; - default: - throw new IllegalStateException(String.format("Invalid displayType %s for property %s", - displayType.name(), key.getName())); - } - } - return value; - } - - private boolean isResolvable(PropertyKey key) { - Object value = mProperties.get(key); - // null values are unresolvable - if (value == null) { - return false; - } - try { - // Lookup to resolve any key before simply returning isSet. An exception will be thrown if - // the key can't be resolved or if a lower level value isn't set. - if (value instanceof String) { - lookup(key, (String) value); - } - } catch (UnresolvablePropertyException e) { - return false; - } - return true; - } - - @Override - public boolean isSet(PropertyKey key) { - return mProperties.isSet(key) && isResolvable(key); - } - - @Override - public boolean isSetByUser(PropertyKey key) { - return mProperties.isSetByUser(key) && isResolvable(key); - } - - /** - * Sets the value for the appropriate key in the {@link Properties}. - * - * @param key the key to set - * @param value the value for the key - */ - public void set(PropertyKey key, Object value) { - set(key, value, Source.RUNTIME); - } - - /** - * Sets the value for the appropriate key in the {@link Properties}. - * - * @param key the key to set - * @param value the value for the key - * - * @deprecated API to aid property key type transition - */ - @java.lang.Deprecated - public void set(@Nonnull PropertyKey key, @Nonnull String value) { - checkArgument(!value.equals(""), - "The key \"%s\" cannot be have an empty string as a value. Use " - + "Configuration.unset to remove a key from the configuration.", key); - if (key.validateValue(value)) { - mProperties.put(key, key.formatValue(value), Source.RUNTIME); - } else { - if (key.getType() == PropertyKey.PropertyType.STRING) { - throw new IllegalArgumentException( - format("Invalid value for property key %s: %s", key, value)); - } - LOG.warn("The value {} for property key {} is invalid. PropertyKey are now typed " - + "and require values to be properly typed. Invalid PropertyKey values will not be " - + "accepted in 3.0", value, key); - mProperties.put(key, key.parseValue(value), Source.RUNTIME); - } - } - - /** - * Sets the value for the appropriate key in the {@link Properties} by source. - * - * @param key the key to set - * @param value the value for the key - * @param source the source of the the properties (e.g., system property, default and etc) - */ - public void set(@Nonnull PropertyKey key, @Nonnull Object value, @Nonnull Source source) { - checkArgument(!value.equals(""), - "The key \"%s\" cannot be have an empty string as a value. Use " - + "Configuration.unset to remove a key from the configuration.", key); - checkArgument(key.validateValue(value), - "Invalid value for property key %s: %s", key, value); - value = key.formatValue(value); - mProperties.put(key, value, source); - } - - /** - * Unsets the value for the appropriate key in the {@link Properties}. If the {@link PropertyKey} - * has a default value, it will still be considered set after executing this method. - * - * @param key the key to unset - */ - public void unset(PropertyKey key) { - Preconditions.checkNotNull(key, "key"); - mProperties.remove(key); - } - - /** - * Merges map of properties into the current alluxio properties. - * - * @param properties map of keys to values - * @param source the source type for these properties - */ - public void merge(Map properties, Source source) { - mProperties.merge(properties, source); - } - - @Override - public Set keySet() { - return mProperties.keySet(); - } - - @Override - public Set userKeySet() { - return mProperties.userKeySet(); - } - - @Override - public String getString(PropertyKey key) { - if (key.getType() != PropertyKey.PropertyType.STRING) { - LOG.warn("PropertyKey {}'s type is {}, please use proper getter method for the type, " - + "getString will no longer work for non-STRING property types in 3.0", - key, key.getType()); - } - Object value = get(key); - if (value instanceof String) { - return (String) value; - } - return value.toString(); - } - - @Override - public int getInt(PropertyKey key) - { - checkArgument(key.getType() == PropertyKey.PropertyType.INTEGER); - return (int) get(key); - } - - @Override - public long getLong(PropertyKey key) { - // Low-precision types int can be implicitly converted to high-precision types long - // without loss of precision - checkArgument(key.getType() == PropertyKey.PropertyType.LONG - || key.getType() == PropertyKey.PropertyType.INTEGER); - return ((Number) get(key)).longValue(); - } - - @Override - public double getDouble(PropertyKey key) { - checkArgument(key.getType() == PropertyKey.PropertyType.DOUBLE); - return (double) get(key); - } - - @Override - public boolean getBoolean(PropertyKey key) { - checkArgument(key.getType() == PropertyKey.PropertyType.BOOLEAN); - return (boolean) get(key); - } - - @Override - public List getList(PropertyKey key) { - checkArgument(key.getType() == PropertyKey.PropertyType.LIST); - String value = (String) get(key); - return ConfigurationUtils.parseAsList(value, key.getDelimiter()); - } - - @Override - public > T getEnum(PropertyKey key, Class enumType) { - checkArgument(key.getEnumType().equals(enumType), "PropertyKey %s is not of enum type", key); - return enumType.cast(get(key)); - } - - @Override - public long getBytes(PropertyKey key) { - checkArgument(key.getType() == PropertyKey.PropertyType.DATASIZE); - return FormatUtils.parseSpaceSize((String) get(key)); - } - - @Override - public long getMs(PropertyKey key) { - checkArgument(key.getType() == PropertyKey.PropertyType.DURATION); - return FormatUtils.parseTimeSize((String) get(key)); - } - - @Override - public Duration getDuration(PropertyKey key) { - return Duration.ofMillis(getMs(key)); - } - - @Override - public Class getClass(PropertyKey key) { - Object value = get(key); - if (value instanceof Class) { - return (Class) value; - } - try { - return (Class) Class.forName((String) value); - } catch (ClassNotFoundException e) { - throw new IllegalStateException( - format("Requested class %s can not be loaded", value)); - } - } - - @Override - public Map getNestedProperties(PropertyKey prefixKey) { - Map ret = Maps.newHashMap(); - for (Map.Entry entry: mProperties.entrySet()) { - String key = entry.getKey().getName(); - if (prefixKey.isNested(key)) { - String suffixKey = key.substring(prefixKey.length() + 1); - ret.put(suffixKey, entry.getValue()); - } - } - return ret; - } - - @Override - public Source getSource(PropertyKey key) { - return mProperties.getSource(key); - } - - @Override - public Map toMap(ConfigurationValueOptions opts) { - Map map = new HashMap<>(); - // Cannot use Collectors.toMap because we support null keys. - keySet().forEach(key -> map.put(key.getName(), getOrDefault(key, null, opts))); - return map; - } - - @Override - public void validate() { - if (!getBoolean(PropertyKey.CONF_VALIDATION_ENABLED)) { - return; - } - for (PropertyKey key : keySet()) { - checkState( - getSource(key).getType() != Source.Type.SITE_PROPERTY || !key.isIgnoredSiteProperty(), - "%s is not accepted in alluxio-site.properties, " - + "and must be specified as a JVM property. " - + "If no JVM property is present, Alluxio will use default value '%s'.", - key.getName(), key.getDefaultValue()); - - if (PropertyKey.isDeprecated(key) && getSource(key).compareTo(Source.DEFAULT) != 0) { - LOG.warn("{} is deprecated. Please avoid using this key in the future. {}", key.getName(), - PropertyKey.getDeprecationMessage(key)); - } - } - - checkTimeouts(); - checkWorkerPorts(); - checkUserFileBufferBytes(); - checkZkConfiguration(); - checkTieredLocality(); - checkTieredStorage(); - checkMasterThrottleThresholds(); - } - - @Override - public boolean clusterDefaultsLoaded() { - return mClusterDefaultsLoaded; - } - - @Override - public String hash() { - return mProperties.hash(); - } - - /** - * Lookup key names to handle ${key} stuff. - * - * @param base the String to look for - * @return resolved String value - */ - private Object lookup(PropertyKey key, final String base) throws UnresolvablePropertyException { - return lookupRecursively(key, base, new HashSet<>()); - } - - /** - * Actual recursive lookup replacement. - * - * @param base the string to resolve - * @param seen strings already seen during this lookup, used to prevent unbound recursion - * @return the resolved string - */ - private Object lookupRecursively(PropertyKey originalKey, String base, Set seen) - throws UnresolvablePropertyException { - // check argument - if (base == null) { - throw new UnresolvablePropertyException("Can't resolve property with null value"); - } - - String resolved = base; - Object resolvedValue = null; - PropertyKey key = null; - // Lets find pattern match to ${key}. - // TODO(hsaputra): Consider using Apache Commons StrSubstitutor. - Matcher matcher = CONF_REGEX.matcher(base); - while (matcher.find()) { - String match = matcher.group(2).trim(); - if (!seen.add(match)) { - throw new RuntimeException(ExceptionMessage.KEY_CIRCULAR_DEPENDENCY.getMessage(match)); - } - if (!PropertyKey.isValid(match)) { - throw new RuntimeException(ExceptionMessage.INVALID_CONFIGURATION_KEY.getMessage(match)); - } - key = PropertyKey.fromString(match); - Object value = mProperties.get(key); - String stringValue = null; - if (value instanceof String) { - stringValue = String.valueOf(lookupRecursively(key, (String) value , seen)); - } - else if (value != null) { - stringValue = String.valueOf(value); - } - seen.remove(match); - if (stringValue == null) { - throw new UnresolvablePropertyException(ExceptionMessage - .UNDEFINED_CONFIGURATION_KEY.getMessage(match)); - } - resolved = resolved.replaceFirst(REGEX_STRING, Matcher.quoteReplacement(stringValue)); - } - if (key != null) { - resolvedValue = originalKey.parseValue(resolved); - } else { - resolvedValue = resolved; - } - return resolvedValue; - } - - /** - * Validates worker port configuration. - * - * @throws IllegalStateException if invalid worker port configuration is encountered - */ - private void checkWorkerPorts() { - int maxWorkersPerHost = getInt(PropertyKey.INTEGRATION_YARN_WORKERS_PER_HOST_MAX); - if (maxWorkersPerHost > 1) { - String message = "%s cannot be specified when allowing multiple workers per host with " - + PropertyKey.Name.INTEGRATION_YARN_WORKERS_PER_HOST_MAX + "=" + maxWorkersPerHost; - checkState(System.getProperty(PropertyKey.Name.WORKER_RPC_PORT) == null, - message, PropertyKey.WORKER_RPC_PORT); - checkState(System.getProperty(PropertyKey.Name.WORKER_WEB_PORT) == null, - message, PropertyKey.WORKER_WEB_PORT); - } - } - - /** - * Validates timeout related configuration. - */ - private void checkTimeouts() { - long waitTime = getMs(PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME); - long retryInterval = getMs(PropertyKey.USER_RPC_RETRY_MAX_SLEEP_MS); - if (waitTime < retryInterval) { - LOG.warn("{}={}ms is smaller than {}={}ms. Workers might not have enough time to register. " - + "Consider either increasing {} or decreasing {}", - PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME, waitTime, - PropertyKey.USER_RPC_RETRY_MAX_SLEEP_MS, retryInterval, - PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME, - PropertyKey.USER_RPC_RETRY_MAX_SLEEP_MS); - } - checkHeartbeatTimeout(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL, - PropertyKey.MASTER_HEARTBEAT_TIMEOUT); - // Skip checking block worker heartbeat config because the timeout is master-side while the - // heartbeat interval is worker-side. - } - - /** - * Checks that the interval is shorter than the timeout. - * - * @param intervalKey property key for an interval - * @param timeoutKey property key for a timeout - */ - private void checkHeartbeatTimeout(PropertyKey intervalKey, PropertyKey timeoutKey) { - long interval = getMs(intervalKey); - long timeout = getMs(timeoutKey); - checkState(interval < timeout, - "heartbeat interval (%s=%s) must be less than heartbeat timeout (%s=%s)", intervalKey, - interval, timeoutKey, timeout); - } - - /** - * Validates the user file buffer size is a non-negative number. - * - * @throws IllegalStateException if invalid user file buffer size configuration is encountered - */ - private void checkUserFileBufferBytes() { - if (!isSet(PropertyKey.USER_FILE_BUFFER_BYTES)) { // load from hadoop conf - return; - } - long usrFileBufferBytes = getBytes(PropertyKey.USER_FILE_BUFFER_BYTES); - checkState((usrFileBufferBytes & Integer.MAX_VALUE) == usrFileBufferBytes, - "Invalid value of %s: %s", - PropertyKey.Name.USER_FILE_BUFFER_BYTES, usrFileBufferBytes); - } - - /** - * Validates Zookeeper-related configuration and prints warnings for possible sources of error. - * - * @throws IllegalStateException if invalid Zookeeper configuration is encountered - */ - private void checkZkConfiguration() { - checkState( - isSet(PropertyKey.ZOOKEEPER_ADDRESS) == getBoolean(PropertyKey.ZOOKEEPER_ENABLED), - "Inconsistent Zookeeper configuration; %s should be set if and only if %s is true", - PropertyKey.Name.ZOOKEEPER_ADDRESS, PropertyKey.Name.ZOOKEEPER_ENABLED); - } - - /** - * Checks that tiered locality configuration is consistent. - * - * @throws IllegalStateException if invalid tiered locality configuration is encountered - */ - private void checkTieredLocality() { - // Check that any custom tiers set by alluxio.locality.{custom_tier}=value are also defined in - // the tier ordering defined by alluxio.locality.order. - Set tiers = Sets.newHashSet(getList(PropertyKey.LOCALITY_ORDER)); - Set predefinedKeys = new HashSet<>(PropertyKey.defaultKeys()); - for (PropertyKey key : mProperties.keySet()) { - if (predefinedKeys.contains(key)) { - // Skip non-templated keys. - continue; - } - Matcher matcher = Template.LOCALITY_TIER.match(key.toString()); - if (matcher.matches() && matcher.group(1) != null) { - String tierName = matcher.group(1); - if (!tiers.contains(tierName)) { - throw new IllegalStateException( - String.format("Tier %s is configured by %s, but does not exist in the tier list %s " - + "configured by %s", tierName, key, tiers, PropertyKey.LOCALITY_ORDER)); - } - } - } - } - - /** - * Checks that tiered storage configuration on worker is consistent with the global configuration. - * - * @throws IllegalStateException if invalid tiered storage configuration is encountered - */ - @VisibleForTesting - void checkTieredStorage() { - int globalTiers = getInt(PropertyKey.MASTER_TIERED_STORE_GLOBAL_LEVELS); - Set globalTierAliasSet = new HashSet<>(); - for (int i = 0; i < globalTiers; i++) { - globalTierAliasSet.add( - getString(PropertyKey.Template.MASTER_TIERED_STORE_GLOBAL_LEVEL_ALIAS.format(i))); - } - int workerTiers = getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS); - checkState(workerTiers <= globalTiers, - "%s tiers on worker (configured by %s), larger than global %s tiers (configured by %s) ", - workerTiers, PropertyKey.WORKER_TIERED_STORE_LEVELS, - globalTiers, PropertyKey.MASTER_TIERED_STORE_GLOBAL_LEVELS); - for (int i = 0; i < workerTiers; i++) { - PropertyKey key = Template.WORKER_TIERED_STORE_LEVEL_ALIAS.format(i); - String alias = getString(key); - checkState(globalTierAliasSet.contains(alias), - "Alias \"%s\" on tier %s on worker (configured by %s) is not found in global tiered " - + "storage setting: %s", - alias, i, key, String.join(", ", globalTierAliasSet)); - } - } - - /** - * @throws IllegalStateException if invalid throttle threshold parameters are found - */ - private void checkMasterThrottleThresholds() { - boolean heapUsedRatioThresholdValid - = (Double - .compare(0, - getDouble(PropertyKey.MASTER_THROTTLE_ACTIVE_HEAP_USED_RATIO)) < 0 - && Double - .compare(getDouble(PropertyKey.MASTER_THROTTLE_ACTIVE_HEAP_USED_RATIO), - getDouble(PropertyKey.MASTER_THROTTLE_STRESSED_HEAP_USED_RATIO)) < 0 - && Double - .compare(getDouble(PropertyKey.MASTER_THROTTLE_STRESSED_HEAP_USED_RATIO), - getDouble(PropertyKey.MASTER_THROTTLE_OVERLOADED_HEAP_USED_RATIO)) < 0 - && Double - .compare(getDouble(PropertyKey.MASTER_THROTTLE_OVERLOADED_HEAP_USED_RATIO), - 1) <= 0); - if (!heapUsedRatioThresholdValid) { - throw new IllegalStateException( - String.format("The heap used ratio thresholds are not set correctly, the values should" - + " be between 0 and 1, it is expected: ACTIVE < STRESSED < OVERLOADED," - + " while they are ACTIVE({}), STRESSED({}), OVERLOADED({})", - getDouble(PropertyKey.MASTER_THROTTLE_ACTIVE_HEAP_USED_RATIO), - getDouble(PropertyKey.MASTER_THROTTLE_STRESSED_HEAP_USED_RATIO), - getDouble(PropertyKey.MASTER_THROTTLE_OVERLOADED_HEAP_USED_RATIO))); - } - - boolean cpuUsedRatioThresholdValid - = (Double - .compare(0, - getDouble(PropertyKey.MASTER_THROTTLE_ACTIVE_CPU_LOAD_RATIO)) < 0 - && Double - .compare(getDouble(PropertyKey.MASTER_THROTTLE_ACTIVE_CPU_LOAD_RATIO), - getDouble(PropertyKey.MASTER_THROTTLE_STRESSED_CPU_LOAD_RATIO)) < 0 - && Double - .compare(getDouble(PropertyKey.MASTER_THROTTLE_STRESSED_CPU_LOAD_RATIO), - getDouble(PropertyKey.MASTER_THROTTLE_OVERLOADED_CPU_LOAD_RATIO)) < 0 - && Double - .compare(getDouble(PropertyKey.MASTER_THROTTLE_OVERLOADED_CPU_LOAD_RATIO), - 1) <= 0); - if (!cpuUsedRatioThresholdValid) { - throw new IllegalStateException( - String.format("The cpu used ratio thresholds are not set correctly, the values should" - + " be between 0 and 1, it is expected: ACTIVE < STRESSED < OVERLOADED," - + " while they are ACTIVE({}), STRESSED({}), OVERLOADED({})", - getDouble(PropertyKey.MASTER_THROTTLE_ACTIVE_CPU_LOAD_RATIO), - getDouble(PropertyKey.MASTER_THROTTLE_STRESSED_CPU_LOAD_RATIO), - getDouble(PropertyKey.MASTER_THROTTLE_OVERLOADED_CPU_LOAD_RATIO))); - } - - boolean heapGCTimeThresholdValid - = (0 < getMs(PropertyKey.MASTER_THROTTLE_ACTIVE_HEAP_GC_TIME) - && getMs(PropertyKey.MASTER_THROTTLE_ACTIVE_HEAP_GC_TIME) - < getMs(PropertyKey.MASTER_THROTTLE_STRESSED_HEAP_GC_TIME) - && getMs(PropertyKey.MASTER_THROTTLE_STRESSED_HEAP_GC_TIME) - < getMs(PropertyKey.MASTER_THROTTLE_OVERLOADED_HEAP_GC_TIME)); - if (!heapGCTimeThresholdValid) { - throw new IllegalStateException( - String.format("The heap GC extra time threshold is not set correctly," - + " it is expected: ACTIVE < STRESSED < OVERLOADED, while they are " - + "ACTIVE({}), STRESSED({}), OVERLOADED({})", - getMs(PropertyKey.MASTER_THROTTLE_ACTIVE_HEAP_GC_TIME), - getMs(PropertyKey.MASTER_THROTTLE_STRESSED_HEAP_GC_TIME), - getMs(PropertyKey.MASTER_THROTTLE_OVERLOADED_HEAP_GC_TIME))); - } - - boolean rpcQueueSizeThresholdValid - = (0 < getInt(PropertyKey.MASTER_THROTTLE_ACTIVE_RPC_QUEUE_SIZE) - && getInt(PropertyKey.MASTER_THROTTLE_ACTIVE_RPC_QUEUE_SIZE) - < getInt(PropertyKey.MASTER_THROTTLE_STRESSED_RPC_QUEUE_SIZE) - && getInt(PropertyKey.MASTER_THROTTLE_STRESSED_RPC_QUEUE_SIZE) - < getInt(PropertyKey.MASTER_THROTTLE_OVERLOADED_RPC_QUEUE_SIZE)); - if (!rpcQueueSizeThresholdValid) { - throw new IllegalStateException( - String.format("The rpc queue size threshold is not set correctly," - + " it is expected: ACTIVE < STRESSED < OVERLOADED, while they are " - + "ACTIVE({}), STRESSED({}), OVERLOADED({})", - getInt(PropertyKey.MASTER_THROTTLE_ACTIVE_RPC_QUEUE_SIZE), - getInt(PropertyKey.MASTER_THROTTLE_STRESSED_RPC_QUEUE_SIZE), - getInt(PropertyKey.MASTER_THROTTLE_OVERLOADED_RPC_QUEUE_SIZE))); - } - } - - private class UnresolvablePropertyException extends Exception { - - public UnresolvablePropertyException(String msg) { - super(msg); - } - } -} diff --git a/core/common/src/main/java/alluxio/conf/path/PathConfiguration.java b/core/common/src/main/java/alluxio/conf/path/PathConfiguration.java deleted file mode 100644 index e4742920e16a..000000000000 --- a/core/common/src/main/java/alluxio/conf/path/PathConfiguration.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf.path; - -import alluxio.AlluxioURI; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; - -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -/** - * Path level configuration. - * - * An {@link AlluxioConfiguration} is set for each path pattern. This class matches an Alluxio - * path against the patterns, then from the configurations of the matched patterns, it chooses the - * configuration that contains the specified property key and belongs to the best matched pattern. - * - * Different implementations have different definitions of the best match. - */ -public interface PathConfiguration { - /** - * @param path the Alluxio path - * @param key the property key - * @return the chosen configuration matching the path and containing the key - */ - Optional getConfiguration(AlluxioURI path, PropertyKey key); - - /** - * @param path the Alluxio path - * @return all property keys in the path level configuration that is applicable to path - */ - Set getPropertyKeys(AlluxioURI path); - - /** - * Factory method to create an implementation of {@link PathConfiguration}. - * - * @param pathConf the map from paths to path level configurations - * @return the implementation of {@link PathConfiguration} - */ - static PathConfiguration create(Map pathConf) { - return new PrefixPathConfiguration(pathConf); - } -} diff --git a/core/common/src/main/java/alluxio/conf/path/PathMatcher.java b/core/common/src/main/java/alluxio/conf/path/PathMatcher.java deleted file mode 100644 index 4a852930f70c..000000000000 --- a/core/common/src/main/java/alluxio/conf/path/PathMatcher.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf.path; - -import alluxio.AlluxioURI; - -import java.util.List; -import java.util.Optional; - -/** - * Matches a path against a set of path patterns. - */ -public interface PathMatcher { - /** - * @param path the path to be matched, must start with "/" - * @return the list of matched path patterns sorted by descending match preferences - */ - Optional> match(AlluxioURI path); -} diff --git a/core/common/src/main/java/alluxio/conf/path/PrefixPathConfiguration.java b/core/common/src/main/java/alluxio/conf/path/PrefixPathConfiguration.java deleted file mode 100644 index 8cedd4ee0b25..000000000000 --- a/core/common/src/main/java/alluxio/conf/path/PrefixPathConfiguration.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf.path; - -import alluxio.AlluxioURI; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; - -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Configurations for path prefixes. - * - * Path patterns are path prefixes, the longest matching prefix is the best match. - * A path is equal to the matched prefix or is under the subtree of the prefix. - */ -@ThreadSafe -public final class PrefixPathConfiguration implements PathConfiguration { - /** - * Map from path prefixes to corresponding configurations. - */ - private final ConcurrentHashMap mConf = new ConcurrentHashMap<>(); - /** - * Matches path patterns. - */ - private final PathMatcher mMatcher; - - /** - * Constructs a new path level configuration. - * - * A shallow copy of the map will be created internally. - * - * @param configurations a map from path patterns to corresponding path level configuration - */ - public PrefixPathConfiguration(Map configurations) { - configurations.forEach((path, conf) -> mConf.put(path, conf)); - mMatcher = new PrefixPathMatcher(configurations.keySet()); - } - - @Override - public Optional getConfiguration(AlluxioURI path, PropertyKey key) { - Optional> patterns = mMatcher.match(path); - if (patterns.isPresent()) { - for (String pattern : patterns.get()) { - if (mConf.get(pattern).isSetByUser(key)) { - return Optional.of(mConf.get(pattern)); - } - } - } - return Optional.empty(); - } - - /** - * @param path the Alluxio path - * @return all property keys applicable to path including ancestor paths' configurations - */ - @Override - public Set getPropertyKeys(AlluxioURI path) { - Set keys = new HashSet<>(); - mMatcher.match(path).ifPresent(patterns -> patterns.forEach(pattern -> - keys.addAll(mConf.get(pattern).userKeySet()))); - return keys; - } -} diff --git a/core/common/src/main/java/alluxio/conf/path/PrefixPathMatcher.java b/core/common/src/main/java/alluxio/conf/path/PrefixPathMatcher.java deleted file mode 100644 index 75c90f8b52cf..000000000000 --- a/core/common/src/main/java/alluxio/conf/path/PrefixPathMatcher.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf.path; - -import alluxio.AlluxioURI; - -import com.google.common.base.Preconditions; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A {@link PathMatcher} to match a path against a set of paths. - * - * Path a matches path b if a is the same as b or a is in the subtree of b. - * For example: - * /a/b/c matches /a/b, but - * /a/bc does not match /a/b because although /a/b is a prefix of /a/bc, /a/bc is not - * in the subtree of /a/b. - */ -@ThreadSafe -public final class PrefixPathMatcher implements PathMatcher { - /** - * Root of the trie for paths. - */ - private final TrieNode mTrie = new TrieNode(); - /** - * A map from terminal trie node to the corresponding path. - */ - private final Map mPaths = new HashMap<>(); - - /** - * Builds internal trie based on paths. - * - * Each path must start with "/". - * - * @param paths the list of path - */ - public PrefixPathMatcher(Set paths) { - for (String path : paths) { - Preconditions.checkArgument(path.startsWith("/"), "Path must start with /"); - TrieNode node = mTrie.insert(path); - mPaths.put(node, path); - } - } - - @Override - public Optional> match(AlluxioURI path) { - List nodes = mTrie.search(path.getPath()); - if (nodes.isEmpty()) { - return Optional.empty(); - } - List matchedPaths = new ArrayList<>(); - for (int i = nodes.size() - 1; i >= 0; i--) { - TrieNode node = nodes.get(i); - if (mPaths.containsKey(node)) { - matchedPaths.add(mPaths.get(node)); - } - } - return Optional.of(matchedPaths); - } -} diff --git a/core/common/src/main/java/alluxio/conf/path/SpecificPathConfiguration.java b/core/common/src/main/java/alluxio/conf/path/SpecificPathConfiguration.java deleted file mode 100644 index 6bb0eb038e23..000000000000 --- a/core/common/src/main/java/alluxio/conf/path/SpecificPathConfiguration.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf.path; - -import alluxio.AlluxioURI; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.AlluxioProperties; -import alluxio.conf.ConfigurationValueOptions; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.conf.Source; - -import com.google.common.collect.ImmutableMap; - -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Configuration for a specific Alluxio path. - * - * Priority for the value of a property follows: - * 1. matched path level configuration; - * 2. cluster level configuration. - */ -@ThreadSafe -public final class SpecificPathConfiguration implements AlluxioConfiguration { - /** - * Cluster level configuration. - */ - private final AlluxioConfiguration mClusterConf; - /** - * Path level configuration. - */ - private final PathConfiguration mPathConf; - /** - * The specific path. - */ - private final AlluxioURI mPath; - - /** - * Constructs a new instance with the specified references without copying the underlying - * properties. - * - * @param clusterConf the cluster level configuration - * @param pathConf the path level configuration - * @param path the specific Alluxio path - */ - public SpecificPathConfiguration(AlluxioConfiguration clusterConf, PathConfiguration pathConf, - AlluxioURI path) { - mClusterConf = clusterConf; - mPathConf = pathConf; - mPath = path; - } - - private AlluxioConfiguration conf(PropertyKey key) { - return mPathConf.getConfiguration(mPath, key).orElse(mClusterConf); - } - - @Override - public Object get(PropertyKey key) { - return conf(key).get(key); - } - - @Override - public Object get(PropertyKey key, ConfigurationValueOptions options) { - return conf(key).get(key, options); - } - - @Override - public boolean isSet(PropertyKey key) { - return conf(key).isSet(key); - } - - @Override - public boolean isSetByUser(PropertyKey key) { - return conf(key).isSetByUser(key); - } - - @Override - public Set keySet() { - return mClusterConf.keySet(); - } - - @Override - public Set userKeySet() { - return mPathConf.getPropertyKeys(mPath); - } - - @Override - public String getString(PropertyKey key) { - return conf(key).getString(key); - } - - @Override - public int getInt(PropertyKey key) { - return conf(key).getInt(key); - } - - @Override - public long getLong(PropertyKey key) { - return conf(key).getLong(key); - } - - @Override - public double getDouble(PropertyKey key) { - return conf(key).getDouble(key); - } - - @Override - public boolean getBoolean(PropertyKey key) { - return conf(key).getBoolean(key); - } - - @Override - public List getList(PropertyKey key) { - return conf(key).getList(key); - } - - @Override - public > T getEnum(PropertyKey key, Class enumType) { - return conf(key).getEnum(key, enumType); - } - - @Override - public long getBytes(PropertyKey key) { - return conf(key).getBytes(key); - } - - @Override - public long getMs(PropertyKey key) { - return conf(key).getMs(key); - } - - @Override - public Duration getDuration(PropertyKey key) { - return conf(key).getDuration(key); - } - - @Override - public Class getClass(PropertyKey key) { - return conf(key).getClass(key); - } - - @Override - public Map getNestedProperties(PropertyKey prefixKey) { - return conf(prefixKey).getNestedProperties(prefixKey); - } - - @Override - public AlluxioProperties copyProperties() { - AlluxioProperties properties = mClusterConf.copyProperties(); - for (PropertyKey key : keySet()) { - mPathConf.getConfiguration(mPath, key).ifPresent( - config -> properties.put(key, config.get(key), Source.PATH_DEFAULT)); - } - return properties; - } - - @Override - public Source getSource(PropertyKey key) { - return conf(key).getSource(key); - } - - @Override - public Map toMap(ConfigurationValueOptions opts) { - ImmutableMap.Builder map = ImmutableMap.builder(); - // Cannot use Collectors.toMap because we support null keys. - keySet().forEach(key -> - map.put(key.getName(), conf(key).getOrDefault(key, null, opts))); - return map.build(); - } - - @Override - public void validate() { - new InstancedConfiguration(copyProperties()).validate(); - } - - @Override - public boolean clusterDefaultsLoaded() { - return mClusterConf.clusterDefaultsLoaded(); - } -} diff --git a/core/common/src/main/java/alluxio/conf/path/TrieNode.java b/core/common/src/main/java/alluxio/conf/path/TrieNode.java deleted file mode 100644 index e741478a9859..000000000000 --- a/core/common/src/main/java/alluxio/conf/path/TrieNode.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf.path; - -import alluxio.collections.Pair; - -import com.google.common.collect.Iterators; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Stream; - -/** - * A node in a trie. - * @param the type of the value held by each node - */ -public final class TrieNode { - private final Map> mChildren = new HashMap<>(); - /** - * A node is terminal if it is the last visited node when inserting a path. - */ - private boolean mIsTerminal = false; - private V mValue; - - /** - * Set the value associated with this node. - * @param value the value - */ - public void setValue(V value) { - mValue = value; - } - - /** - * @return the value associated with this node - */ - public V getValue() { - return mValue; - } - - /** - * Inserts a path into the trie. - * Each path component forms a node in the trie, - * root path "/" will correspond to the root of the trie. - * - * @param path a path with components separated by "/" - * @return the last inserted trie node or the last traversed trie node if no node is inserted - */ - public TrieNode insert(String path) { - TrieNode current = this; - for (String component : path.split("/")) { - if (!current.mChildren.containsKey(component)) { - current.mChildren.put(component, new TrieNode<>()); - } - current = current.mChildren.get(component); - } - current.mIsTerminal = true; - return current; - } - - /** - * Traverses the trie along the path components until the traversal cannot proceed any more. - * - * @param path the target path - * @return the terminal nodes sorted by the time they are visited - */ - public List> search(String path) { - List> terminal = new ArrayList<>(); - TrieNode current = this; - if (current.mIsTerminal) { - terminal.add(current); - } - for (String component : path.split("/")) { - if (current.mChildren.containsKey(component)) { - current = current.mChildren.get(component); - if (current.mIsTerminal) { - terminal.add(current); - } - } else { - break; - } - } - return terminal; - } - - /** - * Find terminal component of the full path if one exists. - * @param path the path - * @return the terminal component - */ - public Optional> searchExact(String path) { - return getNode(path).filter(TrieNode::isTerminal); - } - - /** - * Checks whether the path has terminal nodes as parents or children. - * - * @param path the target path - * @param includeChildren whether the check should succeed if the path has children terminal nodes - * @return the terminal nodes sorted by the time they are visited - */ - public boolean hasTerminal(String path, boolean includeChildren) { - TrieNode current = this; - if (current.mIsTerminal) { - return true; - } - for (String component : path.split("/")) { - TrieNode child = current.mChildren.get(component); - if (child != null) { - current = child; - if (current.mIsTerminal) { - return true; - } - } else { - return false; - } - } - return includeChildren; - } - - /** - * Deletes the path from the Trie if the given predicate is true. - * - * @param path the target path - * @param predicate a predicate to decide whether the node should be deleted or not - * @return the removed terminal node, or null if the node is not found or not terminal - */ - public TrieNode deleteIf(String path, java.util.function.Function, - Boolean> predicate) { - java.util.Stack, String>> parents = new java.util.Stack<>(); - TrieNode current = this; - for (String component : path.split("/")) { - if (!current.mChildren.containsKey(component)) { - return null; - } - parents.push(new Pair<>(current, component)); - current = current.mChildren.get(component); - } - if (!current.mIsTerminal) { - return null; - } - if (!predicate.apply(current)) { - return null; - } - TrieNode nodeToDelete = current; - current.mIsTerminal = false; - while (current.mChildren.isEmpty() && !current.mIsTerminal && !parents.empty()) { - Pair, String> parent = parents.pop(); - current = parent.getFirst(); - current.mChildren.remove(parent.getSecond()); - } - return nodeToDelete; - } - - /** - * @return the iterator of TrieNode that are terminals and have no terminal ancestors - */ - public Iterator> getCommonRoots() { - if (mIsTerminal) { - return Collections.singletonList(this).iterator(); - } - return Iterators.concat(mChildren.values().stream().map(TrieNode::getCommonRoots).iterator()); - } - - /** - * Get the terminal children of path (including path). - * @param path the path - * @return the terminal children - */ - public Stream> getLeafChildren(String path) { - return getNode(path).map(current -> - current.getChildrenInternal().filter(TrieNode::isTerminal)).orElseGet(Stream::empty); - } - - private Optional> getNode(String path) { - TrieNode current = this; - String[] components = path.split("/"); - int i; - for (i = 0; i < components.length; i++) { - if (current.mChildren.containsKey(components[i])) { - current = current.mChildren.get(components[i]); - } else { - break; - } - } - if (i != components.length) { - return Optional.empty(); - } - return Optional.of(current); - } - - private boolean isTerminal() { - return mIsTerminal; - } - - private Stream> getChildrenInternal() { - return Stream.concat(Stream.of(this), mChildren.values().stream().flatMap( - TrieNode::getChildrenInternal)); - } - - /** - * Recursively removes all children. - */ - public void clear() { - for (TrieNode child : mChildren.values()) { - child.clear(); - } - mChildren.clear(); - } -} diff --git a/core/common/src/main/java/alluxio/exception/runtime/BlockDoesNotExistRuntimeException.java b/core/common/src/main/java/alluxio/exception/runtime/BlockDoesNotExistRuntimeException.java deleted file mode 100644 index 5eef0d297afa..000000000000 --- a/core/common/src/main/java/alluxio/exception/runtime/BlockDoesNotExistRuntimeException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.exception.runtime; - -import java.text.MessageFormat; - -/** - * The exception thrown when a block does not exist in Alluxio. - */ -public class BlockDoesNotExistRuntimeException extends NotFoundRuntimeException { - - /** - * Constructs a new exception with the specified detail message and cause. - * - * @param blockId block id - */ - public BlockDoesNotExistRuntimeException(long blockId) { - super(MessageFormat.format("BlockMeta not found for blockId {0,number,#}", blockId)); - } -} diff --git a/core/common/src/main/java/alluxio/grpc/GrpcChannelBuilder.java b/core/common/src/main/java/alluxio/grpc/GrpcChannelBuilder.java deleted file mode 100644 index 709f38188ad2..000000000000 --- a/core/common/src/main/java/alluxio/grpc/GrpcChannelBuilder.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.grpc; - -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.security.authentication.AuthType; - -import javax.security.auth.Subject; - -/** - * A gRPC channel builder that authenticates with {@link GrpcServer} at the target during channel - * building. - */ -public final class GrpcChannelBuilder { - private final GrpcServerAddress mAddress; - private final AlluxioConfiguration mConfiguration; - - private Subject mParentSubject; - private AuthType mAuthType; - private GrpcNetworkGroup mNetworkGroup = GrpcNetworkGroup.RPC; - - private GrpcChannelBuilder(GrpcServerAddress address, AlluxioConfiguration conf) { - mAddress = address; - mConfiguration = conf; - mAuthType = - conf.getEnum(PropertyKey.SECURITY_AUTHENTICATION_TYPE, AuthType.class); - } - - /** - * Create a channel builder for given address using the given configuration. - * - * @param address the host address - * @param conf Alluxio configuration - * @return a new instance of {@link GrpcChannelBuilder} - */ - public static GrpcChannelBuilder newBuilder(GrpcServerAddress address, - AlluxioConfiguration conf) { - return new GrpcChannelBuilder(address, conf); - } - - /** - * Sets {@link Subject} for authentication. - * - * @param subject the subject - * @return the updated {@link GrpcChannelBuilder} instance - */ - public GrpcChannelBuilder setSubject(Subject subject) { - mParentSubject = subject; - return this; - } - - /** - * Disables authentication with the server. - * - * @return the updated {@link GrpcChannelBuilder} instance - */ - public GrpcChannelBuilder disableAuthentication() { - mAuthType = AuthType.NOSASL; - return this; - } - - /** - * Sets the pooling strategy. - * - * @param group the networking group - * @return a new instance of {@link GrpcChannelBuilder} - */ - public GrpcChannelBuilder setNetworkGroup(GrpcNetworkGroup group) { - mNetworkGroup = group; - return this; - } - - /** - * Creates an authenticated channel of type {@link GrpcChannel}. - * - * @return the built {@link GrpcChannel} - */ - public GrpcChannel build() throws AlluxioStatusException { - // Acquire a connection from the pool. - GrpcChannel channel = - GrpcChannelPool.INSTANCE.acquireChannel(mNetworkGroup, mAddress, mConfiguration); - try { - channel.authenticate(mAuthType, mParentSubject, mConfiguration); - } catch (Throwable t) { - try { - channel.close(); - } catch (Exception e) { - throw new RuntimeException( - String.format("Failed to release the connection: %s", channel.getChannelKey()), e); - } - if (t instanceof AlluxioStatusException) { - throw t; - } - throw AlluxioStatusException.fromThrowable(t); - } - return channel; - } -} diff --git a/core/common/src/main/java/alluxio/grpc/ServiceVersionClientServiceHandler.java b/core/common/src/main/java/alluxio/grpc/ServiceVersionClientServiceHandler.java deleted file mode 100644 index e02bb4d4cca1..000000000000 --- a/core/common/src/main/java/alluxio/grpc/ServiceVersionClientServiceHandler.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.grpc; - -import alluxio.Constants; -import alluxio.annotation.SuppressFBWarnings; - -import com.google.common.collect.ImmutableSet; -import io.grpc.Status; -import io.grpc.stub.StreamObserver; - -import java.util.Objects; -import java.util.Set; - -/** - * This class is a gRPC handler that serves Alluxio service versions. - */ -public final class ServiceVersionClientServiceHandler - extends ServiceVersionClientServiceGrpc.ServiceVersionClientServiceImplBase { - /** Set of services that are going to be recognized by this versioning service. */ - private final Set mServices; - - /** - * Creates service version handler that allows given services. - * @param services services to allow - */ - public ServiceVersionClientServiceHandler(Set services) { - mServices = ImmutableSet.copyOf(Objects.requireNonNull(services, "services is null")); - } - - @Override - @SuppressFBWarnings(value = "DB_DUPLICATE_SWITCH_CLAUSES") - public void getServiceVersion(GetServiceVersionPRequest request, - StreamObserver responseObserver) { - - ServiceType serviceType = request.getServiceType(); - if (serviceType != ServiceType.UNKNOWN_SERVICE && !mServices.contains(serviceType)) { - responseObserver.onError(Status.NOT_FOUND - .withDescription(String.format("Service %s is not found.", serviceType.name())) - .asException()); - return; - } - - long serviceVersion; - switch (serviceType) { - case FILE_SYSTEM_MASTER_CLIENT_SERVICE: - serviceVersion = Constants.FILE_SYSTEM_MASTER_CLIENT_SERVICE_VERSION; - break; - case FILE_SYSTEM_MASTER_WORKER_SERVICE: - serviceVersion = Constants.FILE_SYSTEM_MASTER_WORKER_SERVICE_VERSION; - break; - case FILE_SYSTEM_MASTER_JOB_SERVICE: - serviceVersion = Constants.FILE_SYSTEM_MASTER_JOB_SERVICE_VERSION; - break; - case BLOCK_MASTER_CLIENT_SERVICE: - serviceVersion = Constants.BLOCK_MASTER_CLIENT_SERVICE_VERSION; - break; - case BLOCK_MASTER_WORKER_SERVICE: - serviceVersion = Constants.BLOCK_MASTER_WORKER_SERVICE_VERSION; - break; - case BLOCK_WORKER_CLIENT_SERVICE: - serviceVersion = Constants.BLOCK_WORKER_CLIENT_SERVICE_VERSION; - break; - case META_MASTER_CONFIG_SERVICE: - serviceVersion = Constants.META_MASTER_CONFIG_SERVICE_VERSION; - break; - case META_MASTER_CLIENT_SERVICE: - serviceVersion = Constants.META_MASTER_CLIENT_SERVICE_VERSION; - break; - case META_MASTER_MASTER_SERVICE: - serviceVersion = Constants.META_MASTER_MASTER_SERVICE_VERSION; - break; - case METRICS_MASTER_CLIENT_SERVICE: - serviceVersion = Constants.METRICS_MASTER_CLIENT_SERVICE_VERSION; - break; - case JOB_MASTER_CLIENT_SERVICE: - serviceVersion = Constants.JOB_MASTER_CLIENT_SERVICE_VERSION; - break; - case JOB_MASTER_WORKER_SERVICE: - serviceVersion = Constants.JOB_MASTER_WORKER_SERVICE_VERSION; - break; - case JOURNAL_MASTER_CLIENT_SERVICE: - serviceVersion = Constants.JOURNAL_MASTER_CLIENT_SERVICE_VERSION; - break; - case TABLE_MASTER_CLIENT_SERVICE: - serviceVersion = Constants.TABLE_MASTER_CLIENT_SERVICE_VERSION; - break; - case RAFT_JOURNAL_SERVICE: - serviceVersion = Constants.RAFT_JOURNAL_SERVICE_VERSION; - break; - default: - serviceVersion = Constants.UNKNOWN_SERVICE_VERSION; - break; - } - responseObserver - .onNext(GetServiceVersionPResponse.newBuilder().setVersion(serviceVersion).build()); - responseObserver.onCompleted(); - } -} diff --git a/core/common/src/main/java/alluxio/heartbeat/HeartbeatThread.java b/core/common/src/main/java/alluxio/heartbeat/HeartbeatThread.java deleted file mode 100644 index 82a0a504632f..000000000000 --- a/core/common/src/main/java/alluxio/heartbeat/HeartbeatThread.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.heartbeat; - -import alluxio.conf.AlluxioConfiguration; -import alluxio.security.authentication.AuthenticatedClientUser; -import alluxio.security.user.UserState; -import alluxio.util.CommonUtils; -import alluxio.util.SecurityUtils; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Thread class to execute a heartbeat periodically. This thread is daemonic, so it will not prevent - * the JVM from exiting. - */ -@NotThreadSafe -public final class HeartbeatThread implements Runnable { - private static final Logger LOG = LoggerFactory.getLogger(HeartbeatThread.class); - - private final String mThreadName; - private final HeartbeatExecutor mExecutor; - private final UserState mUserState; - private HeartbeatTimer mTimer; - private AlluxioConfiguration mConfiguration; - - /** - * @param executorName the executor name defined in {@link HeartbeatContext} - * @param threadId the thread identifier - * @return the thread name combined of executorName and threadId, if threadId is empty or null, - * return executorName - */ - @VisibleForTesting - public static String generateThreadName(String executorName, String threadId) { - if (threadId == null || threadId.isEmpty()) { - return executorName; - } - return executorName + "-" + threadId; - } - - /** - * Creates a {@link Runnable} to execute heartbeats for the given {@link HeartbeatExecutor}. - * - * This class is responsible for closing the given {@link HeartbeatExecutor} when it finishes. - * - * @param executorName identifies the heartbeat thread's executor, should be those defined in - * {@link HeartbeatContext} - * @param threadId the thread identifier, normally, it is empty, but if a heartbeat - * executor is started in multiple threads, this can be used to distinguish them, the heartbeat - * thread's name is a combination of executorName and threadId - * @param executor identifies the heartbeat thread executor; an instance of a class that - * implements the HeartbeatExecutor interface - * @param intervalMs Sleep time between different heartbeat - * @param conf Alluxio configuration - * @param userState the user state for this heartbeat thread - */ - public HeartbeatThread(String executorName, String threadId, HeartbeatExecutor executor, - long intervalMs, AlluxioConfiguration conf, UserState userState) { - mThreadName = generateThreadName(executorName, threadId); - mExecutor = Preconditions.checkNotNull(executor, "executor"); - Class timerClass = HeartbeatContext.getTimerClass(executorName); - mTimer = CommonUtils.createNewClassInstance(timerClass, new Class[] {String.class, long.class}, - new Object[] {mThreadName, intervalMs}); - mConfiguration = conf; - mUserState = userState; - } - - /** - * Convenience method for - * {@link - * #HeartbeatThread(String, String, HeartbeatExecutor, long, AlluxioConfiguration, UserState)} - * where threadId is null. - * - * @param executorName the executor name that is one of those defined in {@link HeartbeatContext} - * @param executor the heartbeat executor - * @param intervalMs the interval between heartbeats - * @param conf the Alluxio configuration - * @param userState the user state for this heartbeat thread - */ - public HeartbeatThread(String executorName, HeartbeatExecutor executor, long intervalMs, - AlluxioConfiguration conf, UserState userState) { - this(executorName, null, executor, intervalMs, conf, userState); - } - - @Override - public void run() { - try { - if (SecurityUtils.isSecurityEnabled(mConfiguration) - && AuthenticatedClientUser.get(mConfiguration) == null) { - AuthenticatedClientUser.set(mUserState.getUser().getName()); - } - } catch (IOException e) { - LOG.error("Failed to set AuthenticatedClientUser in HeartbeatThread."); - } - - // set the thread name - Thread.currentThread().setName(mThreadName); - try { - // Thread.interrupted() clears the interrupt status. Do not call interrupt again to clear it. - while (!Thread.interrupted()) { - // TODO(peis): Fix this. The current implementation consumes one thread even when ticking. - mTimer.tick(); - mExecutor.heartbeat(); - } - } catch (InterruptedException e) { - // Allow thread to exit. - } catch (Exception e) { - LOG.error("Uncaught exception in heartbeat executor, Heartbeat Thread shutting down", e); - } finally { - mExecutor.close(); - } - } - - /** - * Updates the heartbeat interval. - * - * @param intervalMs the heartbeat interval in ms - */ - public void updateIntervalMs(long intervalMs) { - mTimer.setIntervalMs(intervalMs); - } -} diff --git a/core/common/src/main/java/alluxio/heartbeat/HeartbeatTimer.java b/core/common/src/main/java/alluxio/heartbeat/HeartbeatTimer.java deleted file mode 100644 index e68738f4799b..000000000000 --- a/core/common/src/main/java/alluxio/heartbeat/HeartbeatTimer.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.heartbeat; - -/** - * An interface for heartbeat timers. The {@link HeartbeatThread} calls the {@link #tick()} method. - */ -public interface HeartbeatTimer { - - /** - * Sets the heartbeat interval. - * - * @param intervalMs the heartbeat interval in ms - */ - default void setIntervalMs(long intervalMs) { - throw new UnsupportedOperationException("Setting interval is not supported"); - } - - /** - * Waits until next heartbeat should be executed. - * - * @throws InterruptedException if the thread is interrupted while waiting - */ - void tick() throws InterruptedException; -} diff --git a/core/common/src/main/java/alluxio/heartbeat/SleepingTimer.java b/core/common/src/main/java/alluxio/heartbeat/SleepingTimer.java deleted file mode 100644 index 627ef78dfd56..000000000000 --- a/core/common/src/main/java/alluxio/heartbeat/SleepingTimer.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.heartbeat; - -import alluxio.clock.SystemClock; -import alluxio.time.Sleeper; -import alluxio.time.ThreadSleeper; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Clock; -import java.time.Duration; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class can be used for executing heartbeats periodically. - */ -@NotThreadSafe -public final class SleepingTimer implements HeartbeatTimer { - private long mIntervalMs; - private long mPreviousTickMs; - private final String mThreadName; - private final Logger mLogger; - private final Clock mClock; - private final Sleeper mSleeper; - - /** - * Creates a new instance of {@link SleepingTimer}. - * - * @param threadName the thread name - * @param intervalMs the heartbeat interval - */ - public SleepingTimer(String threadName, long intervalMs) { - this(threadName, intervalMs, LoggerFactory.getLogger(SleepingTimer.class), - new SystemClock(), ThreadSleeper.INSTANCE); - } - - /** - * Creates a new instance of {@link SleepingTimer}. - * - * @param threadName the thread name - * @param intervalMs the heartbeat interval - * @param logger the logger to log to - * @param clock for telling the current time - * @param sleeper the utility to use for sleeping - */ - public SleepingTimer(String threadName, long intervalMs, Logger logger, Clock clock, - Sleeper sleeper) { - mIntervalMs = intervalMs; - mThreadName = threadName; - mLogger = logger; - mClock = clock; - mSleeper = sleeper; - } - - @Override - public void setIntervalMs(long intervalMs) { - mIntervalMs = intervalMs; - } - - /** - * Enforces the thread waits for the given interval between consecutive ticks. - * - * @throws InterruptedException if the thread is interrupted while waiting - */ - @Override - public void tick() throws InterruptedException { - if (mPreviousTickMs != 0) { - long executionTimeMs = mClock.millis() - mPreviousTickMs; - if (executionTimeMs > mIntervalMs) { - mLogger.warn("{} last execution took {} ms. Longer than the interval {}", mThreadName, - executionTimeMs, mIntervalMs); - } else { - mSleeper.sleep(Duration.ofMillis(mIntervalMs - executionTimeMs)); - } - } - mPreviousTickMs = mClock.millis(); - } -} diff --git a/core/common/src/main/java/alluxio/master/PollingMasterInquireClient.java b/core/common/src/main/java/alluxio/master/PollingMasterInquireClient.java deleted file mode 100644 index 96875ec915c0..000000000000 --- a/core/common/src/main/java/alluxio/master/PollingMasterInquireClient.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master; - -import static java.util.stream.Collectors.joining; - -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.CancelledException; -import alluxio.exception.status.DeadlineExceededException; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.GetServiceVersionPRequest; -import alluxio.grpc.GrpcChannel; -import alluxio.grpc.GrpcChannelBuilder; -import alluxio.grpc.GrpcServerAddress; -import alluxio.grpc.ServiceType; -import alluxio.grpc.ServiceVersionClientServiceGrpc; -import alluxio.retry.RetryPolicy; -import alluxio.retry.RetryUtils; -import alluxio.security.user.UserState; -import alluxio.uri.Authority; -import alluxio.uri.MultiMasterAuthority; - -import com.google.common.collect.Lists; -import io.grpc.StatusRuntimeException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.InetSocketAddress; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import javax.annotation.Nullable; - -/** - * PollingMasterInquireClient finds the address of the primary master by polling a list of master - * addresses to see if their RPC servers are serving. This works because only primary masters serve - * RPCs. - */ -public class PollingMasterInquireClient implements MasterInquireClient { - private static final Logger LOG = LoggerFactory.getLogger(PollingMasterInquireClient.class); - - private final MultiMasterConnectDetails mConnectDetails; - private final Supplier mRetryPolicySupplier; - private final AlluxioConfiguration mConfiguration; - private final UserState mUserState; - private final ServiceType mServiceType; - - /** - * @param masterAddresses the potential master addresses - * @param alluxioConf Alluxio configuration - * @param userState user state - * @param serviceType service type - */ - public PollingMasterInquireClient(List masterAddresses, - AlluxioConfiguration alluxioConf, - UserState userState, ServiceType serviceType) { - this(masterAddresses, RetryUtils::defaultClientRetry, - alluxioConf, userState, serviceType); - } - - /** - * @param masterAddresses the potential master addresses - * @param retryPolicySupplier the retry policy supplier - * @param alluxioConf Alluxio configuration - * @param serviceType service type - */ - public PollingMasterInquireClient(List masterAddresses, - Supplier retryPolicySupplier, - AlluxioConfiguration alluxioConf, ServiceType serviceType) { - this(masterAddresses, retryPolicySupplier, alluxioConf, - UserState.Factory.create(alluxioConf), serviceType); - } - - /** - * @param masterAddresses the potential master addresses - * @param retryPolicySupplier the retry policy supplier - * @param alluxioConf Alluxio configuration - * @param userState user state - * @param serviceType service type - */ - public PollingMasterInquireClient(List masterAddresses, - Supplier retryPolicySupplier, - AlluxioConfiguration alluxioConf, - UserState userState, ServiceType serviceType) { - mConnectDetails = new MultiMasterConnectDetails(masterAddresses); - mRetryPolicySupplier = retryPolicySupplier; - mConfiguration = alluxioConf; - mUserState = userState; - mServiceType = serviceType; - } - - @Override - public InetSocketAddress getPrimaryRpcAddress() throws UnavailableException { - RetryPolicy retry = mRetryPolicySupplier.get(); - while (retry.attempt()) { - InetSocketAddress address = getAddress(); - if (address != null) { - return address; - } - } - throw new UnavailableException(String.format( - "Failed to determine primary master rpc address after polling each of %s %d times", - mConnectDetails.getAddresses(), retry.getAttemptCount())); - } - - @Nullable - private InetSocketAddress getAddress() { - // Iterate over the masters and try to connect to each of their RPC ports. - List addresses; - if (mConfiguration.getBoolean(PropertyKey.USER_RPC_SHUFFLE_MASTERS_ENABLED)) { - addresses = - Lists.newArrayList(mConnectDetails.getAddresses()); - Collections.shuffle(addresses); - } else { - addresses = mConnectDetails.getAddresses(); - } - - for (InetSocketAddress address : addresses) { - try { - LOG.debug("Checking whether {} is listening for RPCs", address); - pingMetaService(address); - LOG.debug("Successfully connected to {}", address); - return address; - } catch (UnavailableException e) { - LOG.debug("Failed to connect to {}", address); - } catch (DeadlineExceededException e) { - LOG.debug("Timeout while connecting to {}", address); - } catch (CancelledException e) { - LOG.debug("Cancelled while connecting to {}", address); - } catch (AlluxioStatusException e) { - LOG.error("Error while connecting to {}. {}", address, e); - // Breaking the loop on non filtered error. - break; - } - } - return null; - } - - private void pingMetaService(InetSocketAddress address) throws AlluxioStatusException { - // disable authentication in the channel since version service does not require authentication - GrpcChannel channel = - GrpcChannelBuilder.newBuilder(GrpcServerAddress.create(address), mConfiguration) - .setSubject(mUserState.getSubject()) - .disableAuthentication().build(); - ServiceVersionClientServiceGrpc.ServiceVersionClientServiceBlockingStub versionClient = - ServiceVersionClientServiceGrpc.newBlockingStub(channel) - .withDeadlineAfter(mConfiguration.getMs(PropertyKey.USER_MASTER_POLLING_TIMEOUT), - TimeUnit.MILLISECONDS); - try { - versionClient.getServiceVersion(GetServiceVersionPRequest.newBuilder() - .setServiceType(mServiceType).build()); - } catch (StatusRuntimeException e) { - throw AlluxioStatusException.fromThrowable(e); - } finally { - channel.shutdown(); - } - } - - @Override - public List getMasterRpcAddresses() { - return mConnectDetails.getAddresses(); - } - - @Override - public ConnectDetails getConnectDetails() { - return mConnectDetails; - } - - /** - * Details used to connect to the leader Alluxio master when there are multiple potential leaders. - */ - public static class MultiMasterConnectDetails implements ConnectDetails { - private final List mAddresses; - - /** - * @param addresses a list of addresses - */ - public MultiMasterConnectDetails(List addresses) { - mAddresses = addresses; - } - - /** - * @return the addresses - */ - public List getAddresses() { - return mAddresses; - } - - @Override - public Authority toAuthority() { - return new MultiMasterAuthority(mAddresses.stream() - .map(addr -> addr.getHostString() + ":" + addr.getPort()).collect(joining(","))); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof MultiMasterConnectDetails)) { - return false; - } - MultiMasterConnectDetails that = (MultiMasterConnectDetails) o; - return mAddresses.equals(that.mAddresses); - } - - @Override - public int hashCode() { - return Objects.hash(mAddresses); - } - - @Override - public String toString() { - return toAuthority().toString(); - } - } -} diff --git a/core/common/src/main/java/alluxio/master/metastore/rocks/RocksUtils.java b/core/common/src/main/java/alluxio/master/metastore/rocks/RocksUtils.java deleted file mode 100644 index 493787f00850..000000000000 --- a/core/common/src/main/java/alluxio/master/metastore/rocks/RocksUtils.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.metastore.rocks; - -import alluxio.resource.CloseableIterator; - -import com.google.common.primitives.Longs; -import org.rocksdb.RocksIterator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Convenience methods for working with RocksDB. - */ -public final class RocksUtils { - private static final Logger LOG = LoggerFactory.getLogger(RocksUtils.class); - - private RocksUtils() {} // Utils class. - - /** - * @param long1 a long value - * @param long2 a long value - * @return a byte array formed by writing the two long values as bytes - */ - public static byte[] toByteArray(long long1, long long2) { - byte[] key = new byte[2 * Longs.BYTES]; - for (int i = Longs.BYTES - 1; i >= 0; i--) { - key[i] = (byte) (long1 & 0xffL); - long1 >>= Byte.SIZE; - } - for (int i = 2 * Longs.BYTES - 1; i >= Longs.BYTES; i--) { - key[i] = (byte) (long2 & 0xffL); - long2 >>= Byte.SIZE; - } - return key; - } - - /** - * @param n a long value - * @param str a string value - * @return a byte array formed by writing the bytes of n followed by the bytes of str - */ - public static byte[] toByteArray(long n, String str) { - byte[] strBytes = str.getBytes(); - - byte[] key = new byte[Longs.BYTES + strBytes.length]; - for (int i = Longs.BYTES - 1; i >= 0; i--) { - key[i] = (byte) (n & 0xffL); - n >>= Byte.SIZE; - } - System.arraycopy(strBytes, 0, key, Longs.BYTES, strBytes.length); - return key; - } - - /** - * @param bytes an array of bytes - * @param start the place in the array to read the long from - * @return the long - */ - public static long readLong(byte[] bytes, int start) { - return Longs.fromBytes(bytes[start], bytes[start + 1], bytes[start + 2], bytes[start + 3], - bytes[start + 4], bytes[start + 5], bytes[start + 6], bytes[start + 7]); - } - - /** - * Used to parse current {@link RocksIterator} element. - * - * @param return type of parser's next method - */ - public interface RocksIteratorParser { - /** - * Parses and return next element. - * - * @param iter {@link RocksIterator} instance - * @return parsed value - * @throws Exception if parsing fails - */ - T next(RocksIterator iter) throws Exception; - } - - /** - * Used to wrap an {@link CloseableIterator} over {@link RocksIterator}. - * It seeks given iterator to first entry before returning the iterator. - * - * @param rocksIterator the rocks iterator - * @param parser parser to produce iterated values from rocks key-value - * @param iterator value type - * @return wrapped iterator - */ - public static CloseableIterator createCloseableIterator( - RocksIterator rocksIterator, RocksIteratorParser parser) { - rocksIterator.seekToFirst(); - AtomicBoolean valid = new AtomicBoolean(true); - Iterator iter = new Iterator() { - @Override - public boolean hasNext() { - return valid.get() && rocksIterator.isValid(); - } - - @Override - public T next() { - try { - return parser.next(rocksIterator); - } catch (Exception exc) { - LOG.warn("Iteration aborted because of error", exc); - rocksIterator.close(); - valid.set(false); - throw new RuntimeException(exc); - } finally { - rocksIterator.next(); - if (!rocksIterator.isValid()) { - rocksIterator.close(); - valid.set(false); - } - } - } - }; - - return CloseableIterator.create(iter, (whatever) -> rocksIterator.close()); - } -} diff --git a/core/common/src/main/java/alluxio/network/TieredIdentityFactory.java b/core/common/src/main/java/alluxio/network/TieredIdentityFactory.java deleted file mode 100644 index 7d8cee5e5b99..000000000000 --- a/core/common/src/main/java/alluxio/network/TieredIdentityFactory.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.network; - -import alluxio.Constants; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.conf.PropertyKey.Template; -import alluxio.util.ShellUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.TieredIdentity; -import alluxio.wire.TieredIdentity.LocalityTier; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; - -/** - * Class for getting tiered identity. - */ -public final class TieredIdentityFactory { - private static final Logger LOG = LoggerFactory.getLogger(TieredIdentityFactory.class); - - // Synchronize on this lock to modify sInstance. - private static final Object LOCK = new Object(); - @GuardedBy("LOCK") - private static volatile TieredIdentity sInstance = null; - - /** - * @param conf Alluxio configuration - * @return the singleton tiered identity instance for this JVM - */ - public static TieredIdentity localIdentity(AlluxioConfiguration conf) { - if (sInstance == null) { - synchronized (LOCK) { - if (sInstance == null) { - sInstance = create(conf); - LOG.info("Initialized tiered identity {}", sInstance); - } - } - } - return sInstance; - } - - /** - * Creates a tiered identity based on configuration. - * - * @return the created tiered identity - */ - @VisibleForTesting - static TieredIdentity create(AlluxioConfiguration conf) { - TieredIdentity scriptIdentity = fromScript(conf); - - List tiers = new ArrayList<>(); - List orderedTierNames = conf.getList(PropertyKey.LOCALITY_ORDER); - for (int i = 0; i < orderedTierNames.size(); i++) { - String tierName = orderedTierNames.get(i); - String value = null; - if (scriptIdentity != null) { - LocalityTier scriptTier = scriptIdentity.getTier(i); - Preconditions.checkState(scriptTier.getTierName().equals(tierName)); - value = scriptTier.getValue(); - } - // Explicit configuration overrides script output. - if (conf.isSet(Template.LOCALITY_TIER.format(tierName))) { - value = conf.getString(Template.LOCALITY_TIER.format(tierName)); - } - tiers.add(new LocalityTier(tierName, value)); - } - // If the user doesn't specify the value of the "node" tier, we fill in a sensible default. - if (tiers.size() > 0 && tiers.get(0).getTierName().equals(Constants.LOCALITY_NODE) - && tiers.get(0).getValue() == null) { - String name = NetworkAddressUtils.getLocalNodeName(conf); - tiers.set(0, new LocalityTier(Constants.LOCALITY_NODE, name)); - } - return new TieredIdentity(tiers); - } - - /** - * @param conf Alluxio configuration - * @return a tiered identity created from running the user-provided script - */ - @Nullable - private static TieredIdentity fromScript(AlluxioConfiguration conf) { - String scriptName = conf.getString(PropertyKey.LOCALITY_SCRIPT); - Path script = Paths.get(scriptName); - if (!Files.exists(script)) { - URL resource = TieredIdentityFactory.class.getClassLoader().getResource(scriptName); - if (resource != null) { - script = Paths.get(resource.getPath()); - } else { - return null; - } - } - LOG.debug("Found tiered identity script at {}", script); - String identityString; - try { - identityString = ShellUtils.execCommand(script.toString()); - } catch (IOException e) { - throw new RuntimeException( - String.format("Failed to run script %s: %s", script, e.toString()), e); - } - try { - return fromString(identityString, conf); - } catch (IOException e) { - throw new RuntimeException( - String.format("Failed to parse output of running %s: %s", script, e.getMessage()), e); - } - } - - /** - * @param identityString tiered identity string to parse - * @param conf Alluxio configuration - * @return the parsed tiered identity - */ - public static TieredIdentity fromString(String identityString, AlluxioConfiguration conf) - throws IOException { - Set allTiers = Sets.newHashSet(conf.getList(PropertyKey.LOCALITY_ORDER)); - Map tiers = new HashMap<>(); - for (String tier : identityString.split(",")) { - String[] parts = tier.split("="); - if (parts.length != 2) { - throw new IOException(String - .format("Failed to parse tiered identity. The value should be a comma-separated list " - + "of key=value pairs, but was %s", identityString)); - } - String key = parts[0].trim(); - if (tiers.containsKey(key)) { - throw new IOException(String.format( - "Encountered repeated tier definition for %s when parsing tiered identity from string " - + "%s", - key, identityString)); - } - if (!allTiers.contains(key)) { - throw new IOException(String.format("Unrecognized tier: %s. The tiers defined by %s are %s", - key, PropertyKey.LOCALITY_ORDER.toString(), allTiers)); - } - tiers.put(key, parts[1].trim()); - } - List tieredIdentity = new ArrayList<>(); - for (String localityTier : conf.getList(PropertyKey.LOCALITY_ORDER)) { - String value = tiers.getOrDefault(localityTier, null); - tieredIdentity.add(new LocalityTier(localityTier, value)); - } - return new TieredIdentity(tieredIdentity); - } -} diff --git a/core/common/src/main/java/alluxio/resource/Pool.java b/core/common/src/main/java/alluxio/resource/Pool.java deleted file mode 100644 index 8c210d16f6ca..000000000000 --- a/core/common/src/main/java/alluxio/resource/Pool.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.resource; - -import java.io.Closeable; -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * Interface representing a pool of resources to be temporarily used and returned. - * - * @param the type of resource this pool manages - */ -public interface Pool extends Closeable { - /** - * Acquires a resource from the pool. - * - * @return the acquired resource which should not be null - */ - T acquire() throws IOException; - - /** - * Acquires a resource from the pool. - * - * @param time time it takes before timeout if no resource is available - * @param unit the unit of the time - * @return the acquired resource which should not be null - */ - T acquire(long time, TimeUnit unit) throws TimeoutException, IOException; - - /** - * Releases the resource to the pool. - * - * @param resource the resource to release - */ - void release(T resource); - - /** - * @return the current pool size - */ - int size(); -} diff --git a/core/common/src/main/java/alluxio/security/authentication/ClientIpAddressInjector.java b/core/common/src/main/java/alluxio/security/authentication/ClientIpAddressInjector.java deleted file mode 100644 index 07ff7374ef7c..000000000000 --- a/core/common/src/main/java/alluxio/security/authentication/ClientIpAddressInjector.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.security.authentication; - -import io.grpc.ForwardingServerCallListener; -import io.grpc.Grpc; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; - -/** - * Server side interceptor that is used to put remote client's IP Address to thread local storage. - */ -public class ClientIpAddressInjector implements ServerInterceptor { - - /** - * A {@link ThreadLocal} variable to maintain the client's IP address along with a specific - * thread. - */ - private static ThreadLocal sIpAddressThreadLocal = new ThreadLocal<>(); - - /** - * @return IP address of the gRPC client that is making the call - */ - public static String getIpAddress() { - return sIpAddressThreadLocal.get(); - } - - @Override - public ServerCall.Listener interceptCall(ServerCall call, - Metadata headers, ServerCallHandler next) { - /** - * For streaming calls, below will make sure remote IP address is injected prior to creating the - * stream. - */ - setRemoteIpAddress(call); - - /** - * For non-streaming calls to server, below listener will be invoked in the same thread that is - * serving the call. - */ - return new ForwardingServerCallListener.SimpleForwardingServerCallListener( - next.startCall(call, headers)) { - @Override - public void onHalfClose() { - setRemoteIpAddress(call); - super.onHalfClose(); - } - }; - } - - private void setRemoteIpAddress(ServerCall call) { - String remoteIpAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR).toString(); - sIpAddressThreadLocal.set(remoteIpAddress); - } -} diff --git a/core/common/src/main/java/alluxio/security/authorization/AclAction.java b/core/common/src/main/java/alluxio/security/authorization/AclAction.java deleted file mode 100644 index 17b028d88272..000000000000 --- a/core/common/src/main/java/alluxio/security/authorization/AclAction.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.security.authorization; - -/** - * Actions to be controlled in {@link AccessControlList}. - */ -public enum AclAction { - READ, - WRITE, - EXECUTE; - - /** AclAction values. */ - private static final AclAction[] VALUES = AclAction.values(); - - /** - * @param ordinal the ordinal of the target action in {@link AclAction} - * @return the {@link AclAction} with the specified ordinal - * @throws IndexOutOfBoundsException when ordinal is out of range of valid ordinals - */ - public static AclAction ofOrdinal(int ordinal) { - return VALUES[ordinal]; - } -} diff --git a/core/common/src/main/java/alluxio/time/Sleeper.java b/core/common/src/main/java/alluxio/time/Sleeper.java deleted file mode 100644 index cc972c7cd97e..000000000000 --- a/core/common/src/main/java/alluxio/time/Sleeper.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.time; - -import java.time.Duration; - -/** - * An interface for a utility which provides a sleep method. - */ -public interface Sleeper { - - /** - * Sleeps for the given duration. - * - * @param duration the duration to sleep for - * @throws InterruptedException if the sleep is interrupted - */ - void sleep(Duration duration) throws InterruptedException; -} diff --git a/core/common/src/main/java/alluxio/underfs/AtomicFileOutputStream.java b/core/common/src/main/java/alluxio/underfs/AtomicFileOutputStream.java deleted file mode 100644 index e46967d1cff5..000000000000 --- a/core/common/src/main/java/alluxio/underfs/AtomicFileOutputStream.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.underfs; - -import alluxio.exception.ExceptionMessage; -import alluxio.underfs.options.CreateOptions; -import alluxio.util.IdUtils; -import alluxio.util.io.PathUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.OutputStream; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A {@link AtomicFileOutputStream} writes to a temporary file and renames on close. This ensures - * that writing to the stream is atomic, i.e., all writes become readable only after a close. - */ -@NotThreadSafe -public class AtomicFileOutputStream extends OutputStream { - private static final Logger LOG = LoggerFactory.getLogger(AtomicFileOutputStream.class); - - private AtomicFileOutputStreamCallback mUfs; - private CreateOptions mOptions; - private String mPermanentPath; - private String mTemporaryPath; - private OutputStream mTemporaryOutputStream; - private boolean mClosed = false; - - /** - * Constructs a new {@link AtomicFileOutputStream}. - * - * @param path path being written to - * @param ufs the calling {@link UnderFileSystem} - * @param options create options for destination file - */ - public AtomicFileOutputStream(String path, AtomicFileOutputStreamCallback ufs, - CreateOptions options) throws IOException { - mOptions = options; - mPermanentPath = path; - mTemporaryPath = PathUtils.temporaryFileName(IdUtils.getRandomNonNegativeLong(), path); - mTemporaryOutputStream = ufs.createDirect(mTemporaryPath, options); - mUfs = ufs; - } - - @Override - public void write(int b) throws IOException { - mTemporaryOutputStream.write(b); - } - - @Override - public void write(byte[] b) throws IOException { - mTemporaryOutputStream.write(b, 0, b.length); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - mTemporaryOutputStream.write(b, off, len); - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - mTemporaryOutputStream.close(); - - if (!mUfs.renameFile(mTemporaryPath, mPermanentPath)) { - if (!mUfs.deleteFile(mTemporaryPath)) { - LOG.error("Failed to delete temporary file {}", mTemporaryPath); - } - throw new IOException( - ExceptionMessage.FAILED_UFS_RENAME.getMessage(mTemporaryPath, mPermanentPath)); - } - - // Preserve owner and group in case delegation was used to create the path - if (mOptions.getOwner() != null || mOptions.getGroup() != null) { - try { - mUfs.setOwner(mPermanentPath, mOptions.getOwner(), mOptions.getGroup()); - } catch (Exception e) { - LOG.warn("Failed to update the ufs ownership, default values will be used. " + e); - } - } - // TODO(chaomin): consider setMode of the ufs file. - mClosed = true; - } -} - diff --git a/core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java b/core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java deleted file mode 100644 index 6be963b77103..000000000000 --- a/core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.underfs; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.SyncInfo; -import alluxio.collections.Pair; -import alluxio.conf.AlluxioConfiguration; -import alluxio.security.authorization.AccessControlList; -import alluxio.security.authorization.AclEntry; -import alluxio.security.authorization.DefaultAccessControlList; -import alluxio.underfs.options.CreateOptions; -import alluxio.underfs.options.DeleteOptions; -import alluxio.underfs.options.ListOptions; -import alluxio.underfs.options.MkdirsOptions; -import alluxio.underfs.options.OpenOptions; -import alluxio.util.io.PathUtils; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import javax.annotation.Nullable; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A base abstract {@link UnderFileSystem}. - */ -@ThreadSafe -public abstract class BaseUnderFileSystem implements UnderFileSystem { - private static final Logger LOG = LoggerFactory.getLogger(BaseUnderFileSystem.class); - public static final Pair EMPTY_ACL = - new Pair<>(null, null); - - /** The UFS {@link AlluxioURI} used to create this {@link BaseUnderFileSystem}. */ - protected final AlluxioURI mUri; - - /** UFS Configuration options. */ - protected final UnderFileSystemConfiguration mUfsConf; - - /** - * Constructs an {@link BaseUnderFileSystem}. - * - * @param uri the {@link AlluxioURI} used to create this ufs - * @param ufsConf UFS configuration - */ - protected BaseUnderFileSystem(AlluxioURI uri, UnderFileSystemConfiguration ufsConf) { - mUri = Preconditions.checkNotNull(uri, "uri"); - mUfsConf = Preconditions.checkNotNull(ufsConf, "ufsConf"); - } - - @Override - public OutputStream create(String path) throws IOException { - return create(path, CreateOptions.defaults(mUfsConf).setCreateParent(true)); - } - - @Override - public boolean deleteDirectory(String path) throws IOException { - return deleteDirectory(path, DeleteOptions.defaults()); - } - - @Override - public boolean exists(String path) throws IOException { - return isFile(path) || isDirectory(path); - } - - @Override - public Pair getAclPair(String path) - throws IOException { - return EMPTY_ACL; - } - - @Override - public void setAclEntries(String path, List aclEntries) throws IOException { - // Noop here by default - } - - @Override - public AlluxioConfiguration getConfiguration() { - return mUfsConf; - } - - @Override - public String getFingerprint(String path) { - // TODO(yuzhu): include default ACL in the fingerprint - try { - UfsStatus status = getStatus(path); - Pair aclPair = getAclPair(path); - - if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) { - return Fingerprint.create(getUnderFSType(), status).serialize(); - } else { - return Fingerprint.create(getUnderFSType(), status, aclPair.getFirst()).serialize(); - } - } catch (Exception e) { - // In certain scenarios, it is expected that the UFS path does not exist. - LOG.debug("Failed fingerprint. path: {} error: {}", path, e.toString()); - return Constants.INVALID_UFS_FINGERPRINT; - } - } - - @Override - public Fingerprint getParsedFingerprint(String path) { - try { - UfsStatus status = getStatus(path); - Pair aclPair = getAclPair(path); - - if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) { - return Fingerprint.create(getUnderFSType(), status); - } else { - return Fingerprint.create(getUnderFSType(), status, aclPair.getFirst()); - } - } catch (IOException e) { - return Fingerprint.INVALID_FINGERPRINT; - } - } - - @Override - public UfsMode getOperationMode(Map physicalUfsState) { - UfsMode ufsMode = physicalUfsState.get(mUri.getRootPath()); - if (ufsMode != null) { - return ufsMode; - } - return UfsMode.READ_WRITE; - } - - @Override - public List getPhysicalStores() { - return new ArrayList<>(Arrays.asList(mUri.getRootPath())); - } - - @Override - public boolean isObjectStorage() { - return false; - } - - @Override - public boolean isSeekable() { - return false; - } - - @Override - @Nullable - public UfsStatus[] listStatus(String path, ListOptions options) throws IOException { - if (!options.isRecursive()) { - return listStatus(path); - } - path = validatePath(path); - List returnPaths = new ArrayList<>(); - // Each element is a pair of (full path, UfsStatus) - Queue> pathsToProcess = new ArrayDeque<>(); - // We call list initially, so we can return null if the path doesn't denote a directory - UfsStatus[] statuses = listStatus(path); - if (statuses == null) { - return null; - } else { - for (UfsStatus status : statuses) { - pathsToProcess.add(new Pair<>(PathUtils.concatPath(path, status.getName()), status)); - } - } - while (!pathsToProcess.isEmpty()) { - final Pair pathToProcessPair = pathsToProcess.remove(); - final String pathToProcess = pathToProcessPair.getFirst(); - UfsStatus pathStatus = pathToProcessPair.getSecond(); - int beginIndex = path.endsWith(AlluxioURI.SEPARATOR) ? path.length() : path.length() + 1; - returnPaths.add(pathStatus.setName(pathToProcess.substring(beginIndex))); - - if (pathStatus.isDirectory()) { - // Add all of its subpaths - UfsStatus[] children = listStatus(pathToProcess); - if (children != null) { - for (UfsStatus child : children) { - pathsToProcess.add( - new Pair<>(PathUtils.concatPath(pathToProcess, child.getName()), child)); - } - } - } - } - return returnPaths.toArray(new UfsStatus[returnPaths.size()]); - } - - @Override - public InputStream open(String path) throws IOException { - return open(path, OpenOptions.defaults()); - } - - @Override - public boolean mkdirs(String path) throws IOException { - return mkdirs(path, MkdirsOptions.defaults(mUfsConf)); - } - - @Override - public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) { - return new AlluxioURI(ufsBaseUri, PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath), - false); - } - - @Override - public boolean supportsActiveSync() { - return false; - } - - @Override - public SyncInfo getActiveSyncInfo() { - return SyncInfo.emptyInfo(); - } - - @Override - public boolean startActiveSyncPolling(long txId) throws IOException { - return false; - } - - @Override - public boolean stopActiveSyncPolling() { - return false; - } - - @Override - public void startSync(AlluxioURI uri) { } - - @Override - public void stopSync(AlluxioURI uri) { } - - /** - * Clean the path by creating a URI and turning it back to a string. - * - * @param path the path to validate - * @return validated path - */ - protected static String validatePath(String path) { - return new AlluxioURI(path).toString(); - } -} diff --git a/core/common/src/main/java/alluxio/underfs/UfsStatus.java b/core/common/src/main/java/alluxio/underfs/UfsStatus.java deleted file mode 100644 index 2314dce8cab9..000000000000 --- a/core/common/src/main/java/alluxio/underfs/UfsStatus.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.underfs; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Information about a file or a directory in the under file system. Listing contents in a - * {@link UnderFileSystem} returns entries of this class. - */ -@NotThreadSafe -public abstract class UfsStatus { - protected final boolean mIsDirectory; - /** Last modified epoch time in ms, or null if it is not available. */ - protected final Long mLastModifiedTimeMs; - protected String mName; - - // Permissions - protected final String mOwner; - protected final String mGroup; - protected final short mMode; - - protected final Map mXAttr; - - /** - * Creates new instance of {@link UfsStatus}. - * - * @param name relative path of file or directory - * @param isDirectory whether the path is a directory - * @param owner of the file - * @param group of the file - * @param mode of the file - * @param lastModifiedTimeMs last modified epoch time in ms, or null if it is not available - * @param xAttrs any extended attributes on the inode - */ - protected UfsStatus(String name, boolean isDirectory, String owner, String group, short mode, - @Nullable Long lastModifiedTimeMs, @Nullable Map xAttrs) { - mIsDirectory = isDirectory; - mName = name; - mOwner = owner; - mGroup = group; - mMode = mode; - mLastModifiedTimeMs = lastModifiedTimeMs; - mXAttr = xAttrs; - } - - /** - * Creates a new instance of {@link UfsStatus} as a copy. - * - * @param status file information to copy - */ - protected UfsStatus(UfsStatus status) { - mIsDirectory = status.mIsDirectory; - mName = status.mName; - mOwner = status.mOwner; - mGroup = status.mGroup; - mMode = status.mMode; - mLastModifiedTimeMs = status.mLastModifiedTimeMs; - mXAttr = status.mXAttr == null ? null : new HashMap<>(status.mXAttr); - } - - /** - * Create a copy of {@link UfsStatus}. - * - * @return new instance as a copy - */ - public abstract UfsStatus copy(); - - /** - * Converts an array of UFS file status to a listing result where each element in the array is - * a file or directory name. - * - * @param children array of listing statuses - * @return array of file or directory names, or null if the input is null - */ - @Nullable - public static String[] convertToNames(UfsStatus[] children) { - if (children == null) { - return null; - } - String[] ret = new String[children.length]; - for (int i = 0; i < children.length; ++i) { - ret[i] = children[i].getName(); - } - return ret; - } - - /** - * @return true, if the path is a directory - */ - public boolean isDirectory() { - return mIsDirectory; - } - - /** - * @return true, if the path is a file - */ - public boolean isFile() { - return !mIsDirectory; - } - - /** - * Gets the group of the given path. - * - * @return the group of the file - */ - public String getGroup() { - return mGroup; - } - - /** - * Gets the UTC time of when the indicated path was modified recently in ms, or null if the last - * modified time is not available. - * - * @return modification time in milliseconds - */ - @Nullable - public Long getLastModifiedTime() { - return mLastModifiedTimeMs; - } - - /** - * Gets the mode of the given path in short format, e.g 0700. - * - * @return the mode of the file - */ - public short getMode() { - return mMode; - } - - /** - * @return name of file or directory - */ - public String getName() { - return mName; - } - - /** - * Gets the owner of the given path. - * - * @return the owner of the path - */ - public String getOwner() { - return mOwner; - } - - /** - * Returns the extended attributes from the Ufs, if any. - * @return a map of the extended attributes. If none, the map will be empty - */ - @Nullable - public Map getXAttr() { - return mXAttr; - } - - @Override - public int hashCode() { - return Objects.hashCode(mName, mIsDirectory, mOwner, mGroup, mMode, mXAttr); - } - - /** - * Set the name of file or directory. - * - * @param name of entry - * @return this object - */ - public UfsStatus setName(String name) { - mName = name; - return this; - } - - protected MoreObjects.ToStringHelper toStringHelper() { - return MoreObjects.toStringHelper(this) - .add("isDirectory", mIsDirectory) - .add("lastModifiedTimeMs", mLastModifiedTimeMs) - .add("name", mName) - .add("owner", mOwner) - .add("group", mGroup) - .add("mode", mMode) - .add("xAttr", mXAttr); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof UfsStatus)) { - return false; - } - UfsStatus that = (UfsStatus) o; - return Objects.equal(mName, that.mName) - && Objects.equal(mIsDirectory, that.mIsDirectory) - && Objects.equal(mOwner, that.mOwner) - && Objects.equal(mGroup, that.mGroup) - && Objects.equal(mMode, that.mMode) - && Objects.equal(mXAttr, that.mXAttr); - } -} diff --git a/core/common/src/main/java/alluxio/util/ConfigurationUtils.java b/core/common/src/main/java/alluxio/util/ConfigurationUtils.java deleted file mode 100644 index 77b5ca94a46f..000000000000 --- a/core/common/src/main/java/alluxio/util/ConfigurationUtils.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.util; - -import alluxio.Constants; -import alluxio.cli.CommandUtils; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.util.network.NetworkAddressUtils.ServiceType; - -import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import javax.annotation.Nullable; - -/** - * Utilities for working with Alluxio configurations. - */ -public final class ConfigurationUtils { - private static final Logger LOG = LoggerFactory.getLogger(ConfigurationUtils.class); - - private static final String MASTERS = "masters"; - private static final String WORKERS = "workers"; - - private ConfigurationUtils() {} // prevent instantiation - - /** - * Gets the embedded journal addresses to use for the given service type (either master-raft or - * job-master-raft). - * - * @param conf configuration - * @param serviceType the service to get addresses for - * @return the addresses - */ - public static List getEmbeddedJournalAddresses(AlluxioConfiguration conf, - ServiceType serviceType) { - Preconditions.checkState( - serviceType == ServiceType.MASTER_RAFT || serviceType == ServiceType.JOB_MASTER_RAFT); - if (serviceType == ServiceType.MASTER_RAFT) { - return getMasterEmbeddedJournalAddresses(conf); - } - return getJobMasterEmbeddedJournalAddresses(conf); - } - - /** - * @param conf configuration - * @return the embedded journal addresses to use for the master - */ - public static List getMasterEmbeddedJournalAddresses( - AlluxioConfiguration conf) { - PropertyKey property = PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES; - if (conf.isSet(property)) { - return parseInetSocketAddresses(conf.getList(property)); - } - // Fall back on master_hostname:master_raft_port - return Arrays.asList(NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RAFT, conf)); - } - - /** - * @param conf configuration - * @return the embedded journal addresses to use for the job master - */ - public static List getJobMasterEmbeddedJournalAddresses( - AlluxioConfiguration conf) { - PropertyKey jobMasterProperty = PropertyKey.JOB_MASTER_EMBEDDED_JOURNAL_ADDRESSES; - if (conf.isSet(jobMasterProperty)) { - return parseInetSocketAddresses(conf.getList(jobMasterProperty)); - } - // Fall back on using the master embedded journal addresses, with the job master port. - PropertyKey masterProperty = PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES; - int jobRaftPort = NetworkAddressUtils.getPort(ServiceType.JOB_MASTER_RAFT, conf); - if (conf.isSet(masterProperty)) { - return overridePort(getMasterEmbeddedJournalAddresses(conf), jobRaftPort); - } - // Fall back on job_master_hostname:job_master_raft_port. - return Arrays.asList(NetworkAddressUtils.getConnectAddress(ServiceType.JOB_MASTER_RAFT, conf)); - } - - /** - * Gets the RPC addresses of all masters based on the configuration. - * - * @param conf the configuration to use - * @return the master rpc addresses - */ - public static List getMasterRpcAddresses(AlluxioConfiguration conf) { - // First check whether rpc addresses are explicitly configured. - if (conf.isSet(PropertyKey.MASTER_RPC_ADDRESSES)) { - return parseInetSocketAddresses(conf.getList(PropertyKey.MASTER_RPC_ADDRESSES)); - } - - // Fall back on server-side journal configuration. - int rpcPort = NetworkAddressUtils.getPort(NetworkAddressUtils.ServiceType.MASTER_RPC, conf); - return overridePort(getEmbeddedJournalAddresses(conf, ServiceType.MASTER_RAFT), rpcPort); - } - - /** - * Gets the RPC addresses of all job masters based on the configuration. - * - * @param conf the configuration to use - * @return the job master rpc addresses - */ - public static List getJobMasterRpcAddresses(AlluxioConfiguration conf) { - // First check whether job rpc addresses are explicitly configured. - if (conf.isSet(PropertyKey.JOB_MASTER_RPC_ADDRESSES)) { - return parseInetSocketAddresses( - conf.getList(PropertyKey.JOB_MASTER_RPC_ADDRESSES)); - } - - int jobRpcPort = - NetworkAddressUtils.getPort(NetworkAddressUtils.ServiceType.JOB_MASTER_RPC, conf); - // Fall back on explicitly configured regular master rpc addresses. - if (conf.isSet(PropertyKey.MASTER_RPC_ADDRESSES)) { - List addrs = - parseInetSocketAddresses(conf.getList(PropertyKey.MASTER_RPC_ADDRESSES)); - return overridePort(addrs, jobRpcPort); - } - - // Fall back on server-side journal configuration. - return overridePort(getEmbeddedJournalAddresses(conf, ServiceType.JOB_MASTER_RAFT), jobRpcPort); - } - - private static List overridePort(List addrs, int port) { - return StreamUtils.map(addr -> InetSocketAddress.createUnresolved(addr.getHostString(), port), - addrs); - } - - /** - * @param addresses a list of address strings in the form "hostname:port" - * @return a list of InetSocketAddresses representing the given address strings - */ - public static List parseInetSocketAddresses(List addresses) { - List inetSocketAddresses = new ArrayList<>(addresses.size()); - for (String address : addresses) { - try { - inetSocketAddresses.add(NetworkAddressUtils.parseInetSocketAddress(address)); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to parse host:port: " + address, e); - } - } - return inetSocketAddresses; - } - - /** - * @param conf the configuration to use - * @return whether the configuration describes how to find the job master host, either through - * explicit configuration or through zookeeper - */ - public static boolean jobMasterHostConfigured(AlluxioConfiguration conf) { - boolean usingZk = conf.getBoolean(PropertyKey.ZOOKEEPER_ENABLED) - && conf.isSet(PropertyKey.ZOOKEEPER_ADDRESS); - return conf.isSet(PropertyKey.JOB_MASTER_HOSTNAME) || usingZk - || getJobMasterRpcAddresses(conf).size() > 1; - } - - /** - * Returns a unified message for cases when the master hostname cannot be determined. - * - * @param serviceName the name of the service that couldn't run. i.e. Alluxio worker, fsadmin - * shell, etc. - * @return a string with the message - */ - public static String getMasterHostNotConfiguredMessage(String serviceName) { - return getHostNotConfiguredMessage(serviceName, "master", PropertyKey.MASTER_HOSTNAME, - PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES); - } - - /** - * Returns a unified message for cases when the job master hostname cannot be determined. - * - * @param serviceName the name of the service that couldn't run. i.e. Alluxio worker, fsadmin - * shell, etc. - * @return a string with the message - */ - public static String getJobMasterHostNotConfiguredMessage(String serviceName) { - return getHostNotConfiguredMessage(serviceName, "job master", PropertyKey.JOB_MASTER_HOSTNAME, - PropertyKey.JOB_MASTER_EMBEDDED_JOURNAL_ADDRESSES); - } - - private static String getHostNotConfiguredMessage(String serviceName, String masterName, - PropertyKey masterHostnameKey, PropertyKey embeddedJournalKey) { - return ExceptionMessage.UNABLE_TO_DETERMINE_MASTER_HOSTNAME.getMessage(serviceName, masterName, - masterHostnameKey.getName(), PropertyKey.ZOOKEEPER_ENABLED.getName(), - PropertyKey.ZOOKEEPER_ADDRESS.getName(), embeddedJournalKey.getName()); - } - - /** - * Checks that the given property key is a ratio from 0.0 and 1.0, throwing an exception if it is - * not. - * - * @param conf the configuration for looking up the property key - * @param key the property key - * @return the property value - */ - public static float checkRatio(AlluxioConfiguration conf, PropertyKey key) { - double value = conf.getDouble(key); - Preconditions.checkState(value <= 1.0, "Property %s must not exceed 1, but it is set to %s", - key.getName(), value); - Preconditions.checkState(value >= 0.0, "Property %s must be non-negative, but it is set to %s", - key.getName(), value); - return (float) value; - } - - /** - * @param conf the configuration to use - * @return whether the configuration describes how to find the master host, either through - * explicit configuration or through zookeeper - */ - public static boolean masterHostConfigured(AlluxioConfiguration conf) { - boolean usingZk = conf.getBoolean(PropertyKey.ZOOKEEPER_ENABLED) - && conf.isSet(PropertyKey.ZOOKEEPER_ADDRESS); - return conf.isSet(PropertyKey.MASTER_HOSTNAME) || usingZk - || getMasterRpcAddresses(conf).size() > 1; - } - - /** - * @param conf the configuration use - * @return whether the configuration specifies to run in ha mode - */ - public static boolean isHaMode(AlluxioConfiguration conf) { - return conf.getBoolean(PropertyKey.ZOOKEEPER_ENABLED) || getMasterRpcAddresses(conf).size() > 1; - } - - /** - * @param value the value or null (value is not set) - * @return the value or "(no value set)" when the value is not set - */ - public static String valueAsString(String value) { - return value == null ? "(no value set)" : value; - } - - /** - * @param conf the configuration - * @return the alluxio scheme and authority determined by the configuration - */ - public static String getSchemeAuthority(AlluxioConfiguration conf) { - if (conf.getBoolean(PropertyKey.ZOOKEEPER_ENABLED)) { - return Constants.HEADER + "zk@" + conf.get(PropertyKey.ZOOKEEPER_ADDRESS); - } - List addresses = getMasterRpcAddresses(conf); - - if (addresses.size() > 1) { - return Constants.HEADER + addresses.stream().map(InetSocketAddress::toString) - .collect(Collectors.joining(",")); - } - - return Constants.HEADER + conf.get(PropertyKey.MASTER_HOSTNAME) + ":" + conf - .get(PropertyKey.MASTER_RPC_PORT); - } - - /** - * Returns the input string as a list, splitting on a specified delimiter. - * - * @param value the value to split - * @param delimiter the delimiter to split the values - * @return the list of values for input string - */ - public static List parseAsList(String value, String delimiter) { - return Lists.newArrayList(Splitter.on(delimiter).trimResults().omitEmptyStrings() - .split(value)); - } - - /** - * Reads a list of nodes from given file name ignoring comments and empty lines. - * Can be used to read conf/workers or conf/masters. - * @param fileName name of a file that contains the list of the nodes - * @return list of the node names, null when file fails to read - */ - @Nullable - private static Set readNodeList(String fileName, AlluxioConfiguration conf) { - String confDir = conf.getString(PropertyKey.CONF_DIR); - return CommandUtils.readNodeList(confDir, fileName); - } - - /** - * Gets list of masters in conf directory. - * - * @param conf configuration - * @return master hostnames - */ - public static Set getMasterHostnames(AlluxioConfiguration conf) { - return readNodeList(MASTERS, conf); - } - - /** - * Gets list of workers in conf directory. - * - * @param conf configuration - * @return workers hostnames - */ - public static Set getWorkerHostnames(AlluxioConfiguration conf) { - return readNodeList(WORKERS, conf); - } - - /** - * Gets list of masters/workers in conf directory. - * - * @param conf configuration - * @return server hostnames - */ - public static Set getServerHostnames(AlluxioConfiguration conf) { - return Sets.union(getMasterHostnames(conf), getWorkerHostnames(conf)); - } -} diff --git a/core/common/src/main/java/alluxio/util/TieredIdentityUtils.java b/core/common/src/main/java/alluxio/util/TieredIdentityUtils.java deleted file mode 100644 index f355d445e2e7..000000000000 --- a/core/common/src/main/java/alluxio/util/TieredIdentityUtils.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.util; - -import alluxio.Constants; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.TieredIdentity; - -import java.net.UnknownHostException; -import java.util.List; -import java.util.Optional; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Utility functions for working with tiered identity. - */ -@ThreadSafe -public final class TieredIdentityUtils { - - /** - * Locality comparison for wire type locality tiers, two locality tiers matches if both name and - * values are equal, or for the "node" tier, if the node names resolve to the same IP address. - * - * @param tier a wire type locality tier - * @param otherTier a wire type locality tier to compare to - * @param resolveIpAddress whether or not to resolve hostnames to IP addresses for node locality - * @return true if the wire type locality tier matches the given tier - */ - public static boolean matches(TieredIdentity.LocalityTier tier, - TieredIdentity.LocalityTier otherTier, boolean resolveIpAddress) { - String otherTierName = otherTier.getTierName(); - if (!tier.getTierName().equals(otherTierName)) { - return false; - } - String otherTierValue = otherTier.getValue(); - if (tier.getValue() != null && tier.getValue().equals(otherTierValue)) { - return true; - } - // For node tiers, attempt to resolve hostnames to IP addresses, this avoids common - // misconfiguration errors where a worker is using one hostname and the client is using - // another. - if (resolveIpAddress) { - if (Constants.LOCALITY_NODE.equals(tier.getTierName())) { - try { - String tierIpAddress = NetworkAddressUtils.resolveIpAddress(tier.getValue()); - String otherTierIpAddress = NetworkAddressUtils.resolveIpAddress(otherTierValue); - if (tierIpAddress != null && tierIpAddress.equals(otherTierIpAddress)) { - return true; - } - } catch (UnknownHostException e) { - return false; - } - } - } - return false; - } - - /** - * @param tieredIdentity the tiered identity - * @param identities the tiered identities to compare to - * @param conf Alluxio configuration - * @return the identity closest to this one. If none of the identities match, the first identity - * is returned - */ - public static Optional nearest(TieredIdentity tieredIdentity, - List identities, AlluxioConfiguration conf) { - if (identities.isEmpty()) { - return Optional.empty(); - } - for (TieredIdentity.LocalityTier tier : tieredIdentity.getTiers()) { - if (tier == null) { - return Optional.of(identities.get(0)); - } - for (TieredIdentity identity : identities) { - for (TieredIdentity.LocalityTier otherTier : identity.getTiers()) { - if (matches(tier, otherTier, conf.getBoolean(PropertyKey.LOCALITY_COMPARE_NODE_IP))) { - return Optional.of(identity); - } - } - } - } - return Optional.of(identities.get(0)); - } - - private TieredIdentityUtils() {} // prevent instantiation -} diff --git a/core/common/src/main/java/alluxio/util/executor/ExecutorServiceFactories.java b/core/common/src/main/java/alluxio/util/executor/ExecutorServiceFactories.java deleted file mode 100644 index 56fc5dd403ad..000000000000 --- a/core/common/src/main/java/alluxio/util/executor/ExecutorServiceFactories.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.util.executor; - -import alluxio.util.ThreadFactoryUtils; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * Convenience methods for constructing instances of {@link ExecutorServiceFactory}. - */ -public final class ExecutorServiceFactories { - /** - * Returns a {@link ExecutorServiceFactory} which creates threadpool executors with the given base - * name. Created threads will be daemonic. - * - * @param name the base name for executor thread names - * @return the {@link ExecutorServiceFactory} - */ - public static ExecutorServiceFactory cachedThreadPool(String name) { - return () -> Executors.newCachedThreadPool(ThreadFactoryUtils.build(name + "-%d", true)); - } - - /** - * Returns a {@link ExecutorServiceFactory} which creates threadpool executors with the given base - * name and number of threads. Created threads will be daemonic. - * - * @param name the base name for executor thread names - * @param nThreads the number of threads to create executors with - * @return the {@link ExecutorServiceFactory} - */ - public static ExecutorServiceFactory fixedThreadPool(String name, int nThreads) { - return () -> Executors.newFixedThreadPool(nThreads, - ThreadFactoryUtils.build(name + "-%d", true)); - } - - /** - * @param executorService the executor service to supply - * @return an {@link ExecutorServiceFactory} which always returns the given - * {@link ExecutorService} - */ - public static ExecutorServiceFactory constantExecutorServiceFactory( - ExecutorService executorService) { - return () -> executorService; - } - - private ExecutorServiceFactories() {} // Not intended for instantiation. -} diff --git a/core/common/src/main/java/alluxio/util/proto/ProtoUtils.java b/core/common/src/main/java/alluxio/util/proto/ProtoUtils.java deleted file mode 100644 index 5afd2ec5afb7..000000000000 --- a/core/common/src/main/java/alluxio/util/proto/ProtoUtils.java +++ /dev/null @@ -1,504 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.util.proto; - -import alluxio.grpc.SetAclAction; -import alluxio.proto.journal.File; -import alluxio.proto.shared.Acl; -import alluxio.proto.status.Status.PStatus; -import alluxio.security.authorization.AccessControlList; -import alluxio.security.authorization.AclAction; -import alluxio.security.authorization.AclActions; -import alluxio.security.authorization.AclEntry; -import alluxio.security.authorization.AclEntryType; -import alluxio.security.authorization.DefaultAccessControlList; -import alluxio.security.authorization.ExtendedACLEntries; - -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.InvalidProtocolBufferException; -import io.grpc.Status; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Protobuf related utils. - */ -public final class ProtoUtils { - private ProtoUtils() {} // prevent instantiation - - /** - * A wrapper of {@link CodedInputStream#readRawVarint32(InputStream)}. - * - * @param firstByte first byte in the input stream - * @param input input stream - * @return an int value read from the input stream - */ - public static int readRawVarint32(int firstByte, InputStream input) throws IOException { - return CodedInputStream.readRawVarint32(firstByte, input); - } - - /** - * A wrapper of {@link CodedInputStream#readRawVarint32(InputStream)}. - * - * @param input input stream - * @return an int value read from the input stream - */ - public static int readRawVarint32(InputStream input) throws IOException { - int firstByte = input.read(); - return CodedInputStream.readRawVarint32(firstByte, input); - } - - /** - * Checks whether the exception is an {@link InvalidProtocolBufferException} thrown because of - * a truncated message. - * - * @param e the exception - * @return whether the exception is an {@link InvalidProtocolBufferException} thrown because of - * a truncated message. - */ - public static boolean isTruncatedMessageException(IOException e) { - if (!(e instanceof InvalidProtocolBufferException)) { - return false; - } - String truncatedMessage; - try { - Method method = InvalidProtocolBufferException.class.getMethod("truncatedMessage"); - method.setAccessible(true); - truncatedMessage = (String) method.invoke(null); - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ee) { - throw new RuntimeException(ee); - } - return e.getMessage().equals(truncatedMessage); - } - - /** - * @param acl {@link AccessControlList} - * @return protobuf representation - */ - public static Acl.AccessControlList toProto(AccessControlList acl) { - Acl.AccessControlList.Builder builder = Acl.AccessControlList.newBuilder(); - builder.setOwningUser(acl.getOwningUser()); - builder.setOwningGroup(acl.getOwningGroup()); - - // base entries - builder.addUserActions(Acl.NamedAclActions.newBuilder() - .setName(AccessControlList.OWNING_USER_KEY) - .setActions(toProto(acl.getOwningUserActions())) - .build()); - builder.addGroupActions(Acl.NamedAclActions.newBuilder() - .setName(AccessControlList.OWNING_GROUP_KEY) - .setActions(toProto(acl.getOwningGroupActions())) - .build()); - builder.setOtherActions(toProto(acl.getOtherActions())); - - if (acl.getExtendedEntries() != null) { - builder.addAllUserActions(getNamedUsersProto(acl.getExtendedEntries())); - builder.addAllGroupActions(getNamedGroupsProto(acl.getExtendedEntries())); - builder.setMaskActions(toProto(acl.getExtendedEntries().getMask())); - } - - if (acl instanceof DefaultAccessControlList) { - DefaultAccessControlList defaultAcl = (DefaultAccessControlList) acl; - builder.setIsDefault(true); - builder.setIsEmpty(defaultAcl.isEmpty()); - } else { - builder.setIsDefault(false); - // non default acl is always not empty - builder.setIsEmpty(false); - } - return builder.build(); - } - - /** - * @param actions the {@link AclActions} - * @return the protobuf representation of {@link AclActions} - */ - public static Acl.AclActions toProto(AclActions actions) { - Acl.AclActions.Builder builder = Acl.AclActions.newBuilder(); - for (AclAction action : actions.getActions()) { - Acl.AclAction pAction = toProto(action); - builder.addActions(pAction); - } - return builder.build(); - } - - /** - * @param aclAction the acl action - * @return the protobuf representation of action - */ - public static Acl.AclAction toProto(AclAction aclAction) { - switch (aclAction) { - case READ: - return Acl.AclAction.READ; - case WRITE: - return Acl.AclAction.WRITE; - case EXECUTE: - return Acl.AclAction.EXECUTE; - default: - throw new IllegalStateException("Unknown acl action: " + aclAction); - } - } - - /** - * @param aclEntry the acl entry - * @return the proto representation of instance - */ - public static Acl.AclEntry toProto(AclEntry aclEntry) { - Acl.AclEntry.Builder builder = Acl.AclEntry.newBuilder(); - builder.setType(toProto(aclEntry.getType())); - builder.setSubject(aclEntry.getSubject()); - builder.setIsDefault(aclEntry.isDefault()); - for (AclAction action : aclEntry.getActions().getActions()) { - builder.addActions(toProto(action)); - } - return builder.build(); - } - - /** - * Converts wire type to proto type. - * - * @param aclAction the acl action to convert - * @return {@link File.PSetAclAction} equivalent - */ - public static File.PSetAclAction toProto(SetAclAction aclAction) { - switch (aclAction) { - case REPLACE: - return File.PSetAclAction.REPLACE; - case MODIFY: - return File.PSetAclAction.MODIFY; - case REMOVE: - return File.PSetAclAction.REMOVE; - case REMOVE_ALL: - return File.PSetAclAction.REMOVE_ALL; - case REMOVE_DEFAULT: - return File.PSetAclAction.REMOVE_DEFAULT; - default: - throw new IllegalStateException("Unrecognized set acl action: " + aclAction); - } - } - - /** - * Converts an internal exception status to a protocol buffer type status. - * - * @param status the status to convert - * @return the protocol buffer type status - */ - public static PStatus toProto(Status status) { - switch (status.getCode()) { - case ABORTED: - return PStatus.ABORTED; - case ALREADY_EXISTS: - return PStatus.ALREADY_EXISTS; - case CANCELLED: - return PStatus.CANCELED; - case DATA_LOSS: - return PStatus.DATA_LOSS; - case DEADLINE_EXCEEDED: - return PStatus.DEADLINE_EXCEEDED; - case FAILED_PRECONDITION: - return PStatus.FAILED_PRECONDITION; - case INTERNAL: - return PStatus.INTERNAL; - case INVALID_ARGUMENT: - return PStatus.INVALID_ARGUMENT; - case NOT_FOUND: - return PStatus.NOT_FOUND; - case OK: - return PStatus.OK; - case OUT_OF_RANGE: - return PStatus.OUT_OF_RANGE; - case PERMISSION_DENIED: - return PStatus.PERMISSION_DENIED; - case RESOURCE_EXHAUSTED: - return PStatus.RESOURCE_EXHAUSTED; - case UNAUTHENTICATED: - return PStatus.UNAUTHENTICATED; - case UNAVAILABLE: - return PStatus.UNAVAILABLE; - case UNIMPLEMENTED: - return PStatus.UNIMPLEMENTED; - case UNKNOWN: - return PStatus.UNKNOWN; - default: - return PStatus.UNKNOWN; - } - } - - /** - * @param entryType the acl entry type - * @return the proto representation of enum - */ - public static Acl.AclEntryType toProto(AclEntryType entryType) { - switch (entryType) { - case OWNING_USER: - return Acl.AclEntryType.OWNER; - case NAMED_USER: - return Acl.AclEntryType.NAMED_USER; - case OWNING_GROUP: - return Acl.AclEntryType.OWNING_GROUP; - case NAMED_GROUP: - return Acl.AclEntryType.NAMED_GROUP; - case MASK: - return Acl.AclEntryType.MASK; - case OTHER: - return Acl.AclEntryType.OTHER; - default: - throw new IllegalStateException("Unknown AclEntryType: " + entryType); - } - } - - /** - * @param acl the protobuf representation - * @return {@link AccessControlList} - */ - public static AccessControlList fromProto(Acl.AccessControlList acl) { - AccessControlList ret; - if (acl.hasIsDefault() && acl.getIsDefault()) { - ret = new DefaultAccessControlList(); - } else { - ret = new AccessControlList(); - } - ret.setOwningUser(acl.getOwningUser().intern()); - ret.setOwningGroup(acl.getOwningGroup().intern()); - - if (acl.getIsEmpty()) { - return ret; - } - - // true if there are any extended entries (named user or named group) - boolean hasExtended = false; - - for (Acl.NamedAclActions namedActions : acl.getUserActionsList()) { - String name = namedActions.getName(); - AclActions actions = fromProto(namedActions.getActions()); - AclEntry entry; - if (name.equals(AccessControlList.OWNING_USER_KEY)) { - entry = new AclEntry.Builder().setType(AclEntryType.OWNING_USER) - .setSubject(acl.getOwningUser()).setActions(actions).build(); - } else { - hasExtended = true; - entry = new AclEntry.Builder().setType(AclEntryType.NAMED_USER) - .setSubject(name).setActions(actions).build(); - } - ret.setEntry(entry); - } - - for (Acl.NamedAclActions namedActions : acl.getGroupActionsList()) { - String name = namedActions.getName(); - AclActions actions = fromProto(namedActions.getActions()); - AclEntry entry; - if (name.equals(AccessControlList.OWNING_GROUP_KEY)) { - entry = new AclEntry.Builder().setType(AclEntryType.OWNING_GROUP) - .setSubject(acl.getOwningGroup()).setActions(actions).build(); - } else { - hasExtended = true; - entry = new AclEntry.Builder().setType(AclEntryType.NAMED_GROUP) - .setSubject(name).setActions(actions).build(); - } - ret.setEntry(entry); - } - - if (hasExtended) { - // Only set the mask if there are any extended acl entries. - AclActions actions = fromProto(acl.getMaskActions()); - AclEntry entry = new AclEntry.Builder().setType(AclEntryType.MASK) - .setActions(actions).build(); - ret.setEntry(entry); - } - - AclActions actions = fromProto(acl.getOtherActions()); - AclEntry entry = new AclEntry.Builder().setType(AclEntryType.OTHER) - .setActions(actions).build(); - ret.setEntry(entry); - - return ret; - } - - /** - * @param actions the protobuf representation of {@link AclActions} - * @return the {@link AclActions} decoded from the protobuf representation - */ - public static AclActions fromProto(Acl.AclActions actions) { - AclActions ret = new AclActions(); - for (Acl.AclAction action : actions.getActionsList()) { - ret.add(fromProto(action)); - } - return ret; - } - - /** - * @param action the protobuf representation of {@link AclAction} - * @return the {@link AclAction} decoded from the protobuf representation - */ - public static AclAction fromProto(Acl.AclAction action) { - switch (action) { - case READ: - return AclAction.READ; - case WRITE: - return AclAction.WRITE; - case EXECUTE: - return AclAction.EXECUTE; - default: - throw new IllegalStateException("Unknown protobuf acl action: " + action); - } - } - - /** - * @param pEntry the proto representation - * @return the {@link AclEntry} instance created from the proto representation - */ - public static AclEntry fromProto(Acl.AclEntry pEntry) { - AclEntry.Builder builder = new AclEntry.Builder(); - builder.setType(fromProto(pEntry.getType())); - builder.setSubject(pEntry.getSubject()); - builder.setIsDefault(pEntry.getIsDefault()); - - for (Acl.AclAction pAction : pEntry.getActionsList()) { - builder.addAction(fromProto(pAction)); - } - return builder.build(); - } - - /** - * @param pAclEntryType the proto representation - * @return the {@link AclEntryType} created from the proto representation - */ - public static AclEntryType fromProto(Acl.AclEntryType pAclEntryType) { - switch (pAclEntryType) { - case OWNER: - return AclEntryType.OWNING_USER; - case NAMED_USER: - return AclEntryType.NAMED_USER; - case OWNING_GROUP: - return AclEntryType.OWNING_GROUP; - case NAMED_GROUP: - return AclEntryType.NAMED_GROUP; - case MASK: - return AclEntryType.MASK; - case OTHER: - return AclEntryType.OTHER; - default: - throw new IllegalStateException("Unknown proto AclEntryType: " + pAclEntryType); - } - } - - /** - * Converts proto type to wire type. - * - * @param pSetAclAction {@link File.PSetAclAction} - * @return {@link SetAclAction} equivalent - */ - public static SetAclAction fromProto(File.PSetAclAction pSetAclAction) { - if (pSetAclAction == null) { - throw new IllegalStateException("Null proto set acl action."); - } - switch (pSetAclAction) { - case REPLACE: - return SetAclAction.REPLACE; - case MODIFY: - return SetAclAction.MODIFY; - case REMOVE: - return SetAclAction.REMOVE; - case REMOVE_ALL: - return SetAclAction.REMOVE_ALL; - case REMOVE_DEFAULT: - return SetAclAction.REMOVE_DEFAULT; - default: - throw new IllegalStateException("Unrecognized proto set acl action: " + pSetAclAction); - } - } - - /** - * Creates a {@link Status} from a protocol buffer type status. - * - * @param status the protocol buffer type status - * @return the corresponding {@link Status} - */ - public static Status fromProto(alluxio.proto.status.Status.PStatus status) { - switch (status) { - case ABORTED: - return Status.ABORTED; - case ALREADY_EXISTS: - return Status.ALREADY_EXISTS; - case CANCELED: - return Status.CANCELLED; - case DATA_LOSS: - return Status.DATA_LOSS; - case DEADLINE_EXCEEDED: - return Status.DEADLINE_EXCEEDED; - case FAILED_PRECONDITION: - return Status.FAILED_PRECONDITION; - case INTERNAL: - return Status.INTERNAL; - case INVALID_ARGUMENT: - return Status.INVALID_ARGUMENT; - case NOT_FOUND: - return Status.NOT_FOUND; - case OK: - return Status.OK; - case OUT_OF_RANGE: - return Status.OUT_OF_RANGE; - case PERMISSION_DENIED: - return Status.PERMISSION_DENIED; - case RESOURCE_EXHAUSTED: - return Status.RESOURCE_EXHAUSTED; - case UNAUTHENTICATED: - return Status.UNAUTHENTICATED; - case UNAVAILABLE: - return Status.UNAVAILABLE; - case UNIMPLEMENTED: - return Status.UNIMPLEMENTED; - case UNKNOWN: - return Status.UNKNOWN; - default: - return Status.UNKNOWN; - } - } - - /** - * @param entries the extended acl entries - * @return a list of the proto representation of the named users actions - */ - public static List getNamedUsersProto(ExtendedACLEntries entries) { - List actions = new ArrayList<>(entries.getNamedUserActions().size()); - for (Map.Entry kv : entries.getNamedUserActions().entrySet()) { - Acl.NamedAclActions namedActions = Acl.NamedAclActions.newBuilder() - .setName(kv.getKey()) - .setActions(toProto(kv.getValue())) - .build(); - actions.add(namedActions); - } - return actions; - } - - /** - * @param entries the extended acl entries - * @return a list of the proto representation of the named group actions - */ - public static List getNamedGroupsProto(ExtendedACLEntries entries) { - List actions = new ArrayList<>(entries.getNamedGroupActions().size()); - for (Map.Entry kv : entries.getNamedGroupActions().entrySet()) { - Acl.NamedAclActions namedActions = Acl.NamedAclActions.newBuilder() - .setName(kv.getKey()) - .setActions(toProto(kv.getValue())) - .build(); - actions.add(namedActions); - } - return actions; - } -} diff --git a/core/common/src/main/java/alluxio/wire/ConfigHash.java b/core/common/src/main/java/alluxio/wire/ConfigHash.java deleted file mode 100644 index b41a60e09c46..000000000000 --- a/core/common/src/main/java/alluxio/wire/ConfigHash.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import alluxio.grpc.GetConfigHashPResponse; - -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Hashes of cluster and path level configurations. - */ -@ThreadSafe -public class ConfigHash { - private final String mClusterConfigHash; - private final String mPathConfigHash; - - /** - * Constructs a new ConfigHash. - * - * @param clusterConfigHash cluster configuration hash, cannot be null - * @param pathConfigHash path configuration hash, cannot be null - */ - public ConfigHash(String clusterConfigHash, String pathConfigHash) { - Preconditions.checkNotNull(clusterConfigHash, "clusterConfigHash"); - Preconditions.checkNotNull(pathConfigHash, "pathConfigHash"); - mClusterConfigHash = clusterConfigHash; - mPathConfigHash = pathConfigHash; - } - - private ConfigHash(GetConfigHashPResponse response) { - mClusterConfigHash = response.getClusterConfigHash(); - mPathConfigHash = response.getPathConfigHash(); - } - - /** - * @param response the grpc representation of configuration hash - * @return the wire representation of the proto response - */ - public static ConfigHash fromProto(GetConfigHashPResponse response) { - return new ConfigHash(response); - } - - /** - * @return the proto representation - */ - public GetConfigHashPResponse toProto() { - GetConfigHashPResponse.Builder response = GetConfigHashPResponse.newBuilder(); - if (mClusterConfigHash != null) { - response.setClusterConfigHash(mClusterConfigHash); - } - if (mPathConfigHash != null) { - response.setPathConfigHash(mPathConfigHash); - } - return response.build(); - } - - /** - * @return hash of cluster level configuration - */ - public String getClusterConfigHash() { - return mClusterConfigHash; - } - - /** - * @return hash of path level configuration - */ - public String getPathConfigHash() { - return mPathConfigHash; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof ConfigHash)) { - return false; - } - ConfigHash that = (ConfigHash) o; - return mClusterConfigHash.equals(that.mClusterConfigHash) - && mPathConfigHash.equals(that.mPathConfigHash); - } - - @Override - public int hashCode() { - return Objects.hashCode(mClusterConfigHash, mPathConfigHash); - } -} diff --git a/core/common/src/main/java/alluxio/wire/Configuration.java b/core/common/src/main/java/alluxio/wire/Configuration.java deleted file mode 100644 index 566af58b2b3a..000000000000 --- a/core/common/src/main/java/alluxio/wire/Configuration.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import alluxio.conf.Source; -import alluxio.grpc.ConfigProperties; -import alluxio.grpc.ConfigProperty; -import alluxio.grpc.GetConfigurationPResponse; - -import com.google.common.base.Preconditions; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Represents cluster level and path level configuration returned by meta master. - */ -@ThreadSafe -public final class Configuration { - /** List of cluster level properties. */ - private final List mClusterConf; - /** Map from path to path level properties. */ - private final Map> mPathConf; - /** Cluster configuration hash. */ - private final String mClusterConfHash; - /** Path configuration hash. */ - private final String mPathConfHash; - - /** - * @return new configuration builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Configuration builder. - */ - @NotThreadSafe - public static final class Builder { - private List mClusterConf = new ArrayList<>(); - private Map> mPathConf = new HashMap<>(); - private String mClusterConfHash; - private String mPathConfHash; - - /** - * Adds a cluster level property. - * - * @param name property name - * @param value property value - * @param source property source - */ - public void addClusterProperty(String name, @Nullable Object value, Source source) { - mClusterConf.add(new Property(name, value, source)); - } - - /** - * Adds a path level property. - * - * @param path the path - * @param name property name - * @param value property value - */ - public void addPathProperty(String path, String name, String value) { - mPathConf.computeIfAbsent(path, k -> new ArrayList()).add( - new Property(name, value, Source.PATH_DEFAULT)); - } - - /** - * Sets hash of path configurations. - * - * @param hash the hash - */ - public void setClusterConfHash(String hash) { - Preconditions.checkNotNull(hash, "hash"); - mClusterConfHash = hash; - } - - /** - * Sets hash of path configurations. - * - * @param hash the hash - */ - public void setPathConfHash(String hash) { - Preconditions.checkNotNull(hash, "hash"); - mPathConfHash = hash; - } - - /** - * @return a newly constructed configuration - */ - public Configuration build() { - return new Configuration(mClusterConf, mPathConf, mClusterConfHash, mPathConfHash); - } - } - - private Configuration(List clusterConf, Map> pathConf, - String clusterConfHash, String pathConfHash) { - mClusterConf = clusterConf; - mPathConf = pathConf; - mClusterConfHash = clusterConfHash; - mPathConfHash = pathConfHash; - } - - private Configuration(GetConfigurationPResponse conf) { - mClusterConf = conf.getClusterConfigsList().stream().map(Property::fromProto) - .collect(Collectors.toList()); - - Map pathConf = conf.getPathConfigsMap(); - mPathConf = new HashMap<>(pathConf.size()); - pathConf.forEach((path, prop) -> { - List properties = new ArrayList<>(prop.getPropertiesCount()); - prop.getPropertiesList().forEach(p -> properties.add(Property.fromProto(p))); - mPathConf.put(path, properties); - }); - - mClusterConfHash = conf.getClusterConfigHash(); - mPathConfHash = conf.getPathConfigHash(); - } - - /** - * @param conf the grpc representation of configuration - * @return the wire representation of the proto response - */ - public static Configuration fromProto(GetConfigurationPResponse conf) { - return new Configuration(conf); - } - - /** - * @return the proto representation - */ - public GetConfigurationPResponse toProto() { - GetConfigurationPResponse.Builder response = GetConfigurationPResponse.newBuilder(); - if (mClusterConf != null) { - mClusterConf.forEach(property -> response.addClusterConfigs(property.toProto())); - } - if (mPathConf != null) { - mPathConf.forEach((path, properties) -> { - List propertyList = properties.stream().map(Property::toProto) - .collect(Collectors.toList()); - ConfigProperties configProperties = ConfigProperties.newBuilder() - .addAllProperties(propertyList).build(); - response.putPathConfigs(path, configProperties); - }); - } - if (mClusterConfHash != null) { - response.setClusterConfigHash(mClusterConfHash); - } - if (mPathConfHash != null) { - response.setPathConfigHash(mPathConfHash); - } - return response.build(); - } - - /** - * @return the internal cluster level configuration - */ - public List getClusterConf() { - return mClusterConf; - } - - /** - * @return the internal path level configuration - */ - public Map> getPathConf() { - return mPathConf; - } - - /** - * @return cluster level configuration hash - */ - public String getClusterConfHash() { - return mClusterConfHash; - } - - /** - * @return path level configuration hash - */ - public String getPathConfHash() { - return mPathConfHash; - } -} diff --git a/core/common/src/main/java/alluxio/wire/MasterInfo.java b/core/common/src/main/java/alluxio/wire/MasterInfo.java deleted file mode 100644 index c3df43fe5d09..000000000000 --- a/core/common/src/main/java/alluxio/wire/MasterInfo.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import alluxio.util.CommonUtils; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; - -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Master information. - */ -@NotThreadSafe -public final class MasterInfo { - /** Master's address. */ - private Address mAddress; - /** The id of the master. */ - private long mId; - /** Master's last updated time in ms. */ - private long mLastUpdatedTimeMs; - - /** - * Creates a new instance of {@link MasterInfo}. - */ - public MasterInfo() {} - - /** - * Creates a new instance of {@link MasterInfo}. - * - * @param id the master id to use - * @param address the master address to use - * @param lastUpdatedTimeMs the master lastUpdatedTimeMs to use - */ - public MasterInfo(long id, Address address, long lastUpdatedTimeMs) { - mAddress = Preconditions.checkNotNull(address, "address"); - mId = id; - mLastUpdatedTimeMs = lastUpdatedTimeMs; - } - - /** - * Creates a new instance of {@link MasterInfo}. - * - * @param id the master id to use - * @param address the master address to use - */ - public MasterInfo(long id, Address address) { - mAddress = Preconditions.checkNotNull(address, "address"); - mId = id; - mLastUpdatedTimeMs = System.currentTimeMillis(); - } - - /** - * @return the master's address - */ - public Address getAddress() { - return mAddress; - } - - /** - * @return the id of the master - */ - public long getId() { - return mId; - } - - /** - * @return the last updated time of the master in ms - */ - public long getLastUpdatedTimeMs() { - return mLastUpdatedTimeMs; - } - - /** - * @param address the master address information - * @return the master information - */ - public MasterInfo setAddress(Address address) { - mAddress = address; - return this; - } - - /** - * @param id the master id - * @return the master information - */ - public MasterInfo setId(long id) { - mId = id; - return this; - } - - /** - * @param lastUpdatedTimeMs the last update time in ms - * @return the master information - */ - public MasterInfo setLastUpdatedTimeMs(long lastUpdatedTimeMs) { - mLastUpdatedTimeMs = lastUpdatedTimeMs; - return this; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("id", mId).add("address", mAddress) - .add("lastUpdatedTime", CommonUtils.convertMsToClockTime(mLastUpdatedTimeMs)) - .toString(); - } - - /** - * Updates the last updated time of the master (in milliseconds). - */ - public void updateLastUpdatedTimeMs() { - mLastUpdatedTimeMs = System.currentTimeMillis(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof MasterInfo)) { - return false; - } - MasterInfo that = (MasterInfo) o; - return mId == that.mId && Objects.equal(mAddress, that.mAddress) - && mLastUpdatedTimeMs == that.mLastUpdatedTimeMs; - } - - @Override - public int hashCode() { - return Objects.hashCode(mId, mAddress, mLastUpdatedTimeMs); - } -} diff --git a/core/common/src/main/java/alluxio/wire/MasterWebUIMasters.java b/core/common/src/main/java/alluxio/wire/MasterWebUIMasters.java deleted file mode 100644 index fac2ada17a25..000000000000 --- a/core/common/src/main/java/alluxio/wire/MasterWebUIMasters.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import com.google.common.base.MoreObjects; - -import java.io.Serializable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Alluxio WebUI masters information. - */ -@NotThreadSafe -public final class MasterWebUIMasters implements Serializable { - private static final long serialVersionUID = -2709466215687255197L; - - private boolean mDebug; - private MasterInfo[] mFailedMasterInfos; - private MasterInfo[] mNormalMasterInfos; - private MasterInfo mLeaderMasterInfo; - - /** - * Creates a new instance of {@link MasterWebUIMasters}. - */ - public MasterWebUIMasters() { - } - - /** - * Gets debug. - * - * @return the debug - */ - public boolean getDebug() { - return mDebug; - } - - /** - * Get failed master infos master info [ ]. - * - * @return the master info [ ] - */ - public MasterInfo[] getFailedMasterInfos() { - return mFailedMasterInfos; - } - - /** - * Get leader master info master info. - * - * @return the master info - */ - public MasterInfo[] getNormalMasterInfos() { - return mNormalMasterInfos; - } - - /** - * Get normal master infos master info [ ]. - * - * @return the master info [ ] - */ - public MasterInfo getLeaderMasterInfo() { - return mLeaderMasterInfo; - } - - /** - * Sets debug. - * - * @param debug the debug - * @return the debug master infos - */ - public MasterWebUIMasters setDebug(boolean debug) { - mDebug = debug; - return this; - } - - /** - * Sets failed master infos. - * - * @param failedMasterInfos the failed master infos - * @return the failed master infos - */ - public MasterWebUIMasters setFailedMasterInfos(MasterInfo[] failedMasterInfos) { - mFailedMasterInfos = failedMasterInfos.clone(); - return this; - } - - /** - * Sets normal master infos. - * - * @param normalMasterInfos the normal master infos - * @return the normal master infos - */ - public MasterWebUIMasters setNormalMasterInfos(MasterInfo[] normalMasterInfos) { - mNormalMasterInfos = normalMasterInfos.clone(); - return this; - } - - /** - * Sets leader master info. - * - * @param leaderMasterInfo the normal master info - * @return the leader master info - */ - public MasterWebUIMasters setLeaderMasterInfo(MasterInfo leaderMasterInfo) { - mLeaderMasterInfo = leaderMasterInfo; - return this; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("debug", mDebug) - .add("failedMasterInfos", mFailedMasterInfos) - .add("normalMasterInfos", mNormalMasterInfos) - .add("leaderMasterInfo", mLeaderMasterInfo).toString(); - } -} diff --git a/core/common/src/main/java/alluxio/wire/SyncPointInfo.java b/core/common/src/main/java/alluxio/wire/SyncPointInfo.java deleted file mode 100644 index c0d2377814a8..000000000000 --- a/core/common/src/main/java/alluxio/wire/SyncPointInfo.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import alluxio.AlluxioURI; -import alluxio.annotation.PublicApi; -import alluxio.grpc.SyncPointStatus; - -/** - * This class represents the state of a sync point, whether the intial syncing is done, - * in progress or not going to be done. - */ -@PublicApi -public class SyncPointInfo { - /** - * Indicates the status of the initial sync of the active sync point. - */ - public enum SyncStatus { - // This state is possible when initial syncing is turned off - NOT_INITIALLY_SYNCED, - SYNCING, - INITIALLY_SYNCED - } - - private final AlluxioURI mSyncPointUri; - private final SyncStatus mSyncStatus; - - /** - * Constructs a SyncPointInfo object. - * - * @param syncPoint path to the sync point - * @param syncStatus current syncing status - */ - public SyncPointInfo(AlluxioURI syncPoint, SyncStatus syncStatus) { - mSyncPointUri = syncPoint; - mSyncStatus = syncStatus; - } - - /** - * Get the uri of the sync point. - * @return uri of the sync point - */ - public AlluxioURI getSyncPointUri() { - return mSyncPointUri; - } - - /** - * Get the initial sync status. - * @return the initial sync status - */ - public SyncStatus getSyncStatus() { - return mSyncStatus; - } - - /** - * @return proto representation of the sync point information - */ - public alluxio.grpc.SyncPointInfo toProto() { - SyncPointStatus status; - switch (mSyncStatus) { - case NOT_INITIALLY_SYNCED: - status = SyncPointStatus.Not_Initially_Synced; - break; - case SYNCING: - status = SyncPointStatus.Syncing; - break; - case INITIALLY_SYNCED: - status = SyncPointStatus.Initially_Synced; - break; - default: - status = SyncPointStatus.Not_Initially_Synced; - } - - alluxio.grpc.SyncPointInfo info = alluxio.grpc.SyncPointInfo.newBuilder() - .setSyncPointUri(mSyncPointUri.getPath()).setSyncStatus(status).build(); - return info; - } - - /** - * Generate sync point information from the proto representation. - * @param syncPointInfo the proto representation - * @return sync point info object - */ - public static SyncPointInfo fromProto(alluxio.grpc.SyncPointInfo syncPointInfo) { - SyncStatus syncStatus; - switch (syncPointInfo.getSyncStatus()) { - case Not_Initially_Synced: - syncStatus = SyncStatus.NOT_INITIALLY_SYNCED; - break; - case Syncing: - syncStatus = SyncStatus.SYNCING; - break; - case Initially_Synced: - syncStatus = SyncStatus.INITIALLY_SYNCED; - break; - default: - syncStatus = SyncStatus.NOT_INITIALLY_SYNCED; - } - return new SyncPointInfo(new AlluxioURI(syncPointInfo.getSyncPointUri()), syncStatus); - } -} diff --git a/core/common/src/main/java/alluxio/wire/TieredIdentity.java b/core/common/src/main/java/alluxio/wire/TieredIdentity.java deleted file mode 100644 index 20bedd416a5d..000000000000 --- a/core/common/src/main/java/alluxio/wire/TieredIdentity.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import alluxio.annotation.PublicApi; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Joiner; -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import io.swagger.annotations.ApiModelProperty; - -import java.io.Serializable; -import java.util.List; -import java.util.stream.Collectors; -import javax.annotation.Nullable; - -/** - * Class representing a node's tier identity. A tier identity is a list of locality tiers - * identifying network topology, e.g. (host: hostname, rack: rack1). - */ -@PublicApi -public final class TieredIdentity implements Serializable { - private static final long serialVersionUID = -1920596090085594788L; - - private final List mTiers; - - /** - * @param tiers the tiers of the tier identity - */ - @JsonCreator - public TieredIdentity(@JsonProperty("tiers") List tiers) { - mTiers = ImmutableList.copyOf(Preconditions.checkNotNull(tiers, "tiers")); - } - - /** - * @return the tiers of the tier identity - */ - @ApiModelProperty(value = "Tiers included in the tier identity") - public List getTiers() { - return mTiers; - } - - /** - * @param i a tier index - * @return the ith locality tier - */ - public LocalityTier getTier(int i) { - return mTiers.get(i); - } - - /** - * @param other a tiered identity to compare to - * @return whether the top tier of this tiered identity matches the top tier of other - */ - public boolean topTiersMatch(TieredIdentity other) { - return mTiers.get(0).equals(other.getTier(0)); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof TieredIdentity)) { - return false; - } - TieredIdentity that = (TieredIdentity) o; - return mTiers.equals(that.mTiers); - } - - @Override - public int hashCode() { - return Objects.hashCode(mTiers); - } - - @Override - public String toString() { - String tiers = Joiner.on(", ").join(mTiers.stream() - .map(tier -> tier.getTierName() + "=" + tier.getValue()) - .collect(Collectors.toList())); - return String.format("TieredIdentity(%s)", tiers); - } - - /** - * Class representing a locality tier, e.g. (host: hostname). - */ - public static final class LocalityTier implements Serializable { - private static final long serialVersionUID = 7078638137905293841L; - - private final String mTierName; - private final String mValue; - - /** - * @param tierName the name of the tier - * @param value the value of the tier - */ - @JsonCreator - public LocalityTier(@JsonProperty("tierName") String tierName, - @JsonProperty("value") @Nullable String value) { - mTierName = Preconditions.checkNotNull(tierName, "tierName"); - mValue = value; - } - - /** - * @return the name of the tier - */ - @ApiModelProperty(value = "Name of the tier", example = "host") - public String getTierName() { - return mTierName; - } - - /** - * @return the value - */ - @Nullable - @ApiModelProperty(value = "Value of the tier name", example = "localhost") - public String getValue() { - return mValue; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof LocalityTier)) { - return false; - } - LocalityTier that = (LocalityTier) o; - return mTierName.equals(that.mTierName) && Objects.equal(mValue, that.mValue); - } - - @Override - public int hashCode() { - return Objects.hashCode(mTierName, mValue); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("tierName", mTierName) - .add("value", mValue) - .toString(); - } - } -} diff --git a/core/common/src/main/java/alluxio/wire/WorkerNetAddress.java b/core/common/src/main/java/alluxio/wire/WorkerNetAddress.java deleted file mode 100644 index ed71d37e7652..000000000000 --- a/core/common/src/main/java/alluxio/wire/WorkerNetAddress.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import alluxio.Constants; -import alluxio.annotation.PublicApi; -import alluxio.wire.TieredIdentity.LocalityTier; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import io.swagger.annotations.ApiModelProperty; - -import java.io.Serializable; -import java.util.Arrays; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * The network address of a worker. - */ -@PublicApi -@NotThreadSafe -public final class WorkerNetAddress implements Serializable { - private static final long serialVersionUID = 0L; - - private String mHost = ""; - private String mContainerHost = ""; - private int mRpcPort; - private int mDataPort; - private int mWebPort; - private String mDomainSocketPath = ""; - private TieredIdentity mTieredIdentity; - - /** - * Creates a new instance of {@link WorkerNetAddress}. - */ - public WorkerNetAddress() {} - - /** - * @return the host of the worker - */ - @ApiModelProperty(value = "Host name of the worker") - public String getHost() { - return mHost; - } - - /** - * @return the container host of the worker, default to empty string if the worker - * is not in a container - */ - @ApiModelProperty(value = "Host name of the physical node if running in a container") - public String getContainerHost() { - return mContainerHost; - } - - /** - * @return the RPC port - */ - @ApiModelProperty(value = "Port of the worker's Rpc server for metadata operations") - public int getRpcPort() { - return mRpcPort; - } - - /** - * @return the data port - */ - @ApiModelProperty(value = "Port of the worker's server for data operations") - public int getDataPort() { - return mDataPort; - } - - /** - * @return the web port - */ - @ApiModelProperty(value = "Port which exposes the worker's web UI") - public int getWebPort() { - return mWebPort; - } - - /** - * @return the domain socket path - */ - @ApiModelProperty(value = "The domain socket path used by the worker, disabled if empty") - public String getDomainSocketPath() { - return mDomainSocketPath; - } - - /** - * @return the tiered identity - */ - @ApiModelProperty(value = "The worker's tier identity") - public TieredIdentity getTieredIdentity() { - if (mTieredIdentity != null) { - return mTieredIdentity; - } - return new TieredIdentity(Arrays.asList(new LocalityTier(Constants.LOCALITY_NODE, mHost))); - } - - /** - * @param host the host to use - * @return the worker net address - */ - public WorkerNetAddress setHost(String host) { - Preconditions.checkNotNull(host, "host"); - mHost = host; - return this; - } - - /** - * @param containerHost the host of node, if running in a container - * @return the worker net address - */ - public WorkerNetAddress setContainerHost(String containerHost) { - Preconditions.checkNotNull(containerHost, "containerHost"); - mContainerHost = containerHost; - return this; - } - - /** - * @param rpcPort the rpc port to use - * @return the worker net address - */ - public WorkerNetAddress setRpcPort(int rpcPort) { - mRpcPort = rpcPort; - return this; - } - - /** - * @param dataPort the data port to use - * @return the worker net address - */ - public WorkerNetAddress setDataPort(int dataPort) { - mDataPort = dataPort; - return this; - } - - /** - * @param webPort the web port to use - * @return the worker net address - */ - public WorkerNetAddress setWebPort(int webPort) { - mWebPort = webPort; - return this; - } - - /** - * @param domainSocketPath the domain socket path - * @return the worker net address - */ - public WorkerNetAddress setDomainSocketPath(String domainSocketPath) { - mDomainSocketPath = domainSocketPath; - return this; - } - - /** - * @param tieredIdentity the tiered identity - * @return the worker net address - */ - public WorkerNetAddress setTieredIdentity(TieredIdentity tieredIdentity) { - mTieredIdentity = tieredIdentity; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof WorkerNetAddress)) { - return false; - } - WorkerNetAddress that = (WorkerNetAddress) o; - return mHost.equals(that.mHost) - && mContainerHost.equals(that.mContainerHost) - && mRpcPort == that.mRpcPort - && mDataPort == that.mDataPort - && mWebPort == that.mWebPort - && mDomainSocketPath.equals(that.mDomainSocketPath) - && Objects.equal(mTieredIdentity, that.mTieredIdentity); - } - - @Override - public int hashCode() { - return Objects.hashCode(mHost, mContainerHost, mDataPort, mRpcPort, mWebPort, - mDomainSocketPath, mTieredIdentity); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("host", mHost) - .add("containerHost", mContainerHost) - .add("rpcPort", mRpcPort) - .add("dataPort", mDataPort) - .add("webPort", mWebPort) - .add("domainSocketPath", mDomainSocketPath) - .add("tieredIdentity", mTieredIdentity) - .toString(); - } -} diff --git a/core/common/src/main/java/alluxio/wire/WorkerWebUIConfiguration.java b/core/common/src/main/java/alluxio/wire/WorkerWebUIConfiguration.java deleted file mode 100644 index d1d2435c229b..000000000000 --- a/core/common/src/main/java/alluxio/wire/WorkerWebUIConfiguration.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import com.google.common.base.MoreObjects; -import org.apache.commons.lang3.tuple.Triple; - -import java.io.Serializable; -import java.util.List; -import java.util.TreeSet; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Alluxio WebUI configuration information. - */ -@NotThreadSafe -public final class WorkerWebUIConfiguration implements Serializable { - private static final long serialVersionUID = -2277858633604882055L; - - private List mWhitelist; - private TreeSet> mConfiguration; - - /** - * Creates a new instance of {@link WorkerWebUIConfiguration}. - */ - public WorkerWebUIConfiguration() { - } - - /** - * Gets configuration. - * - * @return the configuration - */ - public TreeSet> getConfiguration() { - return mConfiguration; - } - - /** - * Gets whitelist. - * - * @return the whitelist - */ - public List getWhitelist() { - return mWhitelist; - } - - /** - * Sets configuration. - * - * @param configuration the configuration - * @return the configuration - */ - public WorkerWebUIConfiguration setConfiguration( - TreeSet> configuration) { - mConfiguration = configuration; - return this; - } - - /** - * Sets whitelist. - * - * @param whitelist the whitelist - * @return the whitelist - */ - public WorkerWebUIConfiguration setWhitelist(List whitelist) { - mWhitelist = whitelist; - return this; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("configuration", mConfiguration) - .add("whitelist", mWhitelist).toString(); - } -} diff --git a/core/common/src/main/java/alluxio/worker/block/BlockStore.java b/core/common/src/main/java/alluxio/worker/block/BlockStore.java deleted file mode 100644 index c5329e717610..000000000000 --- a/core/common/src/main/java/alluxio/worker/block/BlockStore.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.grpc.Block; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.UfsReadOptions; -import alluxio.proto.dataserver.Protocol; -import alluxio.worker.SessionCleanable; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.TempBlockMeta; - -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; - -/** - * An abstraction of block store on worker. - */ -public interface BlockStore extends Closeable, SessionCleanable { - - /** - * Aborts a temporary block. The metadata of this block will not be added, its data will be - * deleted and the space will be reclaimed. Since a temp block is "private" to the writer, this - * requires no previously acquired lock. - * - * @param sessionId the id of the session - * @param blockId the id of a temp block - */ - void abortBlock(long sessionId, long blockId); - - /** - * Notifies the block store that a block was accessed so the block store could update accordingly - * the registered listeners such as evictor and allocator on block access. - * //TODO(beinan): looks like we should not expose this method except the test - * - * @param sessionId the id of the session to access a block - * @param blockId the id of an accessed block - */ - void accessBlock(long sessionId, long blockId); - - /** - * Commits a block to Alluxio managed space. The block must be temporary. The block will not be - * persisted or accessible before commitBlock succeeds. - * - * @param sessionId the id of the client - * @param blockId the id of the block to commit - * @param pinOnCreate whether to pin block on create - */ - void commitBlock(long sessionId, long blockId, boolean pinOnCreate); - - /** - * Creates a block in Alluxio managed space. - * Calls {@link #createBlockWriter} to get a writer for writing to the block. - * The block will be temporary until it is committed by {@link #commitBlock} . - * Throws an {@link IllegalArgumentException} if the location does not belong to tiered storage. - * - * @param sessionId the id of the client - * @param blockId the id of the block to create - * @param tier the tier to place the new block in - * {@link BlockStoreLocation#ANY_TIER} for any tier - * @param createBlockOptions the createBlockOptions - * @return a string representing the path to the local file - */ - String createBlock(long sessionId, long blockId, int tier, - CreateBlockOptions createBlockOptions); - - /** - * Creates the block reader to read from Alluxio block or UFS block. - * Owner of this block reader must close it or lock will leak. - * - * @param sessionId the client session ID - * @param blockId the ID of the UFS block to read - * @param offset the offset within the block - * @param positionShort whether the operation is using positioned read to a small buffer size - * @param options the options - * @return a block reader to read data from - * @throws IOException if it fails to get block reader - */ - BlockReader createBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException; - - /** - * Creates a block reader to read a UFS block starting from given block offset. - * Owner of this block reader must close it to cleanup state. - * - * @param sessionId the client session ID - * @param blockId the ID of the UFS block to read - * @param offset the offset within the block - * @param positionShort whether the operation is using positioned read to a small buffer size - * @param options the options - * @return the block reader instance - * @throws IOException if it fails to get block reader - */ - BlockReader createUfsBlockReader(long sessionId, long blockId, long offset, boolean positionShort, - Protocol.OpenUfsBlockOptions options) - throws IOException; - - /** - * Creates a {@link BlockWriter} for an existing temporary block which is already created by - * {@link #createBlock}. - * - * @param sessionId the id of the client - * @param blockId the id of the block to be opened for writing - * @return the block writer for the local block file - */ - BlockWriter createBlockWriter(long sessionId, long blockId) - throws IOException; - - /** - * Gets the metadata of the entire store in a snapshot. There is no guarantee the state will be - * consistent with the snapshot after this method is called. - * This function should be cheap since it is called for every block. - * - * @return store metadata - */ - BlockStoreMeta getBlockStoreMeta(); - - /** - * Similar as {@link #getBlockStoreMeta} except that this includes - * more information about the block store (e.g. blockId list). This is an expensive operation. - * - * @return full store metadata - */ - BlockStoreMeta getBlockStoreMetaFull(); - - /** - * Gets the temp metadata of a specific block from local storage. - * - * @param blockId the id of the block - * @return metadata of the block if the temp block exists - */ - Optional getTempBlockMeta(long blockId); - - /** - * Checks if the storage has a given block. - * - * @param blockId the block id - * @return true if the block is contained, false otherwise - */ - boolean hasBlockMeta(long blockId); - - /** - * Checks if the storage has a given temp block. - * - * @param blockId the temp block id - * @return true if the block is contained, false otherwise - */ - boolean hasTempBlockMeta(long blockId); - - /** - * Gets the metadata of a block given its block id or empty if block does not exist. - * This method does not require a lock id so the block is possible to be moved or removed after it - * returns. - * - * @param blockId the block id - * @return metadata of the block - */ - Optional getVolatileBlockMeta(long blockId); - - /** - * Moves an existing block to a new location. - * - * @param sessionId the id of the session to move a block - * @param blockId the id of an existing block - * @param moveOptions the options for move - */ - void moveBlock(long sessionId, long blockId, AllocateOptions moveOptions) - throws IOException; - - /** - * Pins the block indicating subsequent access. - * - * @param sessionId the id of the session to lock this block - * @param blockId the id of the block to lock - * @return a lock of block to conveniently unpin the block later, or empty - * if the block does not exist - */ - Optional pinBlock(long sessionId, long blockId); - - /** - * Unpins an accessed block based on the id (returned by {@link #pinBlock(long, long)}). - * - * @param lock the lock returned by {@link #pinBlock(long, long)} - */ - void unpinBlock(BlockLock lock); - - /** - * Update the pinned inodes. - * - * @param inodes a set of inodes that are currently pinned - */ - void updatePinnedInodes(Set inodes); - - /** - * Registers a {@link BlockStoreEventListener} to this block store. - * - * @param listener the listener to those events - */ - void registerBlockStoreEventListener(BlockStoreEventListener listener); - - /** - * Removes an existing block. If the block can not be found in this store. - * - * @param sessionId the id of the session to remove a block - * @param blockId the id of an existing block - */ - void removeBlock(long sessionId, long blockId) throws IOException; - - /** - * Remove Storage directories that are no longer accessible. - */ - void removeInaccessibleStorage(); - - /** - * Requests to increase the size of a temp block. Since a temp block is "private" to the writer - * client, this operation requires no previously acquired lock. - * - * @param sessionId the id of the session to request space - * @param blockId the id of the temp block - * @param additionalBytes the amount of more space to request in bytes, never be less than 0 - */ - void requestSpace(long sessionId, long blockId, long additionalBytes); - - /** - * Load blocks into alluxio. - * - * @param fileBlocks list of fileBlocks, one file blocks contains blocks belong to one file - * @param options read ufs options - * @return future of load status for failed blocks - */ - CompletableFuture> load(List fileBlocks, UfsReadOptions options); -} diff --git a/core/common/src/main/java/alluxio/worker/block/BlockWorker.java b/core/common/src/main/java/alluxio/worker/block/BlockWorker.java deleted file mode 100644 index 7346691c68dc..000000000000 --- a/core/common/src/main/java/alluxio/worker/block/BlockWorker.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.exception.AlluxioException; -import alluxio.grpc.AsyncCacheRequest; -import alluxio.grpc.Block; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.GetConfigurationPOptions; -import alluxio.grpc.UfsReadOptions; -import alluxio.proto.dataserver.Protocol; -import alluxio.wire.Configuration; -import alluxio.wire.FileInfo; -import alluxio.worker.SessionCleanable; -import alluxio.worker.Worker; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; - -import java.io.IOException; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicReference; - -/** - * A block worker in the Alluxio system. - */ -public interface BlockWorker extends Worker, SessionCleanable { - /** - * @return the worker id - */ - AtomicReference getWorkerId(); - - /** - * Aborts the temporary block created by the session. - * - * @param sessionId the id of the client - * @param blockId the id of the block to be aborted - */ - void abortBlock(long sessionId, long blockId) throws IOException; - - /** - * Commits a block to Alluxio managed space. The block must be temporary. The block will not be - * persisted or accessible before commitBlock succeeds. - * - * @param sessionId the id of the client - * @param blockId the id of the block to commit - * @param pinOnCreate whether to pin block on create - */ - void commitBlock(long sessionId, long blockId, boolean pinOnCreate); - - /** - * Commits a block in UFS. - * - * @param blockId the id of the block to commit - * @param length length of the block to commit - */ - void commitBlockInUfs(long blockId, long length); - - /** - * Creates a block in Alluxio managed space. - * Calls {@link #createBlockWriter} to get a writer for writing to the block. - * The block will be temporary until it is committed by {@link #commitBlock} . - * Throws an {@link IllegalArgumentException} if the location does not belong to tiered storage. - * - * @param sessionId the id of the client - * @param blockId the id of the block to create - * @param tier the tier to place the new block in - * {@link BlockStoreLocation#ANY_TIER} for any tier - * @param createBlockOptions the createBlockOptions - * @return a string representing the path to the local file - */ - String createBlock(long sessionId, long blockId, int tier, - CreateBlockOptions createBlockOptions); - - /** - * Creates a {@link BlockWriter} for an existing temporary block which is already created by - * {@link #createBlock}. - * - * @param sessionId the id of the client - * @param blockId the id of the block to be opened for writing - * @return the block writer for the local block file - */ - BlockWriter createBlockWriter(long sessionId, long blockId) - throws IOException; - - /** - * Gets a report for the periodic heartbeat to master. Contains the blocks added since the last - * heart beat and blocks removed since the last heartbeat. - * - * @return a block heartbeat report - */ - BlockHeartbeatReport getReport(); - - /** - * Gets the metadata for the entire block store. Contains the block mapping per storage dir and - * the total capacity and used capacity of each tier. This function is cheap. - * - * @return the block store metadata - */ - BlockStoreMeta getStoreMeta(); - - /** - * Similar as {@link BlockWorker#getStoreMeta} except that this also contains full blockId - * list. This function is expensive. - * - * @return the full block store metadata - */ - BlockStoreMeta getStoreMetaFull(); - - /** - * Creates the block reader to read from Alluxio block or UFS block. - * Owner of this block reader must close it or lock will leak. - * - * @param sessionId the client session ID - * @param blockId the ID of the UFS block to read - * @param offset the offset within the block - * @param positionShort whether the operation is using positioned read to a small buffer size - * @param options the options - * @return a block reader to read data from - * @throws IOException if it fails to get block reader - */ - BlockReader createBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException; - - /** - * Creates a block reader to read a UFS block starting from given block offset. - * Owner of this block reader must close it to cleanup state. - * - * @param sessionId the client session ID - * @param blockId the ID of the UFS block to read - * @param offset the offset within the block - * @param positionShort whether the operation is using positioned read to a small buffer size - * @param options the options - * @return the block reader instance - * @throws IOException if it fails to get block reader - */ - BlockReader createUfsBlockReader(long sessionId, long blockId, long offset, boolean positionShort, - Protocol.OpenUfsBlockOptions options) throws IOException; - - /** - * Frees a block from Alluxio managed space. - * - * @param sessionId the id of the client - * @param blockId the id of the block to be freed - */ - void removeBlock(long sessionId, long blockId) throws IOException; - - /** - * Frees all blocks in the current worker by deleting all block store directories. - * Whether this method returns successfully or exceptionally, - * the worker should not be used in any way. - * - * @throws IOException if free fails - */ - void freeWorker() throws IOException; - - /** - * Request an amount of space for a block in its storage directory. The block must be a temporary - * block. - * - * @param sessionId the id of the client - * @param blockId the id of the block to allocate space to - * @param additionalBytes the amount of bytes to allocate - */ - void requestSpace(long sessionId, long blockId, long additionalBytes); - - /** - * Submits the async cache request to async cache manager to execute. - * - * @param request the async cache request - * - * @deprecated This method will be deprecated as of v3.0, use {@link #cache} - */ - @Deprecated - void asyncCache(AsyncCacheRequest request); - - /** - * Submits the cache request to cache manager to execute. - * - * @param request the cache request - */ - void cache(CacheRequest request) throws AlluxioException, IOException; - - /** - * Load blocks into alluxio. - * - * @param fileBlocks list of fileBlocks, one file blocks contains blocks belong to one file - * @param options read ufs options - * @return future of load status for failed blocks - */ - CompletableFuture> load(List fileBlocks, UfsReadOptions options); - - /** - * Sets the pinlist for the underlying block store. - * - * @param pinnedInodes a set of pinned inodes - */ - void updatePinList(Set pinnedInodes); - - /** - * Gets the file information. - * - * @param fileId the file id - * @return the file info - */ - FileInfo getFileInfo(long fileId) throws IOException; - - /** - * Clears the worker metrics. - */ - void clearMetrics(); - - /** - * @param options method options - * @return configuration information list - */ - Configuration getConfiguration(GetConfigurationPOptions options); - - /** - * @return the white list - */ - List getWhiteList(); - - /** - * @return the block store - */ - BlockStore getBlockStore(); -} diff --git a/core/common/src/main/java/alluxio/worker/block/io/DelegatingBlockReader.java b/core/common/src/main/java/alluxio/worker/block/io/DelegatingBlockReader.java deleted file mode 100644 index 19e20bcb3d5f..000000000000 --- a/core/common/src/main/java/alluxio/worker/block/io/DelegatingBlockReader.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.io; - -import com.google.common.io.Closer; -import io.netty.buffer.ByteBuf; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; - -/** - * An delegating reader class. - */ -public class DelegatingBlockReader extends BlockReader { - private final BlockReader mBlockReader; - private final Closer mCloser; - - /** - * Default constructor for the abstract reader implementations. - * @param blockReader block reader - * @param closeable closer - */ - public DelegatingBlockReader(BlockReader blockReader, Closeable closeable) { - mCloser = Closer.create(); - mBlockReader = mCloser.register(blockReader); - mCloser.register(closeable); - } - - /** - * @return the delegate - */ - public BlockReader getDelegate() { - return mBlockReader; - } - - @Override - public ByteBuffer read(long offset, long length) throws IOException { - return mBlockReader.read(offset, length); - } - - @Override - public long getLength() { - return mBlockReader.getLength(); - } - - @Override - public ReadableByteChannel getChannel() { - return mBlockReader.getChannel(); - } - - @Override - public int transferTo(ByteBuf buf) throws IOException { - return mBlockReader.transferTo(buf); - } - - @Override - public boolean isClosed() { - return mBlockReader.isClosed(); - } - - @Override - public String getLocation() { - return mBlockReader.getLocation(); - } - - @Override - public String toString() { - return mBlockReader.toString(); - } - - @Override - public void close() throws IOException { - mCloser.close(); - } -} diff --git a/core/common/src/main/java/alluxio/worker/block/io/LocalFileBlockReader.java b/core/common/src/main/java/alluxio/worker/block/io/LocalFileBlockReader.java deleted file mode 100644 index 39e63919eca8..000000000000 --- a/core/common/src/main/java/alluxio/worker/block/io/LocalFileBlockReader.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.io; - -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; - -import com.codahale.metrics.Counter; -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import io.netty.buffer.ByteBuf; - -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.channels.ReadableByteChannel; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class provides read access to a block data file locally stored in managed storage. - */ -@NotThreadSafe -public class LocalFileBlockReader extends BlockReader { - private static final Counter BLOCKS_READ_LOCAL = - MetricsSystem.counter(MetricKey.WORKER_BLOCKS_READ_LOCAL.getName()); - - private final String mFilePath; - private final RandomAccessFile mLocalFile; - private final FileChannel mLocalFileChannel; - private final Closer mCloser = Closer.create(); - private final long mFileSize; - private boolean mClosed; - private int mUsageCount = 0; - - /** - * Constructs a Block reader given the file path of the block. - * - * @param path file path of the block - */ - public LocalFileBlockReader(String path) throws IOException { - mFilePath = Preconditions.checkNotNull(path, "path"); - mLocalFile = mCloser.register(new RandomAccessFile(mFilePath, "r")); - mFileSize = mLocalFile.length(); - mLocalFileChannel = mCloser.register(mLocalFile.getChannel()); - } - - @Override - public ReadableByteChannel getChannel() { - return mLocalFileChannel; - } - - @Override - public long getLength() { - return mFileSize; - } - - /** - * increase the file reader usage count. - */ - public void increaseUsageCount() { - mUsageCount++; - } - - /** - * decrease the file reader usage count. - */ - public void decreaseUsageCount() { - Preconditions.checkState(mUsageCount > 0); - mUsageCount--; - } - - /** - * @return the file reader usage count - */ - public int getUsageCount() { - return mUsageCount; - } - - /** - * @return the file path - */ - public String getFilePath() { - return mFilePath; - } - - @Override - public ByteBuffer read(long offset, long length) throws IOException { - Preconditions.checkArgument(offset + length <= mFileSize, - "offset=%s, length=%s, exceeding fileSize=%s", offset, length, mFileSize); - return mLocalFileChannel.map(FileChannel.MapMode.READ_ONLY, offset, length); - } - - @Override - public int transferTo(ByteBuf buf) throws IOException { - return buf.writeBytes(mLocalFileChannel, buf.writableBytes()); - } - - @Override - public void close() throws IOException { - super.close(); - Preconditions.checkState(mUsageCount == 0); - if (mClosed) { - return; - } - try { - mCloser.close(); - } finally { - mClosed = true; - BLOCKS_READ_LOCAL.inc(); - } - } - - @Override - public boolean isClosed() { - return mClosed; - } - - @Override - public String getLocation() { - return mFilePath; - } -} diff --git a/core/common/src/main/java/alluxio/worker/block/io/LocalFileBlockWriter.java b/core/common/src/main/java/alluxio/worker/block/io/LocalFileBlockWriter.java deleted file mode 100644 index ac877539516c..000000000000 --- a/core/common/src/main/java/alluxio/worker/block/io/LocalFileBlockWriter.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.io; - -import alluxio.exception.runtime.InternalRuntimeException; -import alluxio.exception.runtime.NotFoundRuntimeException; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.util.io.BufferUtils; - -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import io.netty.buffer.ByteBuf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.channels.WritableByteChannel; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class provides write access to a temp block data file locally stored in managed storage. - */ -@NotThreadSafe -public class LocalFileBlockWriter extends BlockWriter { - private static final Logger LOG = LoggerFactory.getLogger(LocalFileBlockWriter.class); - private final FileChannel mLocalFileChannel; - private final Closer mCloser = Closer.create(); - private long mPosition; - private boolean mClosed; - - /** - * Constructs a Block writer given the file path of the block. - * - * @param path file path of the block - */ - public LocalFileBlockWriter(String path) { - String filePath = Preconditions.checkNotNull(path, "path"); - RandomAccessFile localFile; - try { - localFile = mCloser.register(new RandomAccessFile(filePath, "rw")); - } catch (FileNotFoundException e) { - throw new NotFoundRuntimeException("wrong path when creating RandomAccessFile", e); - } - mLocalFileChannel = mCloser.register(localFile.getChannel()); - } - - @Override - public long append(ByteBuffer inputBuf) { - long bytesWritten; - - int inputBufLength = inputBuf.limit() - inputBuf.position(); - MappedByteBuffer outputBuf; - try { - outputBuf = mLocalFileChannel.map(FileChannel.MapMode.READ_WRITE, mLocalFileChannel.size(), - inputBufLength); - } catch (IOException e) { - throw new InternalRuntimeException(e); - } - outputBuf.put(inputBuf); - int bytesWritten1 = outputBuf.limit(); - BufferUtils.cleanDirectBuffer(outputBuf); - bytesWritten = bytesWritten1; - mPosition += bytesWritten; - return bytesWritten; - } - - @Override - public long append(ByteBuf buf) throws IOException { - long bytesWritten = buf.readBytes(mLocalFileChannel, buf.readableBytes()); - mPosition += bytesWritten; - return bytesWritten; - } - - @Override - public long append(DataBuffer buffer) throws IOException { - ByteBuf bytebuf = null; - try { - bytebuf = (ByteBuf) buffer.getNettyOutput(); - } catch (Throwable e) { - LOG.debug("Failed to get ByteBuf from DataBuffer, write performance may be degraded."); - } - if (bytebuf != null) { - return append(bytebuf); - } - long bytesWritten = write(mLocalFileChannel.size(), buffer); - mPosition += bytesWritten; - return bytesWritten; - } - - @Override - public long getPosition() { - return mPosition; - } - - @Override - public WritableByteChannel getChannel() { - return mLocalFileChannel; - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - mClosed = true; - - super.close(); - mCloser.close(); - mPosition = -1; - } - - private long write(long offset, DataBuffer inputBuf) throws IOException { - int inputBufLength = inputBuf.readableBytes(); - MappedByteBuffer outputBuf = - mLocalFileChannel.map(FileChannel.MapMode.READ_WRITE, offset, inputBufLength); - inputBuf.readBytes(outputBuf); - int bytesWritten = outputBuf.limit(); - BufferUtils.cleanDirectBuffer(outputBuf); - return bytesWritten; - } -} diff --git a/core/common/src/test/java/alluxio/check/UpdateCheckTest.java b/core/common/src/test/java/alluxio/check/UpdateCheckTest.java deleted file mode 100644 index 2d47b28d758f..000000000000 --- a/core/common/src/test/java/alluxio/check/UpdateCheckTest.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.check; - -import alluxio.ProjectConstants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.journal.JournalType; -import alluxio.master.metastore.MetastoreType; -import alluxio.util.EnvironmentUtils; -import alluxio.util.OSUtils; -import alluxio.worker.block.BlockStoreType; - -import com.amazonaws.SdkClientException; -import com.amazonaws.util.EC2MetadataUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Unit tests for {@link UpdateCheck}. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({EnvironmentUtils.class, EC2MetadataUtils.class}) -public class UpdateCheckTest { - - @Before - public void before() { - PowerMockito.mockStatic(EnvironmentUtils.class); - Mockito.when(EnvironmentUtils.isDocker()).thenReturn(false); - Mockito.when(EnvironmentUtils.isKubernetes()).thenReturn(false); - Mockito.when(EnvironmentUtils.isGoogleComputeEngine()).thenReturn(false); - Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(""); - Mockito.when(EnvironmentUtils.isEC2()).thenReturn(false); - Mockito.when(EnvironmentUtils.isCFT(Mockito.anyString())).thenReturn(false); - Mockito.when(EnvironmentUtils.isEMR(Mockito.anyString())).thenReturn(false); - PowerMockito.mockStatic(EC2MetadataUtils.class); - } - - @Test - public void userAgentEnvironmentStringEmpty() { - List info = new ArrayList<>(); - Mockito.when(EC2MetadataUtils.getUserData()) - .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(1, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - } - - @Test - public void userAgentEnvironmentStringDocker() { - Mockito.when(EnvironmentUtils.isDocker()).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()) - .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(2, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - Assert.assertEquals(UpdateCheck.DOCKER_KEY, info.get(1)); - } - - @Test - public void userAgentEnvironmentStringK8s() { - Mockito.when(EnvironmentUtils.isDocker()).thenReturn(true); - Mockito.when(EnvironmentUtils.isKubernetes()).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()) - .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(3, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - Assert.assertEquals(UpdateCheck.DOCKER_KEY, info.get(1)); - Assert.assertEquals(UpdateCheck.KUBERNETES_KEY, info.get(2)); - } - - @Test - public void userAgentEnvironmentStringGCP() { - Mockito.when(EnvironmentUtils.isGoogleComputeEngine()).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()) - .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(2, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - Assert.assertEquals(UpdateCheck.GCE_KEY, info.get(1)); - } - - @Test - public void userAgentEnvironmentStringEC2AMI() { - String randomProductCode = "random123code"; - Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); - Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); - // When no user data in this ec2, null is returned - Mockito.when(EC2MetadataUtils.getUserData()).thenReturn(null); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(3, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - Assert.assertEquals(String.format(UpdateCheck.PRODUCT_CODE_FORMAT, randomProductCode), - info.get(1)); - Assert.assertEquals(UpdateCheck.EC2_KEY, info.get(2)); - } - - @Test - public void userAgentEnvironmentStringEC2CFT() { - String randomProductCode = "random123code"; - Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); - Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); - Mockito.when(EnvironmentUtils.isCFT(Mockito.anyString())).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()).thenReturn("{ \"cft_configure\": {}}"); - - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(4, info.size()); - Assert.assertEquals(String.format(UpdateCheck.PRODUCT_CODE_FORMAT, randomProductCode), - info.get(1)); - Assert.assertEquals(UpdateCheck.CFT_KEY, info.get(2)); - Assert.assertEquals(UpdateCheck.EC2_KEY, info.get(3)); - } - - @Test - public void userAgentEnvironmentStringEC2EMR() { - String randomProductCode = "random123code"; - Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); - Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); - Mockito.when(EnvironmentUtils.isEMR(Mockito.anyString())).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()).thenReturn("emr_apps"); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(4, info.size()); - Assert.assertEquals(String.format(UpdateCheck.PRODUCT_CODE_FORMAT, randomProductCode), - info.get(1)); - Assert.assertEquals(UpdateCheck.EMR_KEY, info.get(2)); - Assert.assertEquals(UpdateCheck.EC2_KEY, info.get(3)); - } - - @Test - public void featureStringEmbeddedJournal() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.EMBEDDED_KEY)); - Configuration.set(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.EMBEDDED_KEY)); - } - - @Test - public void featureStringInodeMetastoreRocks() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_INODE_METASTORE, MetastoreType.ROCKS); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.INODE_METASTORE_ROCKS_KEY)); - Configuration.set(PropertyKey.MASTER_INODE_METASTORE, MetastoreType.HEAP); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.INODE_METASTORE_ROCKS_KEY)); - } - - @Test - public void featureStringBlockMetastoreRocks() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_BLOCK_METASTORE, MetastoreType.ROCKS); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.BLOCK_METASTORE_ROCKS_KEY)); - Configuration.set(PropertyKey.MASTER_BLOCK_METASTORE, MetastoreType.HEAP); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.BLOCK_METASTORE_ROCKS_KEY)); - } - - @Test - public void featureStringZookeeper() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.ZOOKEEPER_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.ZOOKEEPER_KEY)); - Configuration.set(PropertyKey.ZOOKEEPER_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.ZOOKEEPER_KEY)); - } - - @Test - public void featureStringBackupDelegation() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_BACKUP_DELEGATION_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.BACKUP_DELEGATION_KEY)); - Configuration.set(PropertyKey.MASTER_BACKUP_DELEGATION_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.BACKUP_DELEGATION_KEY)); - } - - @Test - public void featureStringDailyBackup() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_DAILY_BACKUP_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.DAILY_BACKUP_KEY)); - Configuration.set(PropertyKey.MASTER_DAILY_BACKUP_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.DAILY_BACKUP_KEY)); - } - - @Test - public void featureStringPersistneceBlacklist() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_PERSISTENCE_BLACKLIST, ".tmp"); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.PERSIST_BLACK_LIST_KEY)); - Configuration.unset(PropertyKey.MASTER_PERSISTENCE_BLACKLIST); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.PERSIST_BLACK_LIST_KEY)); - } - - @Test - public void featureStringUnsafePersist() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_UNSAFE_DIRECT_PERSIST_OBJECT_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.UNSAFE_PERSIST_KEY)); - Configuration.set(PropertyKey.MASTER_UNSAFE_DIRECT_PERSIST_OBJECT_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.UNSAFE_PERSIST_KEY)); - } - - @Test - public void featureStringMasterAuditLogging() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.MASTER_AUDIT_LOG_KEY)); - Configuration.set(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.MASTER_AUDIT_LOG_KEY)); - } - - @Test - public void featureStringPageStore() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.WORKER_BLOCK_STORE_TYPE, BlockStoreType.PAGE); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.PAGE_STORE_KEY)); - Configuration.set(PropertyKey.WORKER_BLOCK_STORE_TYPE, BlockStoreType.FILE); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.PAGE_STORE_KEY)); - } - - @Test - public void userAgent() { - String userAgentString = UpdateCheck.getUserAgentString("cluster1", new ArrayList<>()); - Pattern pattern = Pattern.compile( - String.format("Alluxio\\/%s \\(cluster1(?:.+)[^;]\\)", ProjectConstants.VERSION)); - Matcher matcher = pattern.matcher(userAgentString); - Assert.assertTrue(matcher.matches()); - } - - /** - * Makes sure the list containing the target information. - * - * @param list the list to check - * @param target the target info - * @return true if list contains the target - */ - private boolean listContainsTarget(List list, String target) { - for (String str : list) { - if (str.equals(target)) { - return true; - } - } - return false; - } -} diff --git a/core/common/src/test/java/alluxio/conf/path/PrefixPathMatcherTest.java b/core/common/src/test/java/alluxio/conf/path/PrefixPathMatcherTest.java deleted file mode 100644 index 055b868001a1..000000000000 --- a/core/common/src/test/java/alluxio/conf/path/PrefixPathMatcherTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf.path; - -import alluxio.AlluxioURI; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -/** - * Tests {@link PrefixPathMatcher}. - */ -public class PrefixPathMatcherTest { - @Test - public void empty() { - PrefixPathMatcher matcher = new PrefixPathMatcher(Collections.emptySet()); - checkNoMatch(matcher, "/"); - checkNoMatch(matcher, "/a"); - checkNoMatch(matcher, "/a/bc"); - } - - @Test - public void root() { - Set paths = new HashSet<>(); - paths.add("/"); - PrefixPathMatcher matcher = new PrefixPathMatcher(paths); - checkMatch(matcher, "/", "/"); - checkMatch(matcher, "/a", "/"); - checkMatch(matcher, "/a/b", "/"); - } - - @Test - public void match() { - String[] paths = new String[]{ - "/a", - "/a/b/", - "/a/b/c", - "/a/bc", - }; - Set pathSet = new HashSet<>(); - Collections.addAll(pathSet, paths); - PrefixPathMatcher matcher = new PrefixPathMatcher(pathSet); - checkNoMatch(matcher, "/"); - checkNoMatch(matcher, "/ab"); - checkNoMatch(matcher, "/b"); - checkMatch(matcher, "/a", "/a"); - checkMatch(matcher, "/a/", "/a"); - checkMatch(matcher, "/a/bd", "/a"); - checkMatch(matcher, "/a/b", "/a/b/", "/a"); - checkMatch(matcher, "/a/b/", "/a/b/", "/a"); - checkMatch(matcher, "/a/b/d", "/a/b/", "/a"); - checkMatch(matcher, "/a/b/cd", "/a/b/", "/a"); - checkMatch(matcher, "/a/b/c", "/a/b/c", "/a/b/", "/a"); - checkMatch(matcher, "/a/b/c/d", "/a/b/c", "/a/b/", "/a"); - checkMatch(matcher, "/a/bc", "/a/bc", "/a"); - checkMatch(matcher, "/a/bc/d", "/a/bc", "/a"); - } - - @Test - public void matchWithNoLeadingSlash() { - Set paths = new HashSet<>(); - paths.add("/a/b"); - PrefixPathMatcher matcher = new PrefixPathMatcher(paths); - checkNoMatch(matcher, "a"); - checkNoMatch(matcher, "a/b"); - } - - private void checkMatch(PrefixPathMatcher matcher, String path, String... expected) { - Optional> paths = matcher.match(new AlluxioURI(path)); - Assert.assertTrue(paths.isPresent()); - Assert.assertArrayEquals(expected, paths.get().toArray()); - } - - private void checkNoMatch(PrefixPathMatcher matcher, String path) { - Assert.assertFalse(matcher.match(new AlluxioURI(path)).isPresent()); - } -} diff --git a/core/common/src/test/java/alluxio/conf/path/TrieNodeTest.java b/core/common/src/test/java/alluxio/conf/path/TrieNodeTest.java deleted file mode 100644 index fdb8a53eb49e..000000000000 --- a/core/common/src/test/java/alluxio/conf/path/TrieNodeTest.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.conf.path; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Streams; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Tests {@link PrefixPathMatcher}. - */ -public class TrieNodeTest { - @Test - public void hasTerminalIncludeChildren() { - TrieNode node = new TrieNode<>(); - node.insert("/a/b"); - Assert.assertTrue(node.hasTerminal("/", true)); - Assert.assertFalse(node.hasTerminal("/c", true)); - Assert.assertTrue(node.hasTerminal("/a", true)); - Assert.assertTrue(node.hasTerminal("/a/b", true)); - Assert.assertTrue(node.hasTerminal("/a/b/c", true)); - } - - @Test - public void hasTerminal() { - TrieNode node = new TrieNode<>(); - node.insert("/a/b"); - Assert.assertFalse(node.hasTerminal("/", false)); - Assert.assertFalse(node.hasTerminal("/c", false)); - Assert.assertFalse(node.hasTerminal("/a", false)); - Assert.assertTrue(node.hasTerminal("/a/b", false)); - Assert.assertTrue(node.hasTerminal("/a/b/c", false)); - } - - @Test - public void getCommonRoots() { - TrieNode node = new TrieNode<>(); - TrieNode a = node.insert("/a"); - TrieNode b = node.insert("/a/b"); - TrieNode f = node.insert("/a/e/f"); - TrieNode d = node.insert("/c/d"); - TrieNode g = node.insert("/c/g"); - Set> roots = Streams.stream(node.getCommonRoots()).collect(Collectors.toSet()); - Assert.assertTrue(roots.contains(a)); - Assert.assertFalse(roots.contains(b)); - Assert.assertTrue(roots.contains(d)); - Assert.assertFalse(roots.contains(f)); - Assert.assertTrue(roots.contains(g)); - } - - @Test - public void searchExact() { - TrieNode node = new TrieNode<>(); - TrieNode a = node.insert("/a"); - TrieNode b = node.insert("/a/b"); - TrieNode f = node.insert("/a/e/f"); - TrieNode d = node.insert("/c/d"); - TrieNode g = node.insert("/c/g"); - TrieNode h = node.insert("/u/h"); - Assert.assertEquals(a, node.searchExact("/a").get()); - Assert.assertEquals(b, node.searchExact("/a/b").get()); - Assert.assertEquals(f, node.searchExact("/a/e/f").get()); - Assert.assertEquals(d, node.searchExact("/c/d").get()); - Assert.assertEquals(g, node.searchExact("/c/g").get()); - Assert.assertEquals(h, node.searchExact("/u/h").get()); - Assert.assertEquals(Optional.empty(), node.searchExact("/")); - Assert.assertEquals(Optional.empty(), node.searchExact("/ab")); - Assert.assertEquals(Optional.empty(), node.searchExact("/a/b/c")); - Assert.assertEquals(Optional.empty(), node.searchExact("/a/d")); - } - - @Test - public void deleteIfTrue() { - TrieNode node = new TrieNode<>(); - TrieNode a = node.insert("/a"); - TrieNode b = node.insert("/a/b"); - TrieNode f = node.insert("/a/e/f"); - TrieNode d = node.insert("/c/d"); - TrieNode g = node.insert("/c/g"); - TrieNode h = node.insert("/u/h"); - Assert.assertTrue(node.search("/a/b").contains(b)); - TrieNode b2 = node.deleteIf("/a/b", n -> { - Assert.assertEquals(b, n); - return true; - }); - Assert.assertEquals(b, b2); - Assert.assertFalse(node.search("/a/b").contains(b)); - Assert.assertTrue(node.search("/a").contains(a)); - TrieNode a2 = node.deleteIf("/a", n -> { - Assert.assertEquals(a, n); - return true; - }); - Assert.assertEquals(a, a2); - Assert.assertFalse(node.search("/a").contains(a)); - Assert.assertTrue(node.search("/a/e/f").contains(f)); - TrieNode c2 = node.deleteIf("/c", n -> true); - Assert.assertNull(c2); - Assert.assertTrue(node.search("/c/d").contains(d)); - Assert.assertTrue(node.search("/c/g").contains(g)); - TrieNode h2 = node.deleteIf("/u/h", n -> { - Assert.assertEquals(h, n); - return true; - }); - Assert.assertEquals(h, h2); - TrieNode nil = node.deleteIf("/n", n -> { - Assert.fail(); - return true; - }); - Assert.assertNull(nil); - } - - @Test - public void deleteIfFalse() { - TrieNode node = new TrieNode<>(); - TrieNode a = node.insert("/a"); - TrieNode b = node.insert("/a/b"); - Assert.assertTrue(node.search("/a/b").contains(b)); - TrieNode b2 = node.deleteIf("/a/b", n -> false); - Assert.assertNull(b2); - Assert.assertTrue(node.search("/a").contains(a)); - TrieNode a2 = node.deleteIf("/a", n -> false); - Assert.assertNull(a2); - } - - @Test - public void deleteAndInsert() { - TrieNode node = new TrieNode<>(); - TrieNode a = node.insert("/a"); - TrieNode b = node.insert("/a/b"); - - Assert.assertTrue(node.search("/a/b").contains(b)); - TrieNode b2 = node.deleteIf("/a/b", n -> { - Assert.assertEquals(b, n); - return true; - }); - Assert.assertEquals(b, b2); - Assert.assertFalse(node.search("/a/b").contains(b)); - TrieNode b3 = node.insert("/a/b"); - Assert.assertTrue(node.search("/a/b").contains(b3)); - - Assert.assertTrue(node.search("/a").contains(a)); - Assert.assertTrue(node.search("/a/b").contains(a)); - TrieNode a2 = node.deleteIf("/a", n -> { - Assert.assertEquals(a, n); - return true; - }); - Assert.assertEquals(a, a2); - Assert.assertFalse(node.search("/a/b").contains(a)); - Assert.assertFalse(node.search("/a").contains(a)); - Assert.assertTrue(node.search("/a/b").contains(b3)); - TrieNode a3 = node.insert("/a"); - Assert.assertTrue(node.search("/a/b").contains(b3)); - Assert.assertTrue(node.search("/a").contains(a3)); - Assert.assertTrue(node.search("/a/b").contains(a3)); - } - - @Test - public void getChildren() { - TrieNode node = new TrieNode<>(); - TrieNode a = node.insert("/a"); - TrieNode b = node.insert("/a/b"); - TrieNode f = node.insert("/a/e/f"); - TrieNode d = node.insert("/c/d"); - TrieNode g = node.insert("/c/g"); - TrieNode h = node.insert("/u/h"); - Assert.assertArrayEquals(new TrieNode[] {a, b, f}, - node.getLeafChildren("/a").toArray(TrieNode[]::new)); - Assert.assertArrayEquals(new TrieNode[] {b}, - node.getLeafChildren("/a/b").toArray(TrieNode[]::new)); - Assert.assertArrayEquals(new TrieNode[] {f}, - node.getLeafChildren("/a/e/f").toArray(TrieNode[]::new)); - Assert.assertArrayEquals(new TrieNode[] {d}, - node.getLeafChildren("/c/d").toArray(TrieNode[]::new)); - Assert.assertEquals(new HashSet(Arrays.asList(a, b, f, d, g, h)), - node.getLeafChildren("/").collect(Collectors.toSet())); - } - - @Test - public void clearTrie() { - TrieNode node = new TrieNode<>(); - TrieNode a = node.insert("/a"); - TrieNode b = node.insert("/a/b"); - TrieNode f = node.insert("/a/e/f"); - TrieNode d = node.insert("/c/d"); - TrieNode g = node.insert("/c/g"); - TrieNode h = node.insert("/u/h"); - - node.clear(); - // after clearing, each node should only contain itself - for (TrieNode nxt : ImmutableList.of(a, b, f, d, g, h)) { - Assert.assertEquals(Collections.singletonList(nxt), - nxt.getLeafChildren("/").collect(Collectors.toList())); - } - } -} diff --git a/core/common/src/test/java/alluxio/heartbeat/SleepingTimerTest.java b/core/common/src/test/java/alluxio/heartbeat/SleepingTimerTest.java deleted file mode 100644 index ae8ef03d8aea..000000000000 --- a/core/common/src/test/java/alluxio/heartbeat/SleepingTimerTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.heartbeat; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import alluxio.clock.ManualClock; -import alluxio.time.Sleeper; - -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.slf4j.Logger; - -import java.time.Duration; - -/** - * Unit tests for {@link SleepingTimer}. - */ -public final class SleepingTimerTest { - private static final String THREAD_NAME = "sleepingtimer-test-thread-name"; - private static final long INTERVAL_MS = 500; - private Logger mMockLogger; - private ManualClock mFakeClock; - private Sleeper mMockSleeper; - - @Before - public void before() { - mMockLogger = mock(Logger.class); - mFakeClock = new ManualClock(); - mMockSleeper = mock(Sleeper.class); - } - - @Test - public void warnWhenExecutionTakesLongerThanInterval() throws Exception { - SleepingTimer timer = - new SleepingTimer(THREAD_NAME, INTERVAL_MS, mMockLogger, mFakeClock, mMockSleeper); - - timer.tick(); - mFakeClock.addTimeMs(5 * INTERVAL_MS); - timer.tick(); - - verify(mMockLogger).warn(anyString(), anyString(), Mockito.anyLong(), - Mockito.anyLong()); - } - - @Test - public void sleepForSpecifiedInterval() throws Exception { - final SleepingTimer timer = - new SleepingTimer(THREAD_NAME, INTERVAL_MS, mMockLogger, mFakeClock, mMockSleeper); - timer.tick(); // first tick won't sleep - verify(mMockSleeper, times(0)).sleep(any(Duration.class)); - timer.tick(); - verify(mMockSleeper).sleep(Duration.ofMillis(INTERVAL_MS)); - } - - /** - * Tests that the sleeping timer will attempt to run at the same interval, independently of how - * long the execution between ticks takes. For example, if the interval is 100ms and execution - * takes 80ms, the timer should sleep for only 20ms to maintain the regular interval of 100ms. - */ - @Test - public void maintainInterval() throws Exception { - SleepingTimer stimer = - new SleepingTimer(THREAD_NAME, INTERVAL_MS, mMockLogger, mFakeClock, mMockSleeper); - - stimer.tick(); - mFakeClock.addTimeMs(INTERVAL_MS / 3); - stimer.tick(); - verify(mMockSleeper).sleep(Duration.ofMillis(INTERVAL_MS - (INTERVAL_MS / 3))); - } -} diff --git a/core/common/src/test/java/alluxio/master/PollingMasterInquireClientTest.java b/core/common/src/test/java/alluxio/master/PollingMasterInquireClientTest.java deleted file mode 100644 index a5b1c088d3ca..000000000000 --- a/core/common/src/test/java/alluxio/master/PollingMasterInquireClientTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master; - -import static org.junit.Assert.fail; - -import alluxio.Constants; -import alluxio.conf.ConfigurationBuilder; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.ServiceType; -import alluxio.network.RejectingServer; -import alluxio.retry.CountingRetry; -import alluxio.util.network.NetworkAddressUtils; - -import org.junit.Rule; -import org.junit.Test; - -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.List; - -/** - * Unit tests for {@link PollingMasterInquireClient}. - */ -public class PollingMasterInquireClientTest { - @Rule - public PortReservationRule mPort = new PortReservationRule(); - - @Test(timeout = 10000) - public void pollRejectingDoesntHang() throws Exception { - int port = mPort.getPort(); - InetSocketAddress serverAddress = new InetSocketAddress("127.0.0.1", port); - RejectingServer s = new RejectingServer(serverAddress); - s.start(); - List addrs = Arrays.asList(InetSocketAddress - .createUnresolved(NetworkAddressUtils.getLocalHostName(Constants.SECOND_MS), port)); - PollingMasterInquireClient client = new PollingMasterInquireClient(addrs, - () -> new CountingRetry(0), new ConfigurationBuilder().build(), - ServiceType.META_MASTER_CLIENT_SERVICE); - try { - client.getPrimaryRpcAddress(); - fail("Expected polling to fail"); - } catch (UnavailableException e) { - // Expected - } - } -} diff --git a/core/common/src/test/java/alluxio/underfs/ObjectUnderFileSystemTest.java b/core/common/src/test/java/alluxio/underfs/ObjectUnderFileSystemTest.java deleted file mode 100644 index b1351a0b58db..000000000000 --- a/core/common/src/test/java/alluxio/underfs/ObjectUnderFileSystemTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.underfs; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import alluxio.AlluxioURI; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; - -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.SocketException; - -public class ObjectUnderFileSystemTest { - private static final AlluxioConfiguration CONF = Configuration.global(); - - private ObjectUnderFileSystem mObjectUFS = new MockObjectUnderFileSystem(new AlluxioURI("/"), - UnderFileSystemConfiguration.defaults(CONF)); - - @Test - public void testRetryOnException() { - //if ufs throws SocketException, it will retry. - ObjectUnderFileSystem.ObjectStoreOperation mockOp = Mockito.mock( - ObjectUnderFileSystem.ObjectStoreOperation.class); - try { - Mockito.when(mockOp.apply()).thenThrow(new SocketException()).thenReturn(null); - mObjectUFS.retryOnException(mockOp, () -> "test"); - // retry policy will retry, so op will apply twice. - Mockito.verify(mockOp, Mockito.atLeast(2)).apply(); - } catch (IOException e) { - fail(); - } - - //if ufs throws other Exception, it will throw Exception. - try { - Mockito.reset(mockOp); - Mockito.when(mockOp.apply()).thenThrow(new FileNotFoundException()).thenReturn(null); - mObjectUFS.retryOnException(mockOp, () -> "test"); - } catch (IOException e) { - assertTrue(e instanceof FileNotFoundException); - } - try { - // op only apply once and then throw exception. - Mockito.verify(mockOp, Mockito.atLeast(1)).apply(); - } catch (IOException e) { - fail(); - } - } -} diff --git a/core/common/src/test/java/alluxio/wire/MasterInfoTest.java b/core/common/src/test/java/alluxio/wire/MasterInfoTest.java deleted file mode 100644 index 51b41f6cc1ca..000000000000 --- a/core/common/src/test/java/alluxio/wire/MasterInfoTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.lang3.RandomStringUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Random; - -public class MasterInfoTest { - - @Test - public void json() throws Exception { - MasterInfo masterInfo = createRandom(); - ObjectMapper mapper = new ObjectMapper(); - MasterInfo other = - mapper.readValue(mapper.writeValueAsBytes(masterInfo), MasterInfo.class); - checkEquality(masterInfo, other); - } - - public void checkEquality(MasterInfo a, MasterInfo b) { - Assert.assertEquals(a.getId(), b.getId()); - Assert.assertEquals(a.getAddress(), b.getAddress()); - Assert.assertEquals(a.getLastUpdatedTimeMs(), b.getLastUpdatedTimeMs()); - Assert.assertEquals(a, b); - } - - public static MasterInfo createRandom() { - Random random = new Random(); - long id = random.nextLong(); - Address address = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt()); - - MasterInfo result = new MasterInfo(id, address); - result.updateLastUpdatedTimeMs(); - return result; - } -} diff --git a/core/common/src/test/java/alluxio/wire/TieredIdentityTest.java b/core/common/src/test/java/alluxio/wire/TieredIdentityTest.java deleted file mode 100644 index 34d71614798e..000000000000 --- a/core/common/src/test/java/alluxio/wire/TieredIdentityTest.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; - -import alluxio.conf.InstancedConfiguration; -import alluxio.grpc.GrpcUtils; -import alluxio.network.TieredIdentityFactory; -import alluxio.util.CommonUtils; -import alluxio.util.TieredIdentityUtils; -import alluxio.wire.TieredIdentity.LocalityTier; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Test; - -import java.net.InetAddress; -import java.util.Arrays; -import java.util.List; -import java.util.Random; - -/** - * Unit tests for {@link TieredIdentity}. - */ -public class TieredIdentityTest { - - private final InstancedConfiguration mConfiguration = alluxio.conf.Configuration.copyGlobal(); - - @Test - public void nearest() throws Exception { - TieredIdentity id1 = TieredIdentityFactory.fromString("node=A,rack=rack1", mConfiguration); - TieredIdentity id2 = TieredIdentityFactory.fromString("node=B,rack=rack2", mConfiguration); - TieredIdentity id3 = TieredIdentityFactory.fromString("node=C,rack=rack2", mConfiguration); - List identities = Arrays.asList(id1, id2, id3); - - assertSame(id1, TieredIdentityUtils - .nearest(TieredIdentityFactory.fromString("node=D,rack=rack1", mConfiguration), identities, - mConfiguration).get()); - assertSame(id2, TieredIdentityUtils - .nearest(TieredIdentityFactory.fromString("node=B,rack=rack2", mConfiguration), identities, - mConfiguration).get()); - assertSame(id3, TieredIdentityUtils - .nearest(TieredIdentityFactory.fromString("node=C,rack=rack2", mConfiguration), identities, - mConfiguration).get()); - assertSame(id1, TieredIdentityUtils - .nearest(TieredIdentityFactory.fromString("node=D,rack=rack3", mConfiguration), identities, - mConfiguration).get()); - } - - @Test - public void json() throws Exception { - TieredIdentity tieredIdentity = createRandomTieredIdentity(); - ObjectMapper mapper = new ObjectMapper(); - TieredIdentity other = - mapper.readValue(mapper.writeValueAsBytes(tieredIdentity), TieredIdentity.class); - checkEquality(tieredIdentity, other); - } - - @Test - public void proto() { - TieredIdentity tieredIdentity = createRandomTieredIdentity(); - TieredIdentity other = GrpcUtils.fromProto(GrpcUtils.toProto(tieredIdentity)); - checkEquality(tieredIdentity, other); - } - - @Test - public void matchByStringEquality() { - LocalityTier lt1 = new LocalityTier("node", "NonResolvableHostname-A"); - LocalityTier lt2 = new LocalityTier("node", "NonResolvableHostname-A"); - LocalityTier lt3 = new LocalityTier("node", "NonResolvableHostname-B"); - LocalityTier lt4 = new LocalityTier("rack", "NonResolvableHostname-A"); - LocalityTier lt5 = new LocalityTier("rack", "NonResolvableHostname-B"); - LocalityTier lt6 = new LocalityTier("rack", "NonResolvableHostname-B"); - LocalityTier lt7 = new LocalityTier("rack", ""); - LocalityTier lt8 = new LocalityTier("node", "NonResolvableHostname-A"); - LocalityTier lt9 = new LocalityTier("node", ""); - assertTrue(TieredIdentityUtils.matches(lt1, lt1, true)); - assertTrue(TieredIdentityUtils.matches(lt1, lt2, true)); - assertFalse(TieredIdentityUtils.matches(lt2, lt3, true)); - assertTrue(TieredIdentityUtils.matches(lt5, lt6, true)); - assertFalse(TieredIdentityUtils.matches(lt4, lt5, true)); - assertFalse(TieredIdentityUtils.matches(lt6, lt7, true)); - assertFalse(TieredIdentityUtils.matches(lt8, lt9, true)); - } - - @Test - public void matchByIpResolution() throws Exception { - assumeTrue(InetAddress.getByName("localhost").getHostAddress().equals("127.0.0.1")); - LocalityTier lt1 = new LocalityTier("node", "localhost"); - LocalityTier lt2 = new LocalityTier("node", "127.0.0.1"); - - assertTrue(TieredIdentityUtils.matches(lt1, lt2, true)); - assertFalse(TieredIdentityUtils.matches(lt1, lt2, false)); - } - - public void string() { - TieredIdentity identity = new TieredIdentity( - Arrays.asList(new LocalityTier("k1", "v1"), new LocalityTier("k2", "v2"))); - assertEquals("TieredIdentity(k1=v1, k2=v2)", identity.toString()); - } - - public void checkEquality(TieredIdentity a, TieredIdentity b) { - assertEquals(a.getTiers(), b.getTiers()); - assertEquals(a, b); - } - - public static TieredIdentity createRandomTieredIdentity() { - return new TieredIdentity( - Arrays.asList(createRandomLocalityTier(), createRandomLocalityTier())); - } - - private static LocalityTier createRandomLocalityTier() { - Random random = new Random(); - - String tier = CommonUtils.randomAlphaNumString(random.nextInt(10) + 1); - String value = CommonUtils.randomAlphaNumString(random.nextInt(10) + 1); - return new LocalityTier(tier, value); - } -} diff --git a/core/common/src/test/java/alluxio/wire/WorkerInfoTest.java b/core/common/src/test/java/alluxio/wire/WorkerInfoTest.java deleted file mode 100644 index 00cf71e5390a..000000000000 --- a/core/common/src/test/java/alluxio/wire/WorkerInfoTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import alluxio.Constants; -import alluxio.grpc.GrpcUtils; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.RandomStringUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -public class WorkerInfoTest { - - @Test - public void json() throws Exception { - WorkerInfo workerInfo = createRandom(); - ObjectMapper mapper = new ObjectMapper(); - WorkerInfo other = - mapper.readValue(mapper.writeValueAsBytes(workerInfo), WorkerInfo.class); - checkEquality(workerInfo, other); - } - - @Test - public void proto() { - WorkerInfo workerInfo = createRandom(); - WorkerInfo other = GrpcUtils.fromProto(GrpcUtils.toProto(workerInfo)); - checkEquality(workerInfo, other); - } - - @Test - public void lastContactSecComparator() { - Assert.assertTrue(compareLostWorkersWithTimes(0, 1) < 0); - Assert.assertTrue(compareLostWorkersWithTimes(1, 0) > 0); - Assert.assertTrue(compareLostWorkersWithTimes(1, 1) == 0); - Assert.assertTrue(compareLostWorkersWithTimes(-1, 1) < 0); - Assert.assertTrue(compareLostWorkersWithTimes(1, -1) > 0); - } - - public void checkEquality(WorkerInfo a, WorkerInfo b) { - Assert.assertEquals(a.getId(), b.getId()); - Assert.assertEquals(a.getAddress(), b.getAddress()); - Assert.assertEquals(a.getLastContactSec(), b.getLastContactSec()); - Assert.assertEquals(a.getCapacityBytes(), b.getCapacityBytes()); - Assert.assertEquals(a.getUsedBytes(), b.getUsedBytes()); - Assert.assertEquals(a.getStartTimeMs(), b.getStartTimeMs()); - Assert.assertEquals(a.getState(), b.getState()); - Assert.assertEquals(a.getCapacityBytesOnTiers(), b.getCapacityBytesOnTiers()); - Assert.assertEquals(a.getUsedBytesOnTiers(), b.getUsedBytesOnTiers()); - Assert.assertEquals(a, b); - } - - private static int compareLostWorkersWithTimes(int time1, int time2) { - WorkerInfo.LastContactSecComparator comparator = - new WorkerInfo.LastContactSecComparator(); - WorkerInfo worker1 = createRandom(); - WorkerInfo worker2 = createRandom(); - worker1.setLastContactSec(time1); - worker2.setLastContactSec(time2); - return comparator.compare(worker1, worker2); - } - - public static WorkerInfo createRandom() { - WorkerInfo result = new WorkerInfo(); - Random random = new Random(); - - long id = random.nextLong(); - WorkerNetAddress address = WorkerNetAddressTest.createRandom(); - int lastContactSec = random.nextInt(); - long capacityBytes = random.nextLong(); - long usedBytes = random.nextLong(); - long startTimeMs = random.nextLong(); - Map capacityBytesOnTiers = new HashMap<>(); - capacityBytesOnTiers.put(Constants.MEDIUM_MEM, capacityBytes); - Map usedBytesOnTiers = new HashMap<>(); - usedBytesOnTiers.put(Constants.MEDIUM_MEM, usedBytes); - String state = random.nextInt(2) == 1 ? "In Service" : "Out of Service"; - String version = String.format("%d.%d.%d", random.nextInt(10), - random.nextInt(20), random.nextInt(10)); - String revision = DigestUtils.sha1Hex(RandomStringUtils.random(10)); - - result.setId(id); - result.setAddress(address); - result.setLastContactSec(lastContactSec); - result.setCapacityBytes(capacityBytes); - result.setUsedBytes(usedBytes); - result.setStartTimeMs(startTimeMs); - result.setState(state); - result.setCapacityBytesOnTiers(capacityBytesOnTiers); - result.setUsedBytesOnTiers(usedBytesOnTiers); - result.setVersion(version); - result.setRevision(revision); - return result; - } -} diff --git a/core/common/src/test/java/alluxio/wire/WorkerNetAddressTest.java b/core/common/src/test/java/alluxio/wire/WorkerNetAddressTest.java deleted file mode 100644 index 4540a2c88887..000000000000 --- a/core/common/src/test/java/alluxio/wire/WorkerNetAddressTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.wire; - -import alluxio.grpc.GrpcUtils; -import alluxio.util.CommonUtils; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Random; - -public class WorkerNetAddressTest { - - @Test - public void json() throws Exception { - WorkerNetAddress workerNetAddress = createRandom(); - ObjectMapper mapper = new ObjectMapper(); - WorkerNetAddress other = - mapper.readValue(mapper.writeValueAsBytes(workerNetAddress), WorkerNetAddress.class); - checkEquality(workerNetAddress, other); - } - - @Test - public void proto() { - WorkerNetAddress workerNetAddress = createRandom(); - WorkerNetAddress other = GrpcUtils.fromProto(GrpcUtils.toProto(workerNetAddress)); - checkEquality(workerNetAddress, other); - } - - public void checkEquality(WorkerNetAddress a, WorkerNetAddress b) { - Assert.assertEquals(a.getHost(), b.getHost()); - Assert.assertEquals(a.getRpcPort(), b.getRpcPort()); - Assert.assertEquals(a.getDataPort(), b.getDataPort()); - Assert.assertEquals(a.getWebPort(), b.getWebPort()); - Assert.assertEquals(a.getTieredIdentity(), b.getTieredIdentity()); - Assert.assertEquals(a, b); - } - - public static WorkerNetAddress createRandom() { - WorkerNetAddress result = new WorkerNetAddress(); - Random random = new Random(); - - String host = CommonUtils.randomAlphaNumString(random.nextInt(10)); - int rpcPort = random.nextInt(); - int dataPort = random.nextInt(); - int webPort = random.nextInt(); - TieredIdentity identity = TieredIdentityTest.createRandomTieredIdentity(); - - result.setHost(host); - result.setRpcPort(rpcPort); - result.setDataPort(dataPort); - result.setWebPort(webPort); - result.setTieredIdentity(identity); - - return result; - } -} diff --git a/core/common/src/test/java/alluxio/worker/block/io/LocalFileBlockReaderTest.java b/core/common/src/test/java/alluxio/worker/block/io/LocalFileBlockReaderTest.java deleted file mode 100644 index 1e5ab857926b..000000000000 --- a/core/common/src/test/java/alluxio/worker/block/io/LocalFileBlockReaderTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.io; - -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import alluxio.exception.status.FailedPreconditionException; -import alluxio.util.io.BufferUtils; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; - -/** - * Tests for the {@link LocalFileBlockReader} class. - */ -public class LocalFileBlockReaderTest { - private static final long TEST_BLOCK_SIZE = 1024; - private LocalFileBlockReader mReader; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - /** - * Sets up the file path and file block reader before a test runs. - */ - @Before - public void before() throws Exception { - String testFilePath = mFolder.newFile().getAbsolutePath(); - byte[] buffer = BufferUtils.getIncreasingByteArray((int) TEST_BLOCK_SIZE); - BufferUtils.writeBufferToFile(testFilePath, buffer); - mReader = new LocalFileBlockReader(testFilePath); - } - - /** - * Test for the {@link LocalFileBlockReader#getChannel()} method. - */ - @Test - public void getChannel() throws Exception { - ReadableByteChannel channel = mReader.getChannel(); - Assert.assertNotNull(channel); - ByteBuffer buffer = ByteBuffer.allocate((int) TEST_BLOCK_SIZE); - int bytesRead = channel.read(buffer); - Assert.assertEquals(TEST_BLOCK_SIZE, bytesRead); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE, buffer)); - } - - @Test - public void getLocation() { - Assert.assertEquals(mReader.getFilePath(), mReader.getLocation()); - } - - /** - * Test for the {@link LocalFileBlockReader#getLength()} method. - */ - @Test - public void getLength() { - Assert.assertEquals(TEST_BLOCK_SIZE, mReader.getLength()); - } - - /** - * Tests that an exception is thrown if the read exceeds the file length limit. - */ - @Test - public void readWithInvalidArgument() throws Exception { - Exception e = assertThrows(IllegalArgumentException.class, () -> { - mReader.read(TEST_BLOCK_SIZE - 1, 2); - }); - assertTrue(e.getMessage().contains("exceeding fileSize")); - } - - /** - * Test for the {@link LocalFileBlockReader#read(long, long)} method. - */ - @Test - public void read() throws Exception { - ByteBuffer buffer; - - // Read 1/4 block by setting the length to be 1/4 of the block size. - buffer = mReader.read(0, TEST_BLOCK_SIZE / 4); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE / 4, buffer)); - - // Read entire block by setting the length to be block size. - buffer = mReader.read(0, TEST_BLOCK_SIZE); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE, buffer)); - } - - /** - * Tests that a {@link FailedPreconditionException} is thrown when trying to read from a reader - * after closing it. - */ - @Test - public void close() throws Exception { - mReader.close(); - assertThrows(IOException.class, () -> { - mReader.read(0, TEST_BLOCK_SIZE); - }); - } -} diff --git a/core/common/src/test/java/alluxio/worker/block/io/LocalFileBlockWriterTest.java b/core/common/src/test/java/alluxio/worker/block/io/LocalFileBlockWriterTest.java deleted file mode 100644 index 5f321193e1d4..000000000000 --- a/core/common/src/test/java/alluxio/worker/block/io/LocalFileBlockWriterTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.io; - -import alluxio.exception.runtime.InternalRuntimeException; -import alluxio.util.io.BufferUtils; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Paths; - -/** - * Tests for the {@link LocalFileBlockWriter} class. - */ -public final class LocalFileBlockWriterTest { - private static final int TEST_BLOCK_SIZE = 1024; - - private LocalFileBlockWriter mWriter; - private String mTestFilePath; - - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - @Before - public void before() throws Exception { - mTestFilePath = mFolder.newFile().getAbsolutePath(); - mWriter = new LocalFileBlockWriter(mTestFilePath); - } - - @After - public void after() throws Exception { - mWriter.close(); - } - - @Test - public void appendByteBuf() throws Exception { - ByteBuf buffer = Unpooled.wrappedBuffer( - BufferUtils.getIncreasingByteBuffer(TEST_BLOCK_SIZE)); - buffer.markReaderIndex(); - Assert.assertEquals(TEST_BLOCK_SIZE, mWriter.append(buffer)); - buffer.resetReaderIndex(); - Assert.assertEquals(TEST_BLOCK_SIZE, mWriter.append(buffer)); - mWriter.close(); - Assert.assertEquals(2 * TEST_BLOCK_SIZE, new File(mTestFilePath).length()); - ByteBuffer result = ByteBuffer.wrap(Files.readAllBytes(Paths.get(mTestFilePath))); - result.position(0).limit(TEST_BLOCK_SIZE); - BufferUtils.equalIncreasingByteBuffer(0, TEST_BLOCK_SIZE, result.slice()); - result.position(TEST_BLOCK_SIZE).limit(TEST_BLOCK_SIZE * 2); - BufferUtils.equalIncreasingByteBuffer(0, TEST_BLOCK_SIZE, result.slice()); - } - - @Test - public void append() throws Exception { - Assert.assertEquals(TEST_BLOCK_SIZE, - mWriter.append(BufferUtils.getIncreasingByteBuffer(TEST_BLOCK_SIZE))); - Assert.assertEquals(TEST_BLOCK_SIZE, - mWriter.append(BufferUtils.getIncreasingByteBuffer(TEST_BLOCK_SIZE))); - mWriter.close(); - Assert.assertEquals(2 * TEST_BLOCK_SIZE, new File(mTestFilePath).length()); - ByteBuffer result = ByteBuffer.wrap(Files.readAllBytes(Paths.get(mTestFilePath))); - result.position(0).limit(TEST_BLOCK_SIZE); - BufferUtils.equalIncreasingByteBuffer(0, TEST_BLOCK_SIZE, result.slice()); - result.position(TEST_BLOCK_SIZE).limit(TEST_BLOCK_SIZE * 2); - BufferUtils.equalIncreasingByteBuffer(0, TEST_BLOCK_SIZE, result.slice()); - } - - @Test - public void close() throws Exception { - ByteBuffer buf = BufferUtils.getIncreasingByteBuffer(TEST_BLOCK_SIZE); - Assert.assertEquals(TEST_BLOCK_SIZE, mWriter.append(buf)); - mWriter.close(); - // Append after close, expect append to fail and throw exception - Assert.assertThrows(InternalRuntimeException.class, () -> mWriter.append(buf)); - } -} diff --git a/core/pom.xml b/core/pom.xml deleted file mode 100644 index 1010e55ade22..000000000000 --- a/core/pom.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - 4.0.0 - - org.alluxio - alluxio-parent - 2.10.0-SNAPSHOT - - alluxio-core - pom - Alluxio Core - Parent POM for Alluxio core - - - client - common - transport - server - - - - - ${project.parent.basedir}/build - - diff --git a/core/server/common/pom.xml b/core/server/common/pom.xml deleted file mode 100644 index 457fb9013a5e..000000000000 --- a/core/server/common/pom.xml +++ /dev/null @@ -1,173 +0,0 @@ - - - 4.0.0 - - alluxio-core-server - org.alluxio - 2.10.0-SNAPSHOT - - alluxio-core-server-common - jar - Alluxio Core - Server - Common Utilities - Common utilities for Alluxio core services - - - - - ${project.parent.parent.parent.basedir}/build - 2.4.0 - - - - - - com.esotericsoftware - kryo - - - ratis-server - org.apache.ratis - ${alluxio.ratis.version} - - - ratis-grpc - org.apache.ratis - ${alluxio.ratis.version} - - - - io.atomix.catalyst - catalyst-transport - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.google.guava - guava - - - - com.sun.xml.bind - jaxb-impl - runtime - - - commons-cli - commons-cli - - - io.dropwizard.metrics - metrics-core - - - io.dropwizard.metrics - metrics-json - - - io.prometheus - simpleclient - - - io.prometheus - simpleclient_servlet - - - io.prometheus - simpleclient_dropwizard - - - javax.servlet - javax.servlet-api - - - jakarta.ws.rs - jakarta.ws.rs-api - - - org.apache.commons - commons-compress - - - org.apache.commons - commons-lang3 - - - org.apache.curator - curator-recipes - - - org.eclipse.jetty - jetty-server - - - org.eclipse.jetty - jetty-servlet - - - org.eclipse.jetty - jetty-util - - - com.hubspot.jackson - jackson-datatype-protobuf - 0.9.12 - - - com.google.code.findbugs - annotations - - - - - - - org.glassfish.jersey.core - jersey-server - test - - - org.glassfish.jersey.inject - jersey-hk2 - test - - - - - org.alluxio - alluxio-core-common - ${project.version} - - - org.alluxio - alluxio-core-transport - ${project.version} - - - - - org.alluxio - alluxio-core-common - ${project.version} - test-jar - test - - - - diff --git a/core/server/common/src/main/java/alluxio/ProcessUtils.java b/core/server/common/src/main/java/alluxio/ProcessUtils.java deleted file mode 100644 index 802d6d77a986..000000000000 --- a/core/server/common/src/main/java/alluxio/ProcessUtils.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; - -import com.google.common.base.Throwables; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utility methods for Alluxio {@link Process}es. - */ -public final class ProcessUtils { - private static final Logger LOG = LoggerFactory.getLogger(ProcessUtils.class); - - /** - * Runs the given {@link Process}. This method should only be called from {@code main()} methods. - * - * @param process the {@link Process} to run - */ - public static void run(Process process) { - try { - LOG.info("Starting {}.", process); - LOG.info("Alluxio version: {}-{}", RuntimeConstants.VERSION, ProjectConstants.REVISION); - LOG.info("Java version: {}", System.getProperty("java.version")); - process.start(); - LOG.info("Stopping {}.", process); - System.exit(0); - } catch (Throwable t) { - LOG.error("Uncaught exception while running {}, stopping it and exiting. " - + "Exception \"{}\", Root Cause \"{}\"", process, t, Throwables.getRootCause(t), t); - try { - process.stop(); - } catch (Throwable t2) { - // continue to exit - LOG.error("Uncaught exception while stopping {}, simply exiting. " - + "Exception \"{}\", Root Cause \"{}\"", process, t2, Throwables.getRootCause(t2), - t2); - } - System.exit(-1); - } - } - - /** - * Logs a fatal error and then exits the system. - * - * @param logger the logger to log to - * @param format the error message format string - * @param args args for the format string - */ - public static void fatalError(Logger logger, String format, Object... args) { - fatalError(logger, new Throwable(), format, args); - } - - /** - * Logs a fatal error and then exits the system. - * - * @param logger the logger to log to - * @param t the throwable causing the fatal error - * @param format the error message format string - * @param args args for the format string - */ - public static void fatalError(Logger logger, Throwable t, String format, Object... args) { - String message = String.format("Fatal error: " + format, args); - if (t != null) { - message += "\n" + Throwables.getStackTraceAsString(t); - } - if (Configuration.getBoolean(PropertyKey.TEST_MODE)) { - throw new RuntimeException(message); - } - logger.error(message); - System.exit(-1); - } - - /** - * Adds a shutdown hook that will be invoked when a signal is sent to this process. - * - * The process may be utilizing some resources, and this shutdown hook will be invoked by - * JVM when a SIGTERM is sent to the process by "kill" command. The shutdown hook calls - * {@link Process#stop()} method to cleanly release the resources and exit. - * - * @param process the data structure representing the process to terminate - */ - public static void stopProcessOnShutdown(final Process process) { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - process.stop(); - } catch (Throwable t) { - LOG.error("Failed to stop process", t); - } - }, "alluxio-process-shutdown-hook")); - } - - private ProcessUtils() {} // prevent instantiation -} diff --git a/core/server/common/src/main/java/alluxio/Registry.java b/core/server/common/src/main/java/alluxio/Registry.java deleted file mode 100644 index 920088fcb475..000000000000 --- a/core/server/common/src/main/java/alluxio/Registry.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio; - -import alluxio.resource.LockResource; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; - -import com.google.common.collect.Lists; - -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Class for registering individual {@link Server}s that run within an Alluxio process. The intended - * use of this class is for the Alluxio process to register the individual {@link Server}s using an - * instance of this class. The registry is passed as an argument to constructors of individual - * {@link Server}s who are expected to add themselves to the registry. - * - * The reason for using an instance as opposed to a static class is to enable dependency - * injection in tests. - * - * @param the type of the {@link Server} - * @param the type of the {@link Server#start} method options - */ -@ThreadSafe -public class Registry, U> { - private final Map, T> mRegistry = new HashMap<>(); - private final Lock mLock = new ReentrantLock(); - - /** - * Creates a new instance of {@link Registry}. - */ - public Registry() {} - - /** - * Convenience method for calling {@link #get(Class, int)} with a default timeout. - * - * @param clazz the class of the {@link Server} to get - * @param the type of the {@link Server} to get - * @return the {@link Server} instance - */ - public W get(final Class clazz) { - return get(clazz, Constants.DEFAULT_REGISTRY_GET_TIMEOUT_MS); - } - - /** - * Attempts to look up the {@link Server} for the given class. Because {@link Server}s can be - * looked up and added to the registry in parallel, the lookup is retried until the given timeout - * period has elapsed. - * - * @param clazz the class of the {@link Server} to get - * @param timeoutMs timeout for looking up the server - * @param the type of the {@link Server} to get - * @return the {@link Server} instance - */ - public W get(final Class clazz, int timeoutMs) { - try { - CommonUtils.waitFor("server " + clazz.getName() + " to be created", () -> { - try (LockResource r = new LockResource(mLock)) { - return mRegistry.get(clazz) != null; - } - }, WaitForOptions.defaults().setTimeoutMs(timeoutMs)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } catch (TimeoutException e) { - throw new RuntimeException(e); - } - T server = mRegistry.get(clazz); - if (!(clazz.isInstance(server))) { - throw new RuntimeException("Server is not an instance of " + clazz.getName()); - } - return clazz.cast(server); - } - - /** - * @param clazz the class of the {@link Server} to add - * @param server the {@link Server} to add - * @param the type of the {@link Server} to add - */ - public void add(Class clazz, T server) { - try (LockResource r = new LockResource(mLock)) { - mRegistry.put(clazz, server); - } - } - - /** - * @return a list of all the registered {@link Server}s, order by dependency relation - */ - public List getServers() { - List servers = new ArrayList<>(mRegistry.values()); - servers.sort(new DependencyComparator()); - return servers; - } - - /** - * Starts all {@link Server}s in dependency order. If A depends on B, A is started before B. - * - * If a {@link Server} fails to start, already-started {@link Server}s will be stopped. - * - * @param options the start options - */ - public void start(U options) throws IOException { - List servers = new ArrayList<>(); - for (T server : getServers()) { - try { - server.start(options); - servers.add(server); - } catch (IOException e) { - for (T started : servers) { - started.stop(); - } - throw e; - } - } - } - - /** - * Stops all {@link Server}s in reverse dependency order. If A depends on B, A is stopped - * before B. - */ - public void stop() throws IOException { - for (T server : Lists.reverse(getServers())) { - server.stop(); - } - } - - /** - * Closes all {@link Server}s in reverse dependency order. If A depends on B, A is closed - * before B. - */ - public void close() throws IOException { - for (T server : Lists.reverse(getServers())) { - server.close(); - } - } - - /** - * Computes a transitive closure of the {@link Server} dependencies. - * - * @param server the {@link Server} to compute transitive dependencies for - * @return the transitive dependencies - */ - private Set getTransitiveDeps(T server) { - Set result = new HashSet<>(); - Deque queue = new ArrayDeque<>(); - queue.add(server); - while (!queue.isEmpty()) { - Set> deps = queue.pop().getDependencies(); - if (deps == null) { - continue; - } - for (Class clazz : deps) { - T dep = mRegistry.get(clazz); - if (dep == null) { - continue; - } - if (dep.equals(server)) { - throw new RuntimeException("Dependency cycle encountered"); - } - if (result.contains(dep)) { - continue; - } - queue.add(dep); - result.add(dep); - } - } - return result; - } - - /** - * Used for computing topological sort of {@link Server}s with respect to the dependency relation. - */ - private final class DependencyComparator implements Comparator { - /** - * Creates a new instance of {@link DependencyComparator}. - */ - DependencyComparator() {} - - @Override - public int compare(T left, T right) { - Set leftDeps = getTransitiveDeps(left); - Set rightDeps = getTransitiveDeps(right); - if (leftDeps.contains(right)) { - return 1; - } - if (rightDeps.contains(left)) { - return -1; - } - return left.getName().compareTo(right.getName()); - } - } -} diff --git a/core/server/common/src/main/java/alluxio/cli/Format.java b/core/server/common/src/main/java/alluxio/cli/Format.java deleted file mode 100644 index 0f002792c0d4..000000000000 --- a/core/server/common/src/main/java/alluxio/cli/Format.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.cli; - -import alluxio.RuntimeConstants; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.NoopMaster; -import alluxio.master.NoopUfsManager; -import alluxio.master.ServiceUtils; -import alluxio.master.journal.JournalSystem; -import alluxio.master.journal.JournalUtils; -import alluxio.util.CommonUtils; -import alluxio.util.io.FileUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Set; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Formats the Alluxio file system. - */ -@ThreadSafe -public final class Format { - private static final Logger LOG = LoggerFactory.getLogger(Format.class); - private static final String USAGE = String.format("java -cp %s %s ", - RuntimeConstants.ALLUXIO_JAR, Format.class.getCanonicalName()); - - /** - * The format mode. - */ - public enum Mode { - MASTER, - WORKER, - } - - /** - * Formats the worker data folder. - * - * @param folder folder path - */ - private static void formatWorkerDataFolder(String folder) throws IOException { - Path path = Paths.get(folder); - FileUtils.deletePathRecursively(folder); - Files.createDirectory(path); - // For short-circuit read/write to work, others needs to be able to access this directory. - // Therefore, default is 777 but if the user specifies the permissions, respect those instead. - String permissions = Configuration.getString(PropertyKey.WORKER_DATA_FOLDER_PERMISSIONS); - Set perms = PosixFilePermissions.fromString(permissions); - Files.setPosixFilePermissions(path, perms); - FileUtils.setLocalDirStickyBit(path.toAbsolutePath().toString()); - } - - /** - * Formats the Alluxio file system. - * - * @param args either {@code MASTER} or {@code WORKER} - */ - public static void main(String[] args) { - if (args.length != 1) { - LOG.info(USAGE); - System.exit(-1); - } - AlluxioConfiguration conf = Configuration.global(); - // Set the process type as "MASTER" since format needs to access the journal like the master. - CommonUtils.PROCESS_TYPE.set(CommonUtils.ProcessType.MASTER); - Mode mode = null; - try { - mode = Mode.valueOf(args[0].toUpperCase()); - } catch (IllegalArgumentException e) { - LOG.error("Unrecognized format mode: {}", args[0]); - LOG.error("Usage: {}", USAGE); - System.exit(-1); - } - try { - format(mode, conf); - } catch (Exception e) { - LOG.error("Failed to format", e); - System.exit(-1); - } - LOG.info("Formatting complete"); - System.exit(0); - } - - /** - * Formats the Alluxio file system. - * - * @param mode either {@code MASTER} or {@code WORKER} - * @param alluxioConf Alluxio configuration - */ - public static void format(Mode mode, AlluxioConfiguration alluxioConf) throws IOException { - NoopUfsManager noopUfsManager = new NoopUfsManager(); - switch (mode) { - case MASTER: - URI journalLocation = JournalUtils.getJournalLocation(); - LOG.info("Formatting master journal: {}", journalLocation); - JournalSystem journalSystem = new JournalSystem.Builder() - .setLocation(journalLocation).build(CommonUtils.ProcessType.MASTER); - for (String masterServiceName : ServiceUtils.getMasterServiceNames()) { - journalSystem.createJournal(new NoopMaster(masterServiceName, noopUfsManager)); - } - journalSystem.format(); - break; - case WORKER: - String workerDataFolder = Configuration.getString(PropertyKey.WORKER_DATA_FOLDER); - LOG.info("Formatting worker data folder: {}", workerDataFolder); - int storageLevels = Configuration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS); - for (int level = 0; level < storageLevels; level++) { - PropertyKey tierLevelDirPath = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level); - String[] dirPaths = Configuration.getString(tierLevelDirPath).split(","); - String name = "Data path for tier " + level; - for (String dirPath : dirPaths) { - String dirWorkerDataFolder = CommonUtils.getWorkerDataDirectory(dirPath, alluxioConf); - LOG.info("Formatting {}:{}", name, dirWorkerDataFolder); - formatWorkerDataFolder(dirWorkerDataFolder); - } - } - break; - default: - throw new RuntimeException(String.format("Unrecognized format mode: %s", mode)); - } - } - - private Format() {} // prevent instantiation -} diff --git a/core/server/common/src/main/java/alluxio/master/AlluxioExecutorService.java b/core/server/common/src/main/java/alluxio/master/AlluxioExecutorService.java deleted file mode 100644 index a18c82554ca8..000000000000 --- a/core/server/common/src/main/java/alluxio/master/AlluxioExecutorService.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master; - -import alluxio.concurrent.jsr.ForkJoinPool; - -import java.util.Collection; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * Forwarder over ExecutorService interface for exposing internal queue length. - */ -public class AlluxioExecutorService implements ExecutorService { - private ExecutorService mExecutor; - - /** - * Creates Alluxio ExecutorService wrapper. - * - * @param executor underlying executor - */ - public AlluxioExecutorService(ExecutorService executor) { - mExecutor = executor; - } - - /** - * @return the current RPC queue size - */ - public long getRpcQueueLength() { - if (mExecutor instanceof ThreadPoolExecutor) { - return ((ThreadPoolExecutor) mExecutor).getQueue().size(); - } else if (mExecutor instanceof ForkJoinPool) { - return ((ForkJoinPool) mExecutor).getQueuedSubmissionCount(); - } else { - throw new IllegalArgumentException( - String.format("Not supported internal executor: %s", mExecutor.getClass().getName())); - } - } - - /** - * @return the current RPC active thread count - */ - public long getActiveCount() { - if (mExecutor instanceof ThreadPoolExecutor) { - return ((ThreadPoolExecutor) mExecutor).getActiveCount(); - } else if (mExecutor instanceof ForkJoinPool) { - return ((ForkJoinPool) mExecutor).getActiveThreadCount(); - } else { - throw new IllegalArgumentException( - String.format("Not supported internal executor: %s", mExecutor.getClass().getName())); - } - } - - /** - * @return the current RPC thread pool size - */ - public long getPoolSize() { - if (mExecutor instanceof ThreadPoolExecutor) { - return ((ThreadPoolExecutor) mExecutor).getPoolSize(); - } else if (mExecutor instanceof ForkJoinPool) { - return ((ForkJoinPool) mExecutor).getPoolSize(); - } else { - throw new IllegalArgumentException( - String.format("Not supported internal executor: %s", mExecutor.getClass().getName())); - } - } - - @Override - public void shutdown() { - mExecutor.shutdown(); - } - - @Override - public List shutdownNow() { - return mExecutor.shutdownNow(); - } - - @Override - public boolean isShutdown() { - return mExecutor.isShutdown(); - } - - @Override - public boolean isTerminated() { - return mExecutor.isTerminated(); - } - - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - return mExecutor.awaitTermination(timeout, unit); - } - - @Override - public Future submit(Callable task) { - return mExecutor.submit(task); - } - - @Override - public Future submit(Runnable task, T result) { - return mExecutor.submit(task, result); - } - - @Override - public Future submit(Runnable task) { - return mExecutor.submit(task); - } - - @Override - public List> invokeAll(Collection> tasks) - throws InterruptedException { - return mExecutor.invokeAll(tasks); - } - - @Override - public List> invokeAll(Collection> tasks, long timeout, - TimeUnit unit) throws InterruptedException { - return mExecutor.invokeAll(tasks, timeout, unit); - } - - @Override - public T invokeAny(Collection> tasks) - throws InterruptedException, ExecutionException { - return null; - } - - @Override - public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { - return mExecutor.invokeAny(tasks, timeout, unit); - } - - @Override - public void execute(Runnable command) { - mExecutor.execute(command); - } -} diff --git a/core/server/common/src/main/java/alluxio/master/Master.java b/core/server/common/src/main/java/alluxio/master/Master.java deleted file mode 100644 index 1e475973c7df..000000000000 --- a/core/server/common/src/main/java/alluxio/master/Master.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master; - -import alluxio.Server; -import alluxio.exception.status.UnavailableException; -import alluxio.master.journal.JournalContext; -import alluxio.master.journal.Journaled; - -/** - * This interface contains common operations for all masters. - */ -public interface Master extends Journaled, Server { - /** - * @return a journal context for writing journal entries to the master - */ - JournalContext createJournalContext() throws UnavailableException; - - /** - * @return a master context - */ - MasterContext getMasterContext(); -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/JournalUpgrader.java b/core/server/common/src/main/java/alluxio/master/journal/JournalUpgrader.java deleted file mode 100644 index 15b40de40ddc..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/JournalUpgrader.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal; - -import alluxio.AlluxioURI; -import alluxio.RuntimeConstants; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.MasterFactory; -import alluxio.master.NoopMaster; -import alluxio.master.ServiceUtils; -import alluxio.master.journal.ufs.UfsJournal; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.UnderFileSystemConfiguration; -import alluxio.underfs.options.MkdirsOptions; -import alluxio.util.URIUtils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Tool to upgrade journal from v0 to v1. Format the v1 journal before running this tool. - * - * It is strongly recommended to backup the v0 journal before running this tool to avoid losing - * any data in case of failures. - * - * 
- * java -cp \
- *   assembly/server/target/alluxio-assembly-server-<ALLUXIO-VERSION>
- *   -jar-with-dependencies.jar \
- *   alluxio.master.journal.JournalUpgrader -journalDirectoryV0 YourJournalDirectoryV0
- *
- */ -@NotThreadSafe -public final class JournalUpgrader { - private static final Logger LOG = LoggerFactory.getLogger(JournalUpgrader.class); - - private static final int EXIT_FAILED = -1; - private static final int EXIT_SUCCEEDED = 0; - private static final Options OPTIONS = - new Options().addOption("help", false, "Show help for this test.") - .addOption("journalDirectoryV0", true, - "Where the v0 journal is persisted. It is assumed to be the same as the v1 journal " - + "directory if not set."); - - private static boolean sHelp; - private static String sJournalDirectoryV0; - - /** - * A class that provides a way to upgrade the journal from v0 to v1. - */ - private static final class Upgrader { - private final String mMaster; - private final alluxio.master.journalv0.MutableJournal mJournalV0; - private final UfsJournal mJournalV1; - - private final UnderFileSystem mUfs; - - private final URI mCheckpointV0; - private final URI mCompletedLogsV0; - - private final URI mCheckpointsV1; - private final URI mLogsV1; - - private final AlluxioConfiguration mAlluxioConf; - - private Upgrader(String master, AlluxioConfiguration alluxioConf) { - mMaster = master; - mAlluxioConf = alluxioConf; - mJournalV0 = (new alluxio.master.journalv0.MutableJournal.Factory( - getJournalLocation(sJournalDirectoryV0))).create(master); - mJournalV1 = - new UfsJournal(getJournalLocation(Configuration - .getString(PropertyKey.MASTER_JOURNAL_FOLDER)), new NoopMaster(master), 0, - Collections::emptySet); - - mUfs = UnderFileSystem.Factory.create(sJournalDirectoryV0, - UnderFileSystemConfiguration.defaults(alluxioConf)); - - mCheckpointV0 = URIUtils.appendPathOrDie(mJournalV0.getLocation(), "checkpoint.data"); - mCompletedLogsV0 = URIUtils.appendPathOrDie(mJournalV0.getLocation(), "completed"); - - mCheckpointsV1 = URIUtils.appendPathOrDie(mJournalV1.getLocation(), "checkpoints"); - mLogsV1 = URIUtils.appendPathOrDie(mJournalV1.getLocation(), "logs"); - } - - /** - * Upgrades journal from v0 to v1. - */ - void upgrade() throws IOException { - if (!mUfs.exists(mCheckpointV0.toString())) { - LOG.info("No checkpoint is found for {}. No upgrade is required.", mMaster); - return; - } - prepare(); - - LOG.info("Starting to upgrade {} journal.", mMaster); - boolean checkpointUpgraded = false; - int logNumber = 1; - URI completedLog; - while (mUfs.exists((completedLog = getCompletedLogV0(logNumber)).toString())) { - long start = -1; - long end = -1; - logNumber++; - try (JournalFileParser parser = JournalFileParser.Factory.create(completedLog)) { - alluxio.proto.journal.Journal.JournalEntry entry; - while ((entry = parser.next()) != null) { - if (start == -1) { - start = entry.getSequenceNumber(); - } - end = entry.getSequenceNumber(); - } - } - - if (!checkpointUpgraded) { - renameCheckpoint(start); - checkpointUpgraded = true; - } - - URI dst = URIUtils.appendPathOrDie(mLogsV1, String.format("0x%x-0x%x", start, end + 1)); - if (!mUfs.renameFile(completedLog.toString(), dst.toString()) && !mUfs - .exists(dst.toString())) { - throw new IOException( - String.format("Failed to rename %s to %s.", completedLog, dst)); - } - } - - if (!checkpointUpgraded) { - renameCheckpoint(1); - } - - LOG.info("Finished upgrading {} journal.", mMaster); - } - - /** - * Prepares journal writer to upgrade journals from v0 to v1. - */ - private void prepare() throws IOException { - alluxio.master.journalv0.JournalWriter journalWriterV0 = mJournalV0.getWriter(); - journalWriterV0.recover(); - journalWriterV0.completeLogs(); - journalWriterV0.close(); - - if (!mJournalV1.isFormatted()) { - LOG.info("Starting to format journal {}.", mJournalV1.getLocation()); - mJournalV1.format(); - LOG.info("Finished formatting journal {}.", mJournalV1.getLocation()); - } - - if (!mUfs.exists(mCheckpointsV1.toString())) { - mUfs.mkdirs(mCheckpointsV1.toString(), MkdirsOptions.defaults(mAlluxioConf) - .setCreateParent(true)); - } - if (!mUfs.exists(mLogsV1.toString())) { - mUfs.mkdirs(mLogsV1.toString(), MkdirsOptions.defaults(mAlluxioConf).setCreateParent(true)); - } - } - - /** - * Renames checkpoint. - * - * @param sequenceNumber the sequence number - */ - private void renameCheckpoint(long sequenceNumber) throws IOException { - URI dst = URIUtils.appendPathOrDie(mCheckpointsV1, String.format("0x0-0x%x", sequenceNumber)); - if (!mUfs.renameFile(mCheckpointV0.toString(), dst.toString()) && !mUfs - .exists(dst.toString())) { - throw new IOException( - String.format("Failed to rename %s to %s.", mCheckpointV0, dst)); - } - } - - /** - * @param logNumber the log number to get the path for - * @return the location of the completed log for a particular log number - */ - private URI getCompletedLogV0(long logNumber) { - return URIUtils - .appendPathOrDie(mCompletedLogsV0, String.format("%s.%020d", "log", logNumber)); - } - - /** - * @return the journal location - */ - private URI getJournalLocation(String journalDirectory) { - if (!journalDirectory.endsWith(AlluxioURI.SEPARATOR)) { - journalDirectory += AlluxioURI.SEPARATOR; - } - try { - return new URI(journalDirectory); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - } - - private JournalUpgrader() {} // prevent instantiation - - /** - * Reads a journal via - * {@code java -cp \ - * assembly/server/target/alluxio-assembly-server--jar-with-dependencies.jar \ - * alluxio.master.journal.JournalUpgrader -master BlockMaster}. - * - * @param args arguments passed to the tool - */ - public static void main(String[] args) { - if (!parseInputArgs(args)) { - usage(); - System.exit(EXIT_FAILED); - } - if (sHelp) { - usage(); - System.exit(EXIT_SUCCEEDED); - } - - List masters = new ArrayList<>(); - for (MasterFactory factory : ServiceUtils.getMasterServiceLoader()) { - masters.add(factory.getName()); - } - - for (String master : masters) { - Upgrader upgrader = new Upgrader(master, Configuration.global()); - try { - upgrader.upgrade(); - } catch (IOException e) { - LOG.error("Failed to upgrade the journal for {}.", master, e); - System.exit(EXIT_FAILED); - } - } - } - - /** - * Parses the input args with a command line format, using - * {@link org.apache.commons.cli.CommandLineParser}. - * - * @param args the input args - * @return true if parsing succeeded - */ - private static boolean parseInputArgs(String[] args) { - CommandLineParser parser = new DefaultParser(); - CommandLine cmd; - try { - cmd = parser.parse(OPTIONS, args); - } catch (ParseException e) { - System.out.println("Failed to parse input args: " + e); - return false; - } - sHelp = cmd.hasOption("help"); - sJournalDirectoryV0 = cmd.getOptionValue("journalDirectoryV0", - Configuration.getString(PropertyKey.MASTER_JOURNAL_FOLDER)); - return true; - } - - /** - * Prints the usage. - */ - private static void usage() { - new HelpFormatter().printHelp("java -cp alluxio-" + RuntimeConstants.VERSION - + "-jar-with-dependencies.jar alluxio.master.journal.JournalUpgrader", - "Upgrades journal from v0 to v1", OPTIONS, "", true); - } -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/checkpoint/Checkpointed.java b/core/server/common/src/main/java/alluxio/master/journal/checkpoint/Checkpointed.java deleted file mode 100644 index 7ba2f5098433..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/checkpoint/Checkpointed.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.checkpoint; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * Base class for Alluxio classes which can be written to and read from metadata checkpoints. - */ -public interface Checkpointed { - /** - * @return a name for this checkpointed class - */ - CheckpointName getCheckpointName(); - - /** - * Writes a checkpoint of all state to the given output stream. - * - * Implementations should make an effort to throw {@link InterruptedException} if they get - * interrupted while running. - * - * @param output the output stream to write to - */ - void writeToCheckpoint(OutputStream output) throws IOException, InterruptedException; - - /** - * Restores state from a checkpoint. - * - * @param input an input stream with checkpoint data - */ - void restoreFromCheckpoint(CheckpointInputStream input) throws IOException; -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/JournalStateMachine.java b/core/server/common/src/main/java/alluxio/master/journal/raft/JournalStateMachine.java deleted file mode 100644 index a96f69717490..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/raft/JournalStateMachine.java +++ /dev/null @@ -1,815 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.raft; - -import alluxio.Constants; -import alluxio.ProcessUtils; -import alluxio.annotation.SuppressFBWarnings; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.AddQuorumServerRequest; -import alluxio.grpc.JournalQueryRequest; -import alluxio.master.StateLockManager; -import alluxio.master.StateLockOptions; -import alluxio.master.journal.CatchupFuture; -import alluxio.master.journal.JournalUtils; -import alluxio.master.journal.Journaled; -import alluxio.master.journal.checkpoint.CheckpointInputStream; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proto.journal.Journal.JournalEntry; -import alluxio.resource.LockResource; -import alluxio.util.FormatUtils; -import alluxio.util.LogUtils; -import alluxio.util.StreamUtils; -import alluxio.util.logging.SamplingLogger; - -import com.codahale.metrics.Timer; -import com.google.common.base.Preconditions; -import org.apache.ratis.io.MD5Hash; -import org.apache.ratis.proto.RaftProtos; -import org.apache.ratis.protocol.Message; -import org.apache.ratis.protocol.RaftGroup; -import org.apache.ratis.protocol.RaftGroupId; -import org.apache.ratis.protocol.RaftGroupMemberId; -import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.server.RaftServer; -import org.apache.ratis.server.protocol.TermIndex; -import org.apache.ratis.server.raftlog.RaftLog; -import org.apache.ratis.server.storage.RaftStorage; -import org.apache.ratis.statemachine.SnapshotInfo; -import org.apache.ratis.statemachine.StateMachineStorage; -import org.apache.ratis.statemachine.TransactionContext; -import org.apache.ratis.statemachine.impl.BaseStateMachine; -import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage; -import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo; -import org.apache.ratis.util.LifeCycle; -import org.apache.ratis.util.MD5FileUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A state machine for managing all of Alluxio's journaled state. Entries applied to this state - * machine will be forwarded to the appropriate internal master. - * - * The state machine starts by resetting all state, then applying the entries offered by Ratis. - * When the master becomes primary, it should wait until the state machine is up to date and no - * other primary master is serving, then call {@link #upgrade}. Once the state machine is upgraded, - * it will ignore all entries appended by Ratis because those entries are applied to primary - * master state before being written to Rati. - * - */ -@ThreadSafe -public class JournalStateMachine extends BaseStateMachine { - private static final Logger LOG = LoggerFactory.getLogger(JournalStateMachine.class); - private static final Logger SAMPLING_LOG = new SamplingLogger(LOG, 10L * Constants.SECOND_MS); - - private static final CompletableFuture EMPTY_FUTURE = - CompletableFuture.completedFuture(Message.EMPTY); - - /** Journals managed by this applier. */ - private final Map mJournals; - private final RaftJournalSystem mJournalSystem; - private final SnapshotReplicationManager mSnapshotManager; - private final AtomicReference mStateLockManagerRef - = new AtomicReference<>(null); - @GuardedBy("this") - private boolean mIgnoreApplys = false; - @GuardedBy("this") - private boolean mClosed = false; - - private final Lock mGroupLock = new ReentrantLock(); - @GuardedBy("mGroupLock") - private boolean mServerClosing = false; - - private volatile long mLastAppliedCommitIndex = -1; - // The last special "primary start" sequence number applied to this state machine. These special - // sequence numbers are identified by being negative. - private volatile long mLastPrimaryStartSequenceNumber = 0; - private volatile long mNextSequenceNumberToRead = 0; - private volatile boolean mSnapshotting = false; - private volatile boolean mIsLeader = false; - - private final ExecutorService mJournalPool; - - /** - * This callback is used for interrupting someone who suspends the journal applier to work on - * the states. It helps prevent dirty read/write of the states when the journal is reloading. - * - * Here is an example of interrupting backup tasks when the state machine reloads: - * - * - Backup worker suspends state machine before backup, passing in the callback. - * - Backup worker writes journal entries to UFS. - * - Raft state machine downloads a new snapshot from leader. - * - Raft state machine transitions to PAUSE state and invokes the callback. - * - Backup worker handles the callback and interrupts the backup tasks. - * - Raft state machine starts reloading the states. - * - Raft state machine finished the reload and transitions back to RUNNING state. - */ - private volatile Runnable mInterruptCallback; - - // The last index of the latest journal snapshot - // created by this master or downloaded from other masters - private volatile long mSnapshotLastIndex = -1; - /** Used to control applying to masters. */ - private BufferedJournalApplier mJournalApplier; - private final SimpleStateMachineStorage mStorage = new SimpleStateMachineStorage(); - private RaftGroupId mRaftGroupId; - private RaftServer mServer; - private long mLastCheckPointTime = -1; - - /** - * @param journals master journals; these journals are still owned by the caller, not by the - * journal state machine - * @param journalSystem the raft journal system - */ - public JournalStateMachine(Map journals, RaftJournalSystem journalSystem) { - int maxConcurrencyPoolSize = - Configuration.getInt(PropertyKey.MASTER_JOURNAL_LOG_CONCURRENCY_MAX); - mJournalPool = new ForkJoinPool(maxConcurrencyPoolSize); - LOG.info("Ihe max concurrency for notifyTermIndexUpdated is loading with max threads {}", - maxConcurrencyPoolSize); - mJournals = journals; - mJournalApplier = new BufferedJournalApplier(journals, - () -> journalSystem.getJournalSinks(null)); - resetState(); - LOG.info("Initialized new journal state machine"); - mJournalSystem = journalSystem; - mSnapshotManager = new SnapshotReplicationManager(journalSystem, mStorage); - - MetricsSystem.registerGaugeIfAbsent( - MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_LAST_INDEX.getName(), - () -> mSnapshotLastIndex); - MetricsSystem.registerGaugeIfAbsent( - MetricKey.MASTER_JOURNAL_ENTRIES_SINCE_CHECKPOINT.getName(), - () -> getLastAppliedTermIndex().getIndex() - mSnapshotLastIndex); - MetricsSystem.registerGaugeIfAbsent( - MetricKey.MASTER_JOURNAL_LAST_CHECKPOINT_TIME.getName(), - () -> mLastCheckPointTime); - MetricsSystem.registerGaugeIfAbsent( - MetricKey.MASTER_JOURNAL_LAST_APPLIED_COMMIT_INDEX.getName(), - () -> mLastAppliedCommitIndex); - MetricsSystem.registerGaugeIfAbsent( - MetricKey.MASTER_JOURNAL_CHECKPOINT_WARN.getName(), - () -> getLastAppliedTermIndex().getIndex() - mSnapshotLastIndex - > Configuration.getInt(PropertyKey.MASTER_JOURNAL_CHECKPOINT_PERIOD_ENTRIES) - && System.currentTimeMillis() - mLastCheckPointTime > Configuration.getMs( - PropertyKey.MASTER_WEB_JOURNAL_CHECKPOINT_WARNING_THRESHOLD_TIME) - ); - } - - @Override - public void initialize(RaftServer server, RaftGroupId groupId, - RaftStorage raftStorage) throws IOException { - getLifeCycle().startAndTransition(() -> { - super.initialize(server, groupId, raftStorage); - mServer = server; - mRaftGroupId = groupId; - mStorage.init(raftStorage); - loadSnapshot(mStorage.getLatestSnapshot()); - synchronized (mSnapshotManager) { - mSnapshotManager.notifyAll(); - } - }); - } - - @Override - public void reinitialize() throws IOException { - LOG.info("Reinitializing state machine."); - mStorage.loadLatestSnapshot(); - loadSnapshot(mStorage.getLatestSnapshot()); - unpause(); - synchronized (mSnapshotManager) { - mSnapshotManager.notifyAll(); - } - } - - private synchronized void loadSnapshot(SingleFileSnapshotInfo snapshot) throws IOException { - if (snapshot == null) { - LOG.info("No snapshot to load"); - return; - } - LOG.info("Loading Snapshot {}", snapshot); - final File snapshotFile = snapshot.getFile().getPath().toFile(); - if (!snapshotFile.exists()) { - throw new FileNotFoundException( - String.format("The snapshot file %s does not exist", snapshotFile.getPath())); - } - try { - resetState(); - setLastAppliedTermIndex(snapshot.getTermIndex()); - install(snapshotFile); - mSnapshotLastIndex = getLatestSnapshot() != null ? getLatestSnapshot().getIndex() : -1; - synchronized (mSnapshotManager) { - mSnapshotManager.notifyAll(); - } - } catch (Exception e) { - throw new IOException(String.format("Failed to load snapshot %s", snapshot), e); - } - } - - /** - * Called by {@link RaftJournalSystem} stop internal method before - * shutting down the raft server to prevent a deadlock on - * the lock in RaftServerProxy. - */ - protected void setServerClosing() { - try (LockResource ignored = new LockResource(mGroupLock)) { - mServerClosing = true; - } - } - - /** - * Called by {@link RaftJournalSystem} stop internal method after - * shutting down the raft server. - */ - protected void afterServerClosing() { - try (LockResource ignored = new LockResource(mGroupLock)) { - mServerClosing = false; - } - } - - /** - * Allows leader to take snapshots. This is used exclusively for the - * `bin/alluxio fsadmin journal checkpoint` command. - * @param manager allows the state machine to take a snapshot as leader by using it - */ - void allowLeaderSnapshots(StateLockManager manager) { - mStateLockManagerRef.set(manager); - } - - void disallowLeaderSnapshots() { - mStateLockManagerRef.set(null); - } - - @Override - public long takeSnapshot() { - long index; - StateLockManager stateLockManager = mStateLockManagerRef.get(); - if (!mIsLeader) { - index = takeLocalSnapshot(false); - } else if (stateLockManager != null) { - // the leader has been allowed to take a local snapshot by being given a non-null - // StateLockManager through the #allowLeaderSnapshots method - try (LockResource stateLock = stateLockManager.lockExclusive(StateLockOptions.defaults())) { - index = takeLocalSnapshot(true); - } catch (Exception e) { - return RaftLog.INVALID_LOG_INDEX; - } - } else { - RaftGroup group; - try (LockResource ignored = new LockResource(mGroupLock)) { - if (mServerClosing) { - return RaftLog.INVALID_LOG_INDEX; - } - // These calls are protected by mGroupLock and mServerClosing - // as they will access the lock in RaftServerProxy.java - // which is also accessed during raft server shutdown which - // can cause a deadlock as the shutdown takes the lock while - // waiting for this thread to finish - Preconditions.checkState(mServer.getGroups().iterator().hasNext()); - group = mServer.getGroups().iterator().next(); - } catch (IOException e) { - SAMPLING_LOG.warn("Failed to get raft group info: {}", e.getMessage()); - return RaftLog.INVALID_LOG_INDEX; - } - if (group.getPeers().size() < 2) { - SAMPLING_LOG.warn("No follower to perform delegated snapshot. Please add more masters to " - + "the quorum or manually take snapshot using 'alluxio fsadmin journal checkpoint'"); - return RaftLog.INVALID_LOG_INDEX; - } else { - index = mSnapshotManager.maybeCopySnapshotFromFollower(); - } - } - // update metrics if took a snapshot - if (index != RaftLog.INVALID_LOG_INDEX) { - mSnapshotLastIndex = index; - mLastCheckPointTime = System.currentTimeMillis(); - } - return index; - } - - @Override - public SnapshotInfo getLatestSnapshot() { - return mStorage.getLatestSnapshot(); - } - - @Override - public StateMachineStorage getStateMachineStorage() { - return mStorage; - } - - @Override - public CompletableFuture query(Message request) { - CompletableFuture future = new CompletableFuture<>(); - try { - JournalQueryRequest queryRequest = JournalQueryRequest.parseFrom( - request.getContent().asReadOnlyByteBuffer()); - LOG.debug("Received query request: {}", queryRequest); - // give snapshot manager a chance to handle snapshot related requests - Message reply = mSnapshotManager.handleRequest(queryRequest); - if (reply != null) { - future.complete(reply); - return future; - } - // Snapshot manager returned null indicating the request is not handled. Check and handle - // other type of requests. - if (queryRequest.hasAddQuorumServerRequest()) { - AddQuorumServerRequest addRequest = queryRequest.getAddQuorumServerRequest(); - return CompletableFuture.supplyAsync(() -> { - try { - mJournalSystem.addQuorumServer(addRequest.getServerAddress()); - } catch (IOException e) { - throw new CompletionException(e); - } - return Message.EMPTY; - }); - } - } catch (Exception e) { - LOG.error("failed processing request {}", request, e); - future.completeExceptionally(e); - return future; - } - return super.query(request); - } - - @Override - public void close() { - mClosed = true; - synchronized (mSnapshotManager) { - mSnapshotManager.notifyAll(); - } - } - - @Override - public CompletableFuture applyTransaction(TransactionContext trx) { - try { - applyJournalEntryCommand(trx); - RaftProtos.LogEntryProto entry = Objects.requireNonNull(trx.getLogEntry()); - updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); - // explicitly return empty future since no response message is expected by the journal writer - // avoid using super.applyTransaction() since it will echo the message and add overhead - return EMPTY_FUTURE; - } catch (Exception e) { - return RaftJournalUtils.completeExceptionally(e); - } - } - - @Override - public void notifyNotLeader(Collection pendingEntries) { - mIsLeader = false; - mJournalSystem.notifyLeadershipStateChanged(false); - synchronized (mSnapshotManager) { - mSnapshotManager.notifyAll(); - } - } - - @Override - public void notifyConfigurationChanged(long term, long index, - RaftProtos.RaftConfigurationProto newRaftConfiguration) { - CompletableFuture.runAsync(mJournalSystem::updateGroup, mJournalPool); - } - - private long getNextIndex() { - try { - return mServer.getDivision(mRaftGroupId).getRaftLog().getNextIndex(); - } catch (IOException e) { - throw new IllegalStateException("Cannot obtain raft log index", e); - } - } - - @Override - public CompletableFuture notifyInstallSnapshotFromLeader( - RaftProtos.RoleInfoProto roleInfoProto, TermIndex firstTermIndexInLog) { - if (roleInfoProto.getRole() != RaftProtos.RaftPeerRole.FOLLOWER) { - return RaftJournalUtils.completeExceptionally( - new IllegalStateException(String.format( - "Server should be a follower when installing a snapshot from leader. Actual: %s", - roleInfoProto.getRole()))); - } - return mSnapshotManager.installSnapshotFromLeader().thenApply(snapshotIndex -> { - long latestJournalIndex = getNextIndex() - 1; - if (latestJournalIndex >= snapshotIndex.getIndex()) { - // do not reload the state machine if the downloaded snapshot is older than the latest entry - // fail the request after installation so the leader will stop sending the same request - throw new IllegalArgumentException( - String.format("Downloaded snapshot index %d is older than the latest entry index %d", - snapshotIndex.getIndex(), latestJournalIndex)); - } - mSnapshotLastIndex = snapshotIndex.getIndex(); - synchronized (mSnapshotManager) { - mSnapshotManager.notifyAll(); - } - return snapshotIndex; - }); - } - - @Override - public synchronized void pause() { - LOG.info("Pausing raft state machine."); - getLifeCycle().transition(LifeCycle.State.PAUSING); - if (mInterruptCallback != null) { - LOG.info("Invoking suspension interrupt callback."); - mInterruptCallback.run(); - mInterruptCallback = null; - } - try { - if (mJournalApplier.isSuspended()) { - // make sure there are no pending entries - LOG.info("Resuming journal applier."); - mJournalApplier.resume(); - } - } catch (IOException e) { - throw new IllegalStateException("State machine pause failed", e); - } - getLifeCycle().transition(LifeCycle.State.PAUSED); - LOG.info("Raft state machine is paused."); - } - - /** - * Unpause the StateMachine. This should be done after uploading new state to the StateMachine. - */ - public synchronized void unpause() { - LOG.info("Unpausing raft state machine."); - if (mJournalApplier.isSuspended()) { - LOG.warn("Journal should not be suspended while state machine is paused."); - } - getLifeCycle().startAndTransition(() -> { - // nothing to do - just use this method to transition from PAUSE to RUNNING state - }); - LOG.info("Raft state machine is unpaused."); - } - - /** - * Applies a journal entry commit to the state machine. - * @param commit the commit - */ - public synchronized void applyJournalEntryCommand(TransactionContext commit) { - JournalEntry entry; - try { - entry = JournalEntry.parseFrom( - commit.getStateMachineLogEntry().getLogData().asReadOnlyByteBuffer()); - } catch (Exception e) { - ProcessUtils.fatalError(LOG, e, - "Encountered invalid journal entry in commit: %s.", commit); - System.exit(-1); - throw new IllegalStateException(e); // We should never reach here. - } - try { - applyEntry(entry); - } finally { - Preconditions.checkState(commit.getLogEntry().getIndex() > mLastAppliedCommitIndex); - mLastAppliedCommitIndex = commit.getLogEntry().getIndex(); - } - } - - /** - * Applies the journal entry, ignoring empty entries and expanding multi-entries. - * - * @param entry the entry to apply - */ - private void applyEntry(JournalEntry entry) { - if (LOG.isDebugEnabled()) { - // This check is put behind the debug flag as the call to getAllFields creates - // a map and is very expensive - Preconditions.checkState( - entry.getAllFields().size() <= 2 - || (entry.getAllFields().size() == 3 && entry.hasSequenceNumber()), - "Raft journal entries should never set multiple fields in addition to sequence " - + "number, but found %s", - entry); - } - if (entry.getJournalEntriesCount() > 0) { - // This entry aggregates multiple entries. - for (JournalEntry e : entry.getJournalEntriesList()) { - applyEntry(e); - } - } else if (entry.getSequenceNumber() < 0) { - // Negative sequence numbers indicate special entries used to indicate that a new primary is - // starting to serve. - mLastPrimaryStartSequenceNumber = entry.getSequenceNumber(); - } else if (entry.toBuilder().clearSequenceNumber().build() - .equals(JournalEntry.getDefaultInstance())) { - // Ignore empty entries, they are created during snapshotting. - } else { - applySingleEntry(entry); - } - } - - @SuppressFBWarnings(value = "VO_VOLATILE_INCREMENT", - justification = "All calls to applyJournalEntryCommand() are synchronized by ratis") - private void applySingleEntry(JournalEntry entry) { - if (mClosed) { - return; - } - long newSN = entry.getSequenceNumber(); - if (newSN < mNextSequenceNumberToRead) { - // This can happen due to retried writes. For example, if flushing [3, 4] fails, we will - // retry, and the log may end up looking like [1, 2, 3, 4, 3, 4] if the original request - // eventually succeeds. Once we've read the first "4", we must ignore the next two entries. - LOG.debug("Ignoring duplicate journal entry with SN {} when next SN is {}", newSN, - mNextSequenceNumberToRead); - return; - } - if (newSN > mNextSequenceNumberToRead) { - ProcessUtils.fatalError(LOG, - "Unexpected journal entry. The next expected SN is %s, but" - + " encountered an entry with SN %s. Full journal entry: %s", - mNextSequenceNumberToRead, newSN, entry); - } - - mNextSequenceNumberToRead++; - if (!mIgnoreApplys) { - mJournalApplier.processJournalEntry(entry); - } - } - - /** - * Takes a snapshot of local state machine. - * @param hasStateLock indicates whether this method call is guarded by a state lock - * @return the index of last included entry, or {@link RaftLog#INVALID_LOG_INDEX} if it fails - */ - public synchronized long takeLocalSnapshot(boolean hasStateLock) { - // Snapshot format is [snapshotId, name1, bytes1, name2, bytes2, ...]. - if (mClosed) { - SAMPLING_LOG.info("Skip taking snapshot because state machine is closed."); - return RaftLog.INVALID_LOG_INDEX; - } - if (mServer.getLifeCycleState() != LifeCycle.State.RUNNING) { - SAMPLING_LOG.info("Skip taking snapshot because raft server is not in running state: " - + "current state is {}.", mServer.getLifeCycleState()); - return RaftLog.INVALID_LOG_INDEX; - } - if (mJournalApplier.isSuspended()) { - SAMPLING_LOG.info("Skip taking snapshot while journal application is suspended."); - return RaftLog.INVALID_LOG_INDEX; - } - // Recheck mIsLeader (even though it was checked in #takeSnapshot) because mIsLeader is volatile - // synchronized call to #isSnapshotting will prevent gaining leadership while this method is - // executing - if (mIsLeader && !hasStateLock) { - LOG.warn("Tried to take local snapshot as leader without state lock"); - return RaftLog.INVALID_LOG_INDEX; - } - LOG.debug("Calling snapshot"); - Preconditions.checkState(!mSnapshotting, "Cannot call snapshot multiple times concurrently"); - mSnapshotting = true; - try (Timer.Context ctx = MetricsSystem - .timer(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_GENERATE_TIMER.getName()).time()) { - // The start time of the most recent snapshot - long lastSnapshotStartTime = System.currentTimeMillis(); - long snapshotId = mNextSequenceNumberToRead - 1; - TermIndex last = getLastAppliedTermIndex(); - File tempFile; - try { - tempFile = RaftJournalUtils.createTempSnapshotFile(mStorage); - } catch (IOException e) { - LogUtils.warnWithException(LOG, "Failed to create temp snapshot file", e); - return RaftLog.INVALID_LOG_INDEX; - } - LOG.info("Taking a snapshot to file {}", tempFile); - final File snapshotFile = mStorage.getSnapshotFile(last.getTerm(), last.getIndex()); - try (DataOutputStream outputStream = new DataOutputStream(new FileOutputStream(tempFile))) { - outputStream.writeLong(snapshotId); - JournalUtils.writeToCheckpoint(outputStream, getStateMachines()); - } catch (Exception e) { - tempFile.delete(); - LogUtils.warnWithException(LOG, - "Failed to write snapshot {} to file {}", snapshotId, tempFile, e); - return RaftLog.INVALID_LOG_INDEX; - } - try { - final MD5Hash digest = MD5FileUtil.computeMd5ForFile(tempFile); - LOG.info("Saving digest for snapshot file {}", snapshotFile); - MD5FileUtil.saveMD5File(snapshotFile, digest); - LOG.info("Renaming a snapshot file {} to {}", tempFile, snapshotFile); - if (!tempFile.renameTo(snapshotFile)) { - tempFile.delete(); - LOG.warn("Failed to rename snapshot from {} to {}", tempFile, snapshotFile); - return RaftLog.INVALID_LOG_INDEX; - } - LOG.info("Completed snapshot with size {} up to SN {} in {}ms", - FormatUtils.getSizeFromBytes(snapshotFile.length()), - snapshotId, System.currentTimeMillis() - lastSnapshotStartTime); - } catch (Exception e) { - tempFile.delete(); - LogUtils.warnWithException(LOG, - "Failed to complete snapshot: {} - {}", snapshotId, snapshotFile, e); - return RaftLog.INVALID_LOG_INDEX; - } - try { - mStorage.loadLatestSnapshot(); - } catch (Exception e) { - snapshotFile.delete(); - LogUtils.warnWithException(LOG, "Failed to refresh latest snapshot: {}", snapshotId, e); - return RaftLog.INVALID_LOG_INDEX; - } - return last.getIndex(); - } finally { - mSnapshotting = false; - synchronized (mSnapshotManager) { - mSnapshotManager.notifyAll(); - } - } - } - - private void install(File snapshotFile) { - if (mClosed) { - return; - } - if (mIgnoreApplys) { - LOG.warn("Unexpected request to install a snapshot on a read-only journal state machine"); - return; - } - - long snapshotId = 0L; - try (Timer.Context ctx = MetricsSystem.timer(MetricKey - .MASTER_EMBEDDED_JOURNAL_SNAPSHOT_REPLAY_TIMER.getName()).time(); - DataInputStream stream = new DataInputStream(new FileInputStream(snapshotFile))) { - snapshotId = stream.readLong(); - JournalUtils.restoreFromCheckpoint(new CheckpointInputStream(stream), getStateMachines()); - } catch (Exception e) { - JournalUtils.handleJournalReplayFailure(LOG, e, "Failed to install snapshot: %s", snapshotId); - if (Configuration.getBoolean(PropertyKey.MASTER_JOURNAL_TOLERATE_CORRUPTION)) { - return; - } - } - - if (snapshotId < mNextSequenceNumberToRead - 1) { - LOG.warn("Installed snapshot for SN {} but next SN to read is {}", snapshotId, - mNextSequenceNumberToRead); - } - mNextSequenceNumberToRead = snapshotId + 1; - LOG.info("Successfully installed snapshot up to SN {}", snapshotId); - } - - /** - * Suspends applying to masters. - * - * When using suspend, the caller needs to provide a callback method as parameter. This callback - * is invoked when the journal needs to reload and thus cannot suspend the state changes any - * more. The callback should cancel any tasks that access the master states. After the callback - * returns, the journal assumes that the states is no longer being accessed and will reload - * immediately. - * - * @param interruptCallback a callback function to be called when the suspend is interrupted - * @throws IOException if suspension fails - */ - public synchronized void suspend(Runnable interruptCallback) throws IOException { - LOG.info("Suspending raft state machine."); - if (!getLifeCycleState().isRunning()) { - throw new UnavailableException("Cannot suspend journal when state machine is paused."); - } - mJournalApplier.suspend(); - mInterruptCallback = interruptCallback; - LOG.info("Raft state machine is suspended."); - } - - /** - * Resumes applying to masters. - * - * @throws IOException if resuming fails - */ - public synchronized void resume() throws IOException { - LOG.info("Resuming raft state machine"); - mInterruptCallback = null; - if (mJournalApplier.isSuspended()) { - mJournalApplier.resume(); - LOG.info("Raft state machine resumed"); - } else { - LOG.info("Raft state machine is already resumed"); - } - } - - /** - * Initiates catching up of masters to given sequence. - * - * @param sequence the target sequence - * @return the future to track when catching up is done - */ - public synchronized CatchupFuture catchup(long sequence) { - return mJournalApplier.catchup(sequence); - } - - private List getStateMachines() { - return StreamUtils.map(RaftJournal::getStateMachine, mJournals.values()); - } - - private synchronized void resetState() { - if (mClosed) { - return; - } - if (mIgnoreApplys) { - LOG.warn("Unexpected call to resetState() on a read-only journal state machine"); - return; - } - mJournalApplier.close(); - mJournalApplier = new BufferedJournalApplier(mJournals, - () -> mJournalSystem.getJournalSinks(null)); - for (RaftJournal journal : mJournals.values()) { - journal.getStateMachine().resetState(); - } - } - - /** - * Upgrades the journal state machine to primary mode. - * - * @return the last sequence number read while in standby mode - */ - public synchronized long upgrade() { - // Resume the journal applier if was suspended. - if (mJournalApplier.isSuspended()) { - try { - resume(); - } catch (IOException e) { - ProcessUtils.fatalError(LOG, e, "State-machine failed to catch up after suspension."); - } - } - mIgnoreApplys = true; - return mNextSequenceNumberToRead - 1; - } - - /** - * @return the sequence number of the last entry applied to the state machine - */ - public long getLastAppliedSequenceNumber() { - return mNextSequenceNumberToRead - 1; - } - - /** - * @return the last primary term start sequence number applied to this state machine - */ - public long getLastPrimaryStartSequenceNumber() { - return mLastPrimaryStartSequenceNumber; - } - - /** - * @return the last raft log index which was applied to the state machine - */ - public long getLastAppliedCommitIndex() { - return mLastAppliedCommitIndex; - } - - /** - * @return whether the state machine is in the process of taking a snapshot - */ - public synchronized boolean isSnapshotting() { - return mSnapshotting; - } - - @Override - public void notifyLeaderChanged(RaftGroupMemberId groupMemberId, RaftPeerId raftPeerId) { - if (mRaftGroupId == groupMemberId.getGroupId()) { - mIsLeader = groupMemberId.getPeerId() == raftPeerId; - mJournalSystem.notifyLeadershipStateChanged(mIsLeader); - } else { - LOG.warn("Received notification for unrecognized group {}, current group is {}", - groupMemberId.getGroupId(), mRaftGroupId); - } - } - - /** - * @return the snapshot replication manager - */ - public SnapshotReplicationManager getSnapshotReplicationManager() { - return mSnapshotManager; - } - - /** - * @return whether the journal is suspended - */ - public synchronized boolean isSuspended() { - return mJournalApplier.isSuspended(); - } -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalServiceClient.java b/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalServiceClient.java deleted file mode 100644 index 4531c847217f..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalServiceClient.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.raft; - -import alluxio.AbstractMasterClient; -import alluxio.Constants; -import alluxio.grpc.DownloadSnapshotPRequest; -import alluxio.grpc.DownloadSnapshotPResponse; -import alluxio.grpc.RaftJournalServiceGrpc; -import alluxio.grpc.ServiceType; -import alluxio.grpc.UploadSnapshotPRequest; -import alluxio.grpc.UploadSnapshotPResponse; -import alluxio.master.MasterClientContext; - -import io.grpc.stub.StreamObserver; - -/** - * A client for raft journal service. - */ -public class RaftJournalServiceClient extends AbstractMasterClient { - private RaftJournalServiceGrpc.RaftJournalServiceStub mClient = null; - - /** - * @param clientContext master client context - */ - public RaftJournalServiceClient(MasterClientContext clientContext) { - super(clientContext); - } - - @Override - protected ServiceType getRemoteServiceType() { - return ServiceType.RAFT_JOURNAL_SERVICE; - } - - @Override - protected String getServiceName() { - return Constants.RAFT_JOURNAL_SERVICE_NAME; - } - - @Override - protected long getServiceVersion() { - return Constants.RAFT_JOURNAL_SERVICE_VERSION; - } - - @Override - protected void afterConnect() { - mClient = RaftJournalServiceGrpc.newStub(mChannel); - } - - /** - * Uploads a snapshot. - * @param responseObserver the response stream observer - * @return the request stream observer - */ - public StreamObserver uploadSnapshot( - StreamObserver responseObserver) { - return mClient.uploadSnapshot(responseObserver); - } - - /** - * Downloads a snapshot. - * @param responseObserver the response stream observer - * @return the request stream observer - */ - public StreamObserver downloadSnapshot( - StreamObserver responseObserver) { - return mClient.downloadSnapshot(responseObserver); - } -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalServiceHandler.java b/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalServiceHandler.java deleted file mode 100644 index 9bc203a87ed5..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalServiceHandler.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.raft; - -import alluxio.grpc.DownloadSnapshotPRequest; -import alluxio.grpc.DownloadSnapshotPResponse; -import alluxio.grpc.RaftJournalServiceGrpc; -import alluxio.grpc.UploadSnapshotPRequest; -import alluxio.grpc.UploadSnapshotPResponse; - -import io.grpc.stub.StreamObserver; - -/** - * RPC handler for raft journal service. - */ -public class RaftJournalServiceHandler extends RaftJournalServiceGrpc.RaftJournalServiceImplBase { - - private final SnapshotReplicationManager mManager; - - /** - * @param manager the snapshot replication manager - */ - public RaftJournalServiceHandler(SnapshotReplicationManager manager) { - mManager = manager; - } - - @Override - public StreamObserver uploadSnapshot( - StreamObserver responseObserver) { - return mManager.receiveSnapshotFromFollower(responseObserver); - } - - @Override - public StreamObserver downloadSnapshot( - StreamObserver responseObserver) { - return mManager.sendSnapshotToFollower(responseObserver); - } -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalUtils.java b/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalUtils.java deleted file mode 100644 index ae46016840f4..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalUtils.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.raft; - -import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.concurrent.CompletableFuture; - -/** - * Helper class for raft journal operations. - */ -public class RaftJournalUtils { - public static final String RAFT_DIR = "raft"; - - private RaftJournalUtils() { - // prevent instantiation - } - - /** - * Gets the raft peer id. - * - * @param address the address of the server - * @return the raft peer id - */ - public static RaftPeerId getPeerId(InetSocketAddress address) { - return getPeerId(address.getHostString(), address.getPort()); - } - - /** - * Gets the raft peer id. - * - * @param host the hostname of the server - * @param port the port of the server - * @return the raft peer id - */ - public static RaftPeerId getPeerId(String host, int port) { - return RaftPeerId.getRaftPeerId(host + "_" + port); - } - - /** - * Gets the raft journal dir. - * - * @param baseDir the journal base dir - * @return the raft peer id - */ - public static File getRaftJournalDir(File baseDir) { - return new File(baseDir, RAFT_DIR); - } - - /** - * Creates a temporary snapshot file. - * - * @param storage the snapshot storage - * @return the temporary snapshot file - * @throws IOException if error occurred while creating the snapshot file - */ - public static File createTempSnapshotFile(SimpleStateMachineStorage storage) throws IOException { - File tempDir = new File(storage.getSmDir().getParentFile(), "tmp"); - if (!tempDir.isDirectory() && !tempDir.mkdir()) { - throw new IOException( - "Cannot create temporary snapshot directory at " + tempDir.getAbsolutePath()); - } - return File.createTempFile("raft_snapshot_" + System.currentTimeMillis() + "_", - ".dat", tempDir); - } - - /** - * Creates a future that is completed exceptionally. - * - * @param e the exception to be returned by the future - * @param the type of the future - * @return the completed future - */ - public static CompletableFuture completeExceptionally(Exception e) { - final CompletableFuture future = new CompletableFuture<>(); - future.completeExceptionally(e); - return future; - } -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotDownloader.java b/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotDownloader.java deleted file mode 100644 index a632aaef38c9..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotDownloader.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.raft; - -import alluxio.grpc.DownloadSnapshotPRequest; -import alluxio.grpc.DownloadSnapshotPResponse; -import alluxio.grpc.SnapshotData; -import alluxio.grpc.UploadSnapshotPRequest; -import alluxio.grpc.UploadSnapshotPResponse; - -import io.grpc.stub.ClientCallStreamObserver; -import io.grpc.stub.ClientResponseObserver; -import io.grpc.stub.StreamObserver; -import org.apache.ratis.io.MD5Hash; -import org.apache.ratis.server.protocol.TermIndex; -import org.apache.ratis.server.storage.FileInfo; -import org.apache.ratis.statemachine.SnapshotInfo; -import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage; -import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo; -import org.apache.ratis.util.MD5FileUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.concurrent.CompletableFuture; -import java.util.function.Function; - -/** - * A stream observer for downloading a snapshot. - * - * @param type of the message to send - * @param type of the message to receive - */ -public class SnapshotDownloader implements ClientResponseObserver { - private static final Logger LOG = LoggerFactory.getLogger(SnapshotDownloader.class); - - private final SimpleStateMachineStorage mStorage; - private final CompletableFuture mFuture = new CompletableFuture<>(); - private final Function mMessageBuilder; - private final Function mDataGetter; - private final String mSource; - - /** The term and index for the latest journal entry included in the snapshot. */ - private TermIndex mTermIndex; - private File mTempFile; - private FileOutputStream mOutputStream; - private long mBytesWritten = 0; - private StreamObserver mStream; - private SnapshotInfo mSnapshotToInstall; - - /** - * Builds a stream for leader to download a snapshot. - * - * @param storage the snapshot storage - * @param stream the response stream - * @param source the source of the snapshot - * @return the download stream for leader - */ - public static SnapshotDownloader forLeader( - SimpleStateMachineStorage storage, StreamObserver stream, - String source) { - return new SnapshotDownloader<>(storage, - offset -> UploadSnapshotPResponse.newBuilder().setOffsetReceived(offset).build(), - UploadSnapshotPRequest::getData, stream, source); - } - - /** - * Builds a stream for follower to download a snapshot. - * - * @param storage the snapshot storage - * @param source the source of the snapshot - * @return the download stream for follower - */ - public static SnapshotDownloader - forFollower(SimpleStateMachineStorage storage, String source) { - return new SnapshotDownloader<>(storage, - offset -> DownloadSnapshotPRequest.newBuilder().setOffsetReceived(offset).build(), - DownloadSnapshotPResponse::getData, null, source); - } - - private SnapshotDownloader(SimpleStateMachineStorage storage, Function messageBuilder, - Function dataGetter, StreamObserver stream, String source) { - mStorage = storage; - mMessageBuilder = messageBuilder; - mDataGetter = dataGetter; - mStream = stream; - mSource = source; - } - - @Override - public void onNext(R response) { - try { - onNextInternal(response); - } catch (Exception e) { - mStream.onError(e); - mFuture.completeExceptionally(e); - cleanup(); - } - } - - private void cleanup() { - if (mOutputStream != null) { - try { - mOutputStream.close(); - } catch (IOException ioException) { - LOG.error("Error closing snapshot file {}", mTempFile, ioException); - } - } - if (mTempFile != null && !mTempFile.delete()) { - LOG.error("Error deleting snapshot file {}", mTempFile.getPath()); - } - } - - private void onNextInternal(R response) throws IOException { - TermIndex termIndex = TermIndex.valueOf( - mDataGetter.apply(response).getSnapshotTerm(), - mDataGetter.apply(response).getSnapshotIndex()); - if (mTermIndex == null) { - LOG.info("Downloading new snapshot {} from {}", termIndex, mSource); - mTermIndex = termIndex; - // start a new file - mTempFile = RaftJournalUtils.createTempSnapshotFile(mStorage); - - mTempFile.deleteOnExit(); - mStream.onNext(mMessageBuilder.apply(0L)); - } else { - if (!termIndex.equals(mTermIndex)) { - throw new IOException(String.format( - "Mismatched term index when downloading the snapshot. expected: %s actual: %s", - mTermIndex, termIndex)); - } - if (!mDataGetter.apply(response).hasChunk()) { - throw new IOException(String.format( - "A chunk for file %s is missing from the response %s.", mTempFile, response)); - } - // write the chunk - if (mOutputStream == null) { - LOG.info("Start writing to temporary file {}", mTempFile.getPath()); - mOutputStream = new FileOutputStream(mTempFile); - } - long position = mOutputStream.getChannel().position(); - if (position != mDataGetter.apply(response).getOffset()) { - throw new IOException( - String.format("Mismatched offset in file %d, expect %d, bytes written %d", - position, mDataGetter.apply(response).getOffset(), mBytesWritten)); - } - mOutputStream.write(mDataGetter.apply(response).getChunk().toByteArray()); - mBytesWritten += mDataGetter.apply(response).getChunk().size(); - LOG.debug("Written {} bytes to snapshot file {}", mBytesWritten, mTempFile.getPath()); - if (mDataGetter.apply(response).getEof()) { - LOG.debug("Completed writing to temporary file {} with size {}", - mTempFile.getPath(), mOutputStream.getChannel().position()); - mOutputStream.close(); - mOutputStream = null; - final MD5Hash digest = MD5FileUtil.computeMd5ForFile(mTempFile); - mSnapshotToInstall = new SingleFileSnapshotInfo( - new FileInfo(mTempFile.toPath(), digest), - mTermIndex.getTerm(), mTermIndex.getIndex()); - mFuture.complete(mTermIndex); - LOG.info("Finished copying snapshot to local file {}.", mTempFile); - mStream.onCompleted(); - } else { - mStream.onNext(mMessageBuilder.apply(mBytesWritten)); - } - } - } - - @Override - public void onError(Throwable t) { - mFuture.completeExceptionally(t); - cleanup(); - } - - @Override - public void onCompleted() { - if (mOutputStream != null) { - mFuture.completeExceptionally( - new IllegalStateException("Request completed with unfinished upload")); - cleanup(); - } - } - - @Override - public void beforeStart(ClientCallStreamObserver requestStream) { - mStream = requestStream; - } - - /** - * @return a future that tracks when the stream is completed - */ - public CompletableFuture getFuture() { - return mFuture; - } - - /** - * @return the snapshot information if it is downloaded completely, or null otherwise - */ - public SnapshotInfo getSnapshotToInstall() { - return mSnapshotToInstall; - } -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotReplicationManager.java b/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotReplicationManager.java deleted file mode 100644 index d999d0494024..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotReplicationManager.java +++ /dev/null @@ -1,619 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.raft; - -import alluxio.ClientContext; -import alluxio.Constants; -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.AbortedException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.NotFoundException; -import alluxio.grpc.DownloadSnapshotPRequest; -import alluxio.grpc.DownloadSnapshotPResponse; -import alluxio.grpc.GetSnapshotInfoRequest; -import alluxio.grpc.GetSnapshotInfoResponse; -import alluxio.grpc.GetSnapshotRequest; -import alluxio.grpc.JournalQueryRequest; -import alluxio.grpc.JournalQueryResponse; -import alluxio.grpc.QuorumServerState; -import alluxio.grpc.SnapshotData; -import alluxio.grpc.SnapshotMetadata; -import alluxio.grpc.UploadSnapshotPRequest; -import alluxio.grpc.UploadSnapshotPResponse; -import alluxio.master.MasterClientContext; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.resource.LockResource; -import alluxio.security.authentication.ClientIpAddressInjector; -import alluxio.util.FormatUtils; -import alluxio.util.LogUtils; -import alluxio.util.logging.SamplingLogger; - -import com.codahale.metrics.Timer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.protobuf.MessageLite; -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import org.apache.ratis.protocol.Message; -import org.apache.ratis.protocol.RaftClientReply; -import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.server.protocol.TermIndex; -import org.apache.ratis.server.raftlog.RaftLog; -import org.apache.ratis.server.storage.FileInfo; -import org.apache.ratis.statemachine.SnapshotInfo; -import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage; -import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo; -import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; -import org.apache.ratis.util.MD5FileUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.time.Duration; -import java.time.Instant; -import java.util.Map; -import java.util.Objects; -import java.util.PriorityQueue; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Class for managing snapshot replication among masters. - * It manages two snapshot replication workflows - worker to master and master to worker. - * - * 1. Worker to Master - * When a raft leader needs a snapshot, instead of taking snapshot locally it copies a recent - * snapshot from one of the followers. - * - * Workflow: - * - * - Ratis calls leader state machine to take a snapshot - * - leader gets snapshot metadata from follower - * - leader pick one of the follower and send a request for copying the snapshot - * - follower receives the request and calls the leader raft journal service to upload the snapshot - * - after the upload completes, leader remembers the temporary snapshot location and index - * - Ratis calls the leader state machine again to take a snapshot - * - leader moves the temporary snapshot to the journal snapshot folder and returns snapshot index - * - * 2. Master to Worker - * When a raft follower receives a notification to download a snapshot, it downloads the latest - * snapshot from the leader. - * - * Workflow: - * - * - Ratis leader determines one of the follower needs a snapshot because it misses journal entries - * from a long time ago - * - Ratis leader notifies Ratis follower to install a snapshot from leader, the follower calls the - * Alluxio state machine to fulfill this request - * - the follower state machine calls the snapshot manager which calls the raft journal service from - * leader to download a snapshot - * - after the downloads completes, follower moves the file to snapshot directory and gives Ratis - * the snapshot index - */ -public class SnapshotReplicationManager { - private static final Logger LOG = LoggerFactory.getLogger(SnapshotReplicationManager.class); - private static final Logger SAMPLING_LOG = new SamplingLogger(LOG, 5L * Constants.SECOND_MS); - - private final SimpleStateMachineStorage mStorage; - private final RaftJournalSystem mJournalSystem; - private volatile SnapshotInfo mDownloadedSnapshot; - private final PriorityQueue> mSnapshotCandidates; - private Future mRequestDataFuture; - private final Lock mRequestDataLock = new ReentrantLock(); - private final Condition mRequestDataCondition = mRequestDataLock.newCondition(); - private final ExecutorService mRequestDataExecutor = Executors.newSingleThreadExecutor(); - - private static final long SNAPSHOT_INFO_TIMEOUT_MS = - Configuration.getMs(PropertyKey.MASTER_JOURNAL_REQUEST_INFO_TIMEOUT); - private static final long SNAPSHOT_DATA_TIMEOUT_MS = - Configuration.getMs(PropertyKey.MASTER_JOURNAL_REQUEST_DATA_TIMEOUT); - - private enum DownloadState { - /** No snapshot download is in progress. */ - IDLE, - - /** Snapshot information is requested from available followers. */ - REQUEST_INFO, - - /** The latest snapshot data is requested from one of the followers. */ - REQUEST_DATA, - - /** The latest snapshot is being downloaded from one of the followers. */ - STREAM_DATA, - - /** A snapshot is downloaded and ready for installation. */ - DOWNLOADED, - - /** A snapshot is being installed to the journal storage. */ - INSTALLING, - } - - private final AtomicReference mDownloadState = - new AtomicReference<>(DownloadState.IDLE); - - /** - * @param journalSystem the raft journal system - * @param storage the snapshot storage - */ - public SnapshotReplicationManager(RaftJournalSystem journalSystem, - SimpleStateMachineStorage storage) { - mStorage = storage; - mJournalSystem = journalSystem; - mSnapshotCandidates = new PriorityQueue<>((pair1, pair2) -> { - SnapshotMetadata first = pair1.getFirst(); - SnapshotMetadata second = pair2.getFirst(); - // deliberately reversing the compare order to have bigger numbers rise to the top - // bigger terms and indexes means a more recent snapshot - if (first.getSnapshotTerm() == second.getSnapshotTerm()) { - return Long.compare(second.getSnapshotIndex(), first.getSnapshotIndex()); - } - return Long.compare(second.getSnapshotTerm(), first.getSnapshotTerm()); - }); - } - - /** - * Downloads and installs a snapshot from the leader. - * - * @return a future with the term index of the installed snapshot - */ - public CompletableFuture installSnapshotFromLeader() { - if (mJournalSystem.isLeader()) { - return RaftJournalUtils.completeExceptionally( - new IllegalStateException("Abort snapshot installation after becoming a leader")); - } - if (!transitionState(DownloadState.IDLE, DownloadState.STREAM_DATA)) { - return RaftJournalUtils.completeExceptionally( - new IllegalStateException("State is not IDLE when starting a snapshot installation")); - } - try { - RaftJournalServiceClient client = createJournalServiceClient(); - String address = String.valueOf(client.getRemoteSockAddress()); - SnapshotDownloader observer = - SnapshotDownloader.forFollower(mStorage, address); - Timer.Context ctx = MetricsSystem - .timer(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_DOWNLOAD_TIMER.getName()).time(); - client.downloadSnapshot(observer); - return observer.getFuture().thenApplyAsync((termIndex) -> { - ctx.close(); - mDownloadedSnapshot = observer.getSnapshotToInstall(); - transitionState(DownloadState.STREAM_DATA, DownloadState.DOWNLOADED); - long index = installDownloadedSnapshot(); - if (index == RaftLog.INVALID_LOG_INDEX) { - throw new CompletionException(new RuntimeException( - String.format("Failed to install the downloaded snapshot %s", termIndex))); - } - if (index != termIndex.getIndex()) { - throw new CompletionException(new IllegalStateException( - String.format("Mismatched snapshot installed - downloaded %d, installed %d", - termIndex.getIndex(), index))); - } - return termIndex; - }).whenComplete((termIndex, throwable) -> { - if (throwable != null) { - LOG.error("Unexpected exception downloading snapshot from leader {}.", address, - throwable); - transitionState(DownloadState.STREAM_DATA, DownloadState.IDLE); - } - client.close(); - }); - } catch (Exception e) { - transitionState(DownloadState.STREAM_DATA, DownloadState.IDLE); - return RaftJournalUtils.completeExceptionally(e); - } - } - - /** - * Sends a snapshot to the leader. - * - * @throws IOException if error occurs while initializing the data stream - */ - public void sendSnapshotToLeader() throws IOException { - if (mJournalSystem.isLeader()) { - throw new IllegalStateException("Server is no longer a follower"); - } - LOG.debug("Checking latest snapshot to send"); - SnapshotInfo snapshot = mStorage.getLatestSnapshot(); - if (snapshot == null) { - throw new NotFoundException("No snapshot available"); - } - - SnapshotUploader snapshotUploader = - SnapshotUploader.forFollower(mStorage, snapshot); - RaftJournalServiceClient client = createJournalServiceClient(); - LOG.info("Sending stream request to leader {} for snapshot {}", client.getRemoteSockAddress(), - snapshot.getTermIndex()); - StreamObserver requestObserver = - client.uploadSnapshot(snapshotUploader); - requestObserver.onNext(UploadSnapshotPRequest.newBuilder() - .setData(SnapshotData.newBuilder() - .setSnapshotTerm(snapshot.getTerm()) - .setSnapshotIndex(snapshot.getIndex()) - .setOffset(0)) - .build()); - snapshotUploader.getCompletionFuture().whenComplete((info, t) -> client.close()); - } - - /** - * Attempts to copy a snapshot from one of the followers. - * - * The leader state machine calls this method regularly when it needs a new snapshot. - * To avoid blocking normal journal operations, This method always returns a value immediately - * without waiting for download to finish: - * - * - If no download is in progress, it schedules a new download asynchronously and returns - * {@link RaftLog#INVALID_LOG_INDEX}. - * - If a download is in progress, it returns {@link RaftLog#INVALID_LOG_INDEX} immediately. - * - If a download is completed, it moves the downloaded file to the snapshot directory and - * returns the snapshot index. - * - * @return the index of the downloaded snapshot, or {@link RaftLog#INVALID_LOG_INDEX} - * if no snapshot is installed. - */ - public long maybeCopySnapshotFromFollower() { - if (mDownloadState.get() == DownloadState.DOWNLOADED) { - return installDownloadedSnapshot(); - } - SAMPLING_LOG.info("Call copy snapshot from follower in state {}", mDownloadState.get()); - if (mDownloadState.get() == DownloadState.IDLE) { - CompletableFuture.runAsync(this::requestSnapshotFromFollowers); - } - return RaftLog.INVALID_LOG_INDEX; - } - - /** - * Receives a snapshot from follower. - * - * @param responseStreamObserver the response stream observer - * @return the request stream observer - */ - public StreamObserver receiveSnapshotFromFollower( - StreamObserver responseStreamObserver) { - String followerIp = ClientIpAddressInjector.getIpAddress(); - LOG.info("Received upload snapshot request from follower {}", followerIp); - - SnapshotDownloader observer = - SnapshotDownloader.forLeader(mStorage, responseStreamObserver, - followerIp); - if (!transitionState(DownloadState.REQUEST_DATA, DownloadState.STREAM_DATA)) { - responseStreamObserver.onCompleted(); - return observer; - } - observer.getFuture() - .thenApply(termIndex -> { - try (LockResource ignored = new LockResource(mRequestDataLock)) { - mDownloadedSnapshot = observer.getSnapshotToInstall(); - transitionState(DownloadState.STREAM_DATA, DownloadState.DOWNLOADED); - // Cancel any pending data requests since the download was successful - mRequestDataFuture.cancel(true); - mRequestDataCondition.signalAll(); - return termIndex; - } - }).exceptionally(e -> { - try (LockResource ignored = new LockResource(mRequestDataLock)) { - LOG.error("Unexpected exception downloading snapshot from follower {}.", followerIp, e); - // this allows the leading master to request other followers for their snapshots. It - // previously collected information about other snapshots in requestInfo(). If no other - // snapshots are available requestData() will return false and mDownloadState will be - // IDLE - transitionState(DownloadState.STREAM_DATA, DownloadState.REQUEST_DATA); - // Notify the request data tasks to start a request with a new candidate - mRequestDataCondition.signalAll(); - return null; - } - }); - return observer; - } - - /** - * Handles snapshot requests. - * - * @param queryRequest the query request - * @return the response message, or null if the request is not handled - * @throws IOException if any error occurred while handling the request - */ - public Message handleRequest(JournalQueryRequest queryRequest) throws IOException { - if (queryRequest.hasSnapshotInfoRequest()) { - SnapshotMetadata requestSnapshot = queryRequest.getSnapshotInfoRequest().getSnapshotInfo(); - Instant start = Instant.now(); - SnapshotInfo latestSnapshot = mStorage.getLatestSnapshot(); - synchronized (this) { - // We may need to wait for a valid snapshot to be ready - while ((latestSnapshot == null - || (queryRequest.getSnapshotInfoRequest().hasSnapshotInfo() - && (requestSnapshot.getSnapshotTerm() > latestSnapshot.getTerm() - || (requestSnapshot.getSnapshotTerm() == latestSnapshot.getTerm() - && requestSnapshot.getSnapshotIndex() >= latestSnapshot.getIndex())))) - && Duration.between(start, Instant.now()).toMillis() < SNAPSHOT_INFO_TIMEOUT_MS) { - LOG.info("Received snapshot info request from leader - {}, but do not have a " - + "snapshot ready - {}", requestSnapshot, latestSnapshot); - try { - wait(SNAPSHOT_DATA_TIMEOUT_MS - Long.min(SNAPSHOT_DATA_TIMEOUT_MS, - Math.abs(Duration.between(start, Instant.now()).toMillis()))); - } catch (InterruptedException e) { - LOG.debug("Interrupted while waiting for snapshot", e); - break; - } - latestSnapshot = mStorage.getLatestSnapshot(); - } - } - if (latestSnapshot == null) { - LOG.debug("No snapshot to send"); - return toMessage(GetSnapshotInfoResponse.getDefaultInstance()); - } - JournalQueryResponse response = JournalQueryResponse.newBuilder() - .setSnapshotInfoResponse(GetSnapshotInfoResponse.newBuilder().setLatest( - toSnapshotMetadata(latestSnapshot.getTermIndex()))) - .build(); - LOG.info("Sent snapshot info response to leader {}", response); - return toMessage(response); - } - if (queryRequest.hasSnapshotRequest()) { - LOG.info("Start sending snapshot to leader"); - sendSnapshotToLeader(); - return Message.EMPTY; - } - return null; - } - - /** - * Sends a snapshot to a follower. - * - * @param responseObserver the response stream observer - * @return the request stream observer - */ - public StreamObserver sendSnapshotToFollower( - StreamObserver responseObserver) { - SnapshotInfo snapshot = mStorage.getLatestSnapshot(); - LOG.debug("Received snapshot download request from {}", ClientIpAddressInjector.getIpAddress()); - SnapshotUploader requestStreamObserver = - SnapshotUploader.forLeader(mStorage, snapshot, responseObserver); - if (snapshot == null) { - responseObserver.onError(Status.NOT_FOUND - .withDescription("Cannot find a valid snapshot to download.") - .asException()); - return requestStreamObserver; - } - responseObserver.onNext(DownloadSnapshotPResponse.newBuilder() - .setData(SnapshotData.newBuilder() - .setSnapshotTerm(snapshot.getTerm()) - .setSnapshotIndex(snapshot.getIndex()) - .setOffset(0)) - .build()); - return requestStreamObserver; - } - - private static Message toMessage(MessageLite value) { - return Message.valueOf( - UnsafeByteOperations.unsafeWrap(value.toByteString().asReadOnlyByteBuffer())); - } - - private SnapshotMetadata toSnapshotMetadata(TermIndex value) { - return value == null ? null : - SnapshotMetadata.newBuilder() - .setSnapshotTerm(value.getTerm()) - .setSnapshotIndex(value.getIndex()) - .build(); - } - - private boolean transitionState(DownloadState expected, DownloadState update) { - if (!mDownloadState.compareAndSet(expected, update)) { - LOG.warn("Failed to transition from {} to {}: current state is {}", - expected, update, mDownloadState.get()); - return false; - } - LOG.debug("Successfully transitioned from {} to {}", expected, update); - return true; - } - - /** - * Installs a downloaded snapshot in the journal snapshot directory. - * - * @return the index of the installed snapshot - */ - private long installDownloadedSnapshot() { - LOG.info("Call install downloaded snapshot"); - if (!transitionState(DownloadState.DOWNLOADED, DownloadState.INSTALLING)) { - return RaftLog.INVALID_LOG_INDEX; - } - File tempFile = null; - try (Timer.Context ctx = MetricsSystem - .timer(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_INSTALL_TIMER.getName()).time()) { - SnapshotInfo snapshot = mDownloadedSnapshot; - if (snapshot == null) { - throw new IllegalStateException("Snapshot is not completed"); - } - FileInfo fileInfo = snapshot.getFiles().get(0); - tempFile = fileInfo.getPath().toFile(); - if (!tempFile.exists()) { - throw new FileNotFoundException(String.format("Snapshot file %s is not found", tempFile)); - } - SnapshotInfo latestSnapshot = mStorage.getLatestSnapshot(); - TermIndex lastInstalled = latestSnapshot == null ? null : latestSnapshot.getTermIndex(); - TermIndex downloaded = snapshot.getTermIndex(); - if (lastInstalled != null && downloaded.compareTo(lastInstalled) < 0) { - throw new AbortedException( - String.format("Snapshot to be installed %s is older than current snapshot %s", - downloaded, lastInstalled)); - } - final File snapshotFile = mStorage.getSnapshotFile( - downloaded.getTerm(), downloaded.getIndex()); - LOG.debug("Moving temp snapshot {} to file {}", tempFile, snapshotFile); - MD5FileUtil.saveMD5File(snapshotFile, fileInfo.getFileDigest()); - if (!tempFile.renameTo(snapshotFile)) { - throw new IOException(String.format("Failed to rename %s to %s", tempFile, snapshotFile)); - } - synchronized (this) { - mStorage.loadLatestSnapshot(); - notifyAll(); - } - LOG.info("Completed storing snapshot at {} to file {} with size {}", downloaded, - snapshotFile, FormatUtils.getSizeFromBytes(snapshotFile.length())); - return downloaded.getIndex(); - } catch (Exception e) { - LOG.error("Failed to install snapshot", e); - if (tempFile != null) { - tempFile.delete(); - } - return RaftLog.INVALID_LOG_INDEX; - } finally { - transitionState(DownloadState.INSTALLING, DownloadState.IDLE); - } - } - - /** - * Finds a follower with the latest snapshot and sends a request to download it. - */ - private void requestSnapshotFromFollowers() { - if (mDownloadState.get() == DownloadState.IDLE) { - if (!transitionState(DownloadState.IDLE, DownloadState.REQUEST_INFO)) { - return; - } - // we want fresh info not polluted by older requests. This ensures that requestData() requests - // from at most # followers before requesting new info. Otherwise, the candidate queue might - // grow indefinitely. - mSnapshotCandidates.clear(); - requestInfo(); - transitionState(DownloadState.REQUEST_INFO, DownloadState.REQUEST_DATA); - mRequestDataFuture = mRequestDataExecutor.submit(this::requestData, null); - } - } - - private void requestInfo() { - Preconditions.checkState(mDownloadState.get() == DownloadState.REQUEST_INFO); - try { - LOG.info("Call request snapshot info from followers"); - SingleFileSnapshotInfo latestSnapshot = mStorage.getLatestSnapshot(); - SnapshotMetadata snapshotMetadata = latestSnapshot == null ? null : - SnapshotMetadata.newBuilder() - .setSnapshotTerm(latestSnapshot.getTerm()) - .setSnapshotIndex(latestSnapshot.getIndex()) - .build(); - // build SnapshotInfoRequests - GetSnapshotInfoRequest infoRequest; - if (snapshotMetadata == null) { - infoRequest = GetSnapshotInfoRequest.getDefaultInstance(); - } else { - infoRequest = GetSnapshotInfoRequest.newBuilder() - .setSnapshotInfo(snapshotMetadata).build(); - } - Map> jobs = mJournalSystem - .getQuorumServerInfoList() - .stream() - .filter(server -> server.getServerState() == QuorumServerState.AVAILABLE) - .map(server -> RaftJournalUtils.getPeerId( - server.getServerAddress().getHost(), - server.getServerAddress().getRpcPort())) - .filter(peerId -> !peerId.equals(mJournalSystem.getLocalPeerId())) - .collect(Collectors.toMap(Function.identity(), - peerId -> mJournalSystem.sendMessageAsync(peerId, toMessage(JournalQueryRequest - .newBuilder() - .setSnapshotInfoRequest(infoRequest) - .build()), SNAPSHOT_INFO_TIMEOUT_MS))); - // query all secondary masters for information about their latest snapshot - for (Map.Entry> job : jobs.entrySet()) { - RaftPeerId peerId = job.getKey(); - try { - RaftClientReply reply = job.getValue().get(); - if (reply.getException() != null) { - throw reply.getException(); - } - JournalQueryResponse response = JournalQueryResponse.parseFrom( - reply.getMessage().getContent().asReadOnlyByteBuffer()); - if (!response.hasSnapshotInfoResponse()) { - throw new IOException("Invalid response for GetSnapshotInfoRequest " + response); - } - SnapshotMetadata latest = response.getSnapshotInfoResponse().getLatest(); - LOG.info("Received snapshot info from follower {} - {}, my current snapshot is {}", - peerId, latest, snapshotMetadata); - if (snapshotMetadata == null - || (latest.getSnapshotTerm() >= snapshotMetadata.getSnapshotTerm()) - && latest.getSnapshotIndex() > snapshotMetadata.getSnapshotIndex()) { - mSnapshotCandidates.add(new Pair<>(latest, peerId)); - } - } catch (Exception e) { - LOG.warn("Error while requesting snapshot info from {}: {}", peerId, e.toString()); - } - } - } catch (Exception e) { - LogUtils.warnWithException(LOG, "Failed to request snapshot info from followers", e); - } - } - - private void requestData() { - Preconditions.checkState(mDownloadState.get() == DownloadState.REQUEST_DATA); - // request snapshots from the most recent to the least recent - try { - while (!mSnapshotCandidates.isEmpty() && mDownloadState.get() == DownloadState.REQUEST_DATA) { - Pair candidate = mSnapshotCandidates.poll(); - SnapshotMetadata metadata = Objects.requireNonNull(candidate).getFirst(); - RaftPeerId peerId = candidate.getSecond(); - LOG.info("Request data from follower {} for snapshot (t: {}, i: {})", - peerId, metadata.getSnapshotTerm(), metadata.getSnapshotIndex()); - try { - RaftClientReply reply = mJournalSystem.sendMessageAsync(peerId, - toMessage(JournalQueryRequest.newBuilder() - .setSnapshotRequest(GetSnapshotRequest.getDefaultInstance()).build())) - .get(); - if (reply.getException() != null) { - throw reply.getException(); - } - // Wait a timeout before trying the next follower, or until we are awoken - try (LockResource ignored = new LockResource(mRequestDataLock)) { - do { - mRequestDataCondition.await(SNAPSHOT_DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } while (mDownloadState.get() != DownloadState.REQUEST_DATA); - } - } catch (InterruptedException | CancellationException ignored) { - // We are usually interrupted when a snapshot transfer is complete, - // so we can just return without trying a new candidate. - // It is fine even if we are interrupted in other cases as - // a new request info will be initiated by the next takeSnapshot() call. - return; - } catch (Exception e) { - LOG.warn("Failed to request snapshot data from {}: {}", peerId, e); - } - } - } finally { - // Ensure that we return to the IDLE state in case the REQUEST_DATA operations - // were not successful, for example if we were interrupted for some reason - // other than a successful download. - if (mDownloadState.get() == DownloadState.REQUEST_DATA) { - transitionState(DownloadState.REQUEST_DATA, DownloadState.IDLE); - } - } - } - - @VisibleForTesting - synchronized RaftJournalServiceClient createJournalServiceClient() - throws AlluxioStatusException { - RaftJournalServiceClient client = new RaftJournalServiceClient(MasterClientContext - .newBuilder(ClientContext.create(Configuration.global())).build()); - client.connect(); - return client; - } -} diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotUploader.java b/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotUploader.java deleted file mode 100644 index 727f6a288b7c..000000000000 --- a/core/server/common/src/main/java/alluxio/master/journal/raft/SnapshotUploader.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.raft; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.InvalidArgumentException; -import alluxio.grpc.DownloadSnapshotPRequest; -import alluxio.grpc.DownloadSnapshotPResponse; -import alluxio.grpc.SnapshotData; -import alluxio.grpc.UploadSnapshotPRequest; -import alluxio.grpc.UploadSnapshotPResponse; - -import com.google.protobuf.UnsafeByteOperations; -import io.grpc.stub.ClientCallStreamObserver; -import io.grpc.stub.ClientResponseObserver; -import io.grpc.stub.StreamObserver; -import org.apache.commons.io.IOUtils; -import org.apache.ratis.statemachine.SnapshotInfo; -import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.CompletableFuture; -import java.util.function.Function; - -/** - * A stream observer for uploading a snapshot. - * - * @param the message type to send - * @param the message type to receive - */ -public class SnapshotUploader - implements StreamObserver, ClientResponseObserver { - private static final Logger LOG = LoggerFactory.getLogger(SnapshotUploader.class); - private static final int SNAPSHOT_CHUNK_SIZE = (int) Configuration.getBytes( - PropertyKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_REPLICATION_CHUNK_SIZE); - - private final Function mDataMessageBuilder; - private final Function mOffsetGetter; - private final File mSnapshotFile; - private final long mLength; - private final SnapshotInfo mSnapshotInfo; - private long mOffset = 0; - private StreamObserver mStream; - private final CompletableFuture mCompletionFuture = new CompletableFuture<>(); - - /** - * Builds a stream for leader to upload a snapshot. - * - * @param storage the snapshot storage - * @param snapshot the snapshot to upload - * @param stream the download stream - * @return the upload stream for leader - */ - public static SnapshotUploader forLeader( - SimpleStateMachineStorage storage, SnapshotInfo snapshot, - StreamObserver stream) { - return new SnapshotUploader<>(storage, snapshot, stream, - data -> DownloadSnapshotPResponse.getDefaultInstance().toBuilder().setData(data).build(), - DownloadSnapshotPRequest::getOffsetReceived); - } - - /** - * Builds a stream for follower to upload a snapshot. - * - * @param storage the snapshot storage - * @param snapshot the snapshot to upload - * @return the upload stream for follower - */ - public static SnapshotUploader forFollower( - SimpleStateMachineStorage storage, SnapshotInfo snapshot) { - return new SnapshotUploader<>(storage, snapshot, null, - data -> UploadSnapshotPRequest.getDefaultInstance().toBuilder().setData(data).build(), - UploadSnapshotPResponse::getOffsetReceived); - } - - private SnapshotUploader(SimpleStateMachineStorage storage, SnapshotInfo snapshot, - StreamObserver stream, - Function buildFunc, Function offsetGetter) { - mSnapshotInfo = snapshot; - mDataMessageBuilder = buildFunc; - mOffsetGetter = offsetGetter; - mSnapshotFile = storage.getSnapshotFile(snapshot.getTerm(), snapshot.getIndex()); - mLength = mSnapshotFile.length(); - mStream = stream; - } - - @Override - public void onNext(R value) { - try { - onNextInternal(value); - } catch (Exception e) { - LOG.error("Error occurred while sending snapshot", e); - mStream.onError(e); - } - } - - private void onNextInternal(R value) throws IOException { - LOG.debug("Received request {}", value); - if (mStream == null) { - throw new IllegalStateException("No request stream assigned"); - } - if (!mSnapshotFile.exists()) { - throw new FileNotFoundException( - String.format("Snapshot file %s does not exist", mSnapshotFile.getPath())); - } - long offsetReceived = mOffsetGetter.apply(value); - // TODO(feng): implement better flow control - if (mOffset != offsetReceived) { - throw new InvalidArgumentException( - String.format("Received mismatched offset: %d. Expect %d", offsetReceived, mOffset)); - } - LOG.debug("Streaming data at {}", mOffset); - try (InputStream is = new FileInputStream(mSnapshotFile)) { - is.skip(mOffset); - boolean eof = false; - int chunkSize = SNAPSHOT_CHUNK_SIZE; - long available = mLength - mOffset; - if (available <= SNAPSHOT_CHUNK_SIZE) { - eof = true; - chunkSize = (int) available; - } - byte[] buffer = new byte[chunkSize]; - IOUtils.readFully(is, buffer); - LOG.debug("Read {} bytes from file {}", chunkSize, mSnapshotFile); - mStream.onNext(mDataMessageBuilder.apply(SnapshotData.newBuilder() - .setOffset(mOffset) - .setEof(eof) - .setChunk(UnsafeByteOperations.unsafeWrap(buffer)) - .setSnapshotTerm(mSnapshotInfo.getTerm()) - .setSnapshotIndex(mSnapshotInfo.getIndex()) - .build())); - mOffset += chunkSize; - LOG.debug("Uploaded total {} bytes of file {}", mOffset, mSnapshotFile); - } - } - - @Override - public void onError(Throwable t) { - LOG.error("Error sending snapshot {} at {}", mSnapshotFile, mOffset, t); - mStream.onError(t); - mCompletionFuture.completeExceptionally(t); - } - - @Override - public void onCompleted() { - LOG.debug("Received onComplete for {}", mSnapshotInfo); - mStream.onCompleted(); - mCompletionFuture.complete(mSnapshotInfo); - } - - /** - * @return a future used to propagate completion status to {@link SnapshotReplicationManager} - */ - public CompletableFuture getCompletionFuture() { - return mCompletionFuture; - } - - @Override - public void beforeStart(ClientCallStreamObserver requestStream) { - mStream = requestStream; - } -} diff --git a/core/server/common/src/main/java/alluxio/underfs/AbstractUfsManager.java b/core/server/common/src/main/java/alluxio/underfs/AbstractUfsManager.java deleted file mode 100644 index 4de988ca2e1b..000000000000 --- a/core/server/common/src/main/java/alluxio/underfs/AbstractUfsManager.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.underfs; - -import alluxio.AlluxioURI; -import alluxio.concurrent.ManagedBlockingUfsForwarder; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.NotFoundException; -import alluxio.exception.status.UnavailableException; -import alluxio.master.journal.ufs.UfsJournal; -import alluxio.recorder.Recorder; -import alluxio.util.IdUtils; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Basic implementation of {@link UfsManager}. Store the journal UFS and root - * mount point information. - */ -public abstract class AbstractUfsManager implements UfsManager { - private static final Logger LOG = LoggerFactory.getLogger(AbstractUfsManager.class); - - private final Object mLock = new Object(); - - /** - * The key of the UFS cache. - */ - public static class Key { - private final String mScheme; - private final String mAuthority; - private final Map mProperties; - - Key(AlluxioURI uri, Map properties) { - mScheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(); - mAuthority = uri.getAuthority().toString().toLowerCase(); - mProperties = (properties == null || properties.isEmpty()) ? null : properties; - } - - @Override - public int hashCode() { - return Objects.hashCode(mScheme, mAuthority, mProperties); - } - - @Override - public boolean equals(Object object) { - if (object == this) { - return true; - } - - if (!(object instanceof Key)) { - return false; - } - - Key that = (Key) object; - return Objects.equal(mAuthority, that.mAuthority) && Objects - .equal(mProperties, that.mProperties) && Objects.equal(mScheme, that.mScheme); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("authority", mAuthority) - .add("scheme", mScheme) - .add("properties", mProperties) - .toString(); - } - } - - // TODO(binfan): Add refcount to the UFS instance. Once the refcount goes to zero, - // we could close this UFS instance. - /** - * Maps from key to {@link UnderFileSystem} instances. This map keeps the entire set of UFS - * instances, each keyed by their unique combination of Uri and conf information. This map - * helps efficiently identify if a UFS instance in request should be created or can be reused. - */ - protected final ConcurrentHashMap mUnderFileSystemMap = - new ConcurrentHashMap<>(); - /** - * Maps from mount id to {@link UfsClient} instances. This map helps efficiently retrieve - * existing UFS info given its mount id. - */ - private final ConcurrentHashMap mMountIdToUfsInfoMap = - new ConcurrentHashMap<>(); - - private UfsClient mRootUfsClient; - private UfsClient mJournalUfsClient; - protected final Closer mCloser; - - protected AbstractUfsManager() { - mCloser = Closer.create(); - } - - /** - * Return a UFS instance if it already exists in the cache, otherwise, creates a new instance and - * return it. - * - * @param ufsUri the UFS path - * @param ufsConf the UFS configuration - * @return the UFS instance - */ - private UnderFileSystem getOrAdd(AlluxioURI ufsUri, UnderFileSystemConfiguration ufsConf) { - return getOrAddWithRecorder(ufsUri, ufsConf, Recorder.noopRecorder()); - } - - /** - * Return a UFS instance if it already exists in the cache, otherwise, creates a new instance and - * return it and record the execution process. - * - * @param ufsUri the UFS path - * @param ufsConf the UFS configuration - * @param recorder recorder used to record the detailed execution process - * @return the UFS instance - */ - private UnderFileSystem getOrAddWithRecorder(AlluxioURI ufsUri, - UnderFileSystemConfiguration ufsConf, Recorder recorder) { - Key key = new Key(ufsUri, ufsConf.getMountSpecificConf()); - UnderFileSystem cachedFs = mUnderFileSystemMap.get(key); - if (cachedFs != null) { - recorder.record("Using cached instance of UFS {} identified by key {}", - cachedFs.getClass().getSimpleName(), key.toString()); - return cachedFs; - } - // On cache miss, synchronize the creation to ensure ufs is only created once - synchronized (mLock) { - cachedFs = mUnderFileSystemMap.get(key); - if (cachedFs != null) { - recorder.record("Using cached instance of UFS {} identified by key {}", - cachedFs.getClass().getSimpleName(), key.toString()); - return cachedFs; - } - UnderFileSystem fs = UnderFileSystem.Factory.createWithRecorder( - ufsUri.toString(), ufsConf, recorder); - - // Detect whether to use managed blocking on UFS operations. - boolean useManagedBlocking = fs.isObjectStorage(); - if (ufsConf.isSet(PropertyKey.MASTER_UFS_MANAGED_BLOCKING_ENABLED)) { - useManagedBlocking = ufsConf.getBoolean(PropertyKey.MASTER_UFS_MANAGED_BLOCKING_ENABLED); - } - // Wrap UFS under managed blocking forwarder if required. - if (useManagedBlocking) { - fs = new ManagedBlockingUfsForwarder(fs); - } - - if (mUnderFileSystemMap.putIfAbsent(key, fs) != null) { - // This shouldn't occur unless our synchronization is incorrect - LOG.warn("UFS already existed in UFS manager"); - } - mCloser.register(fs); - try { - connectUfs(fs); - } catch (IOException e) { - String message = String.format( - "Failed to perform initial connect to UFS %s: %s", ufsUri, e); - recorder.record(message); - LOG.warn(message); - } - return fs; - } - } - - /** - * Takes any necessary actions required to establish a connection to the under file system. - * The implementation will either call {@link UnderFileSystem#connectFromMaster(String)} or - * {@link UnderFileSystem#connectFromWorker(String)} depending on the running process. - */ - protected abstract void connectUfs(UnderFileSystem fs) throws IOException; - - @Override - public void addMount(long mountId, final AlluxioURI ufsUri, - final UnderFileSystemConfiguration ufsConf) { - addMountWithRecorder(mountId, ufsUri, ufsConf, Recorder.noopRecorder()); - } - - @Override - public void addMountWithRecorder(long mountId, final AlluxioURI ufsUri, - final UnderFileSystemConfiguration ufsConf, Recorder recorder) { - Preconditions.checkArgument(mountId != IdUtils.INVALID_MOUNT_ID, "mountId"); - Preconditions.checkNotNull(ufsUri, "ufsUri"); - Preconditions.checkNotNull(ufsConf, "ufsConf"); - mMountIdToUfsInfoMap.put(mountId, new UfsClient(() -> - getOrAddWithRecorder(ufsUri, ufsConf, recorder), ufsUri)); - } - - @Override - public void removeMount(long mountId) { - Preconditions.checkArgument(mountId != IdUtils.INVALID_MOUNT_ID, "mountId"); - // TODO(binfan): check the refcount of this ufs in mUnderFileSystemMap and remove it if this is - // no more used. Currently, it is possibly used by out mount too. - mMountIdToUfsInfoMap.remove(mountId); - } - - @Override - public UfsClient get(long mountId) throws NotFoundException, UnavailableException { - UfsClient ufsClient = mMountIdToUfsInfoMap.get(mountId); - if (ufsClient == null) { - throw new NotFoundException( - String.format("Mount Id %d not found in cached mount points", mountId)); - } - return ufsClient; - } - - @Override - public UfsClient getRoot() { - synchronized (this) { - if (mRootUfsClient == null) { - String rootUri = Configuration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); - boolean rootReadOnly = - Configuration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_READONLY); - Map rootConf = - Configuration.getNestedProperties(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION); - addMount(IdUtils.ROOT_MOUNT_ID, new AlluxioURI(rootUri), - new UnderFileSystemConfiguration(Configuration.global(), rootReadOnly) - .createMountSpecificConf(rootConf)); - try { - mRootUfsClient = get(IdUtils.ROOT_MOUNT_ID); - } catch (NotFoundException | UnavailableException e) { - throw new RuntimeException("We should never reach here", e); - } - } - return mRootUfsClient; - } - } - - @Override - public UfsClient getJournal(URI location) { - synchronized (this) { - if (mJournalUfsClient == null) { - addMount(IdUtils.UFS_JOURNAL_MOUNT_ID, new AlluxioURI(location.toString()), - UfsJournal.getJournalUfsConf()); - try { - mJournalUfsClient = get(IdUtils.UFS_JOURNAL_MOUNT_ID); - } catch (NotFoundException | UnavailableException e) { - throw new RuntimeException("We should never reach here", e); - } - } - return mJournalUfsClient; - } - } - - @Override - public void close() throws IOException { - mCloser.close(); - } - - @Override - public boolean hasMount(long mountId) { - return mMountIdToUfsInfoMap.containsKey(mountId); - } -} diff --git a/core/server/common/src/main/java/alluxio/util/TarUtils.java b/core/server/common/src/main/java/alluxio/util/TarUtils.java deleted file mode 100644 index e1822c9bf80b..000000000000 --- a/core/server/common/src/main/java/alluxio/util/TarUtils.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.util; - -import static java.util.stream.Collectors.toList; - -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; -import org.apache.commons.io.IOUtils; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.stream.Stream; - -/** - * Utility methods for working with tar archives. - */ -public final class TarUtils { - /** - * Creates a gzipped tar archive from the given path, streaming the data to the give output - * stream. - * - * @param dirPath the path to archive - * @param output the output stream to write the data to - */ - public static void writeTarGz(Path dirPath, OutputStream output) - throws IOException, InterruptedException { - GzipCompressorOutputStream zipStream = new GzipCompressorOutputStream(output); - TarArchiveOutputStream archiveStream = new TarArchiveOutputStream(zipStream); - archiveStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - archiveStream.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); - try (final Stream stream = Files.walk(dirPath)) { - for (Path subPath : stream.collect(toList())) { - if (Thread.interrupted()) { - throw new InterruptedException(); - } - File file = subPath.toFile(); - TarArchiveEntry entry = new TarArchiveEntry(file, dirPath.relativize(subPath).toString()); - archiveStream.putArchiveEntry(entry); - if (file.isFile()) { - try (InputStream fileIn = Files.newInputStream(subPath)) { - IOUtils.copy(fileIn, archiveStream); - } - } - archiveStream.closeArchiveEntry(); - } - } - archiveStream.finish(); - zipStream.finish(); - } - - /** - * Reads a gzipped tar archive from a stream and writes it to the given path. - * - * @param dirPath the path to write the archive to - * @param input the input stream - */ - public static void readTarGz(Path dirPath, InputStream input) throws IOException { - InputStream zipStream = new GzipCompressorInputStream(input); - TarArchiveInputStream archiveStream = new TarArchiveInputStream(zipStream); - TarArchiveEntry entry; - while ((entry = (TarArchiveEntry) archiveStream.getNextEntry()) != null) { - File outputFile = new File(dirPath.toFile(), entry.getName()); - if (entry.isDirectory()) { - outputFile.mkdirs(); - } else { - outputFile.getParentFile().mkdirs(); - try (FileOutputStream fileOut = new FileOutputStream(outputFile)) { - IOUtils.copy(archiveStream, fileOut); - } - } - } - } - - private TarUtils() {} // Utils class -} diff --git a/core/server/common/src/main/java/alluxio/worker/WorkerFactory.java b/core/server/common/src/main/java/alluxio/worker/WorkerFactory.java deleted file mode 100644 index 2b8746e44f95..000000000000 --- a/core/server/common/src/main/java/alluxio/worker/WorkerFactory.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker; - -import alluxio.underfs.UfsManager; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Interface for factory of {@link Worker}. - */ -@ThreadSafe -public interface WorkerFactory { - /** - * @return whether the worker is enabled - */ - boolean isEnabled(); - - /** - * Factory method to create a new worker instance. - * - * @param registry the worker registry - * @param ufsManager the UFS manager - * @return a new {@link Worker} instance - */ - Worker create(WorkerRegistry registry, UfsManager ufsManager); -} diff --git a/core/server/common/src/test/java/alluxio/cli/FormatTest.java b/core/server/common/src/test/java/alluxio/cli/FormatTest.java deleted file mode 100644 index 96df2292cc62..000000000000 --- a/core/server/common/src/test/java/alluxio/cli/FormatTest.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.cli; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import alluxio.ConfigurationRule; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.util.io.FileUtils; -import alluxio.util.io.PathUtils; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.Closeable; -import java.io.File; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.HashMap; - -/** - * Unit tests for {@link Format}. - */ -public final class FormatTest { - - @Rule - public TemporaryFolder mTemporaryFolder = new TemporaryFolder(); - - @Test - public void formatWorker() throws Exception { - final int storageLevels = 3; - final String perms = "rwx------"; - String workerDataFolder; - final File[] dirs = new File[] { - mTemporaryFolder.newFolder("level0"), - mTemporaryFolder.newFolder("level1"), - mTemporaryFolder.newFolder("level2") - }; - for (File dir : dirs) { - workerDataFolder = CommonUtils.getWorkerDataDirectory(dir.getPath(), - Configuration.global()); - FileUtils.createDir(PathUtils.concatPath(workerDataFolder, "subdir")); - FileUtils.createFile(PathUtils.concatPath(workerDataFolder, "file")); - } - try (Closeable r = new ConfigurationRule(new HashMap() { - { - put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, dirs[0].getPath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVEL1_DIRS_PATH, dirs[1].getPath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVEL2_DIRS_PATH, dirs[2].getPath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVELS, storageLevels); - put(PropertyKey.WORKER_DATA_FOLDER_PERMISSIONS, perms); - } - }, Configuration.modifiableGlobal()).toResource()) { - Format.format(Format.Mode.WORKER, Configuration.global()); - for (File dir : dirs) { - workerDataFolder = CommonUtils.getWorkerDataDirectory(dir.getPath(), - Configuration.global()); - assertTrue(FileUtils.exists(dir.getPath())); - assertTrue(FileUtils.exists(workerDataFolder)); - assertEquals(PosixFilePermissions.fromString(perms), Files.getPosixFilePermissions(Paths - .get(workerDataFolder))); - try (DirectoryStream directoryStream = Files - .newDirectoryStream(Paths.get(workerDataFolder))) { - for (Path child : directoryStream) { - fail("No sub dirs or files are expected in " + child.toString()); - } - } - } - } - } - - @Test - public void formatWorkerDeleteFileSameName() throws Exception { - final int storageLevels = 3; - String workerDataFolder; - final File[] dirs = new File[] { - mTemporaryFolder.newFolder("level0"), - mTemporaryFolder.newFolder("level1"), - mTemporaryFolder.newFolder("level2") - }; - // Have files of same name as the target worker data dir in each tier - for (File dir : dirs) { - workerDataFolder = CommonUtils.getWorkerDataDirectory(dir.getPath(), - Configuration.global()); - FileUtils.createFile(workerDataFolder); - } - try (Closeable r = new ConfigurationRule(new HashMap() { - { - put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, dirs[0].getPath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVEL1_DIRS_PATH, dirs[1].getPath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVEL2_DIRS_PATH, dirs[2].getPath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVELS, storageLevels); - } - }, Configuration.modifiableGlobal()).toResource()) { - final String perms = Configuration.getString( - PropertyKey.WORKER_DATA_FOLDER_PERMISSIONS); - Format.format(Format.Mode.WORKER, Configuration.global()); - for (File dir : dirs) { - workerDataFolder = CommonUtils.getWorkerDataDirectory(dir.getPath(), - Configuration.global()); - assertTrue(Files.isDirectory(Paths.get(workerDataFolder))); - assertEquals(PosixFilePermissions.fromString(perms), Files.getPosixFilePermissions(Paths - .get(workerDataFolder))); - try (DirectoryStream directoryStream = - Files.newDirectoryStream(Paths.get(workerDataFolder))) { - for (Path child : directoryStream) { - fail("No sub dirs or files are expected in " + child.toString()); - } - } - } - } - } -} diff --git a/core/server/common/src/test/java/alluxio/master/journal/JournalUtilsTest.java b/core/server/common/src/test/java/alluxio/master/journal/JournalUtilsTest.java deleted file mode 100644 index bc211e90bf69..000000000000 --- a/core/server/common/src/test/java/alluxio/master/journal/JournalUtilsTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal; - -import static org.junit.Assert.assertEquals; - -import alluxio.master.journal.checkpoint.CheckpointInputStream; -import alluxio.master.journal.checkpoint.CheckpointName; -import alluxio.master.journal.checkpoint.CheckpointOutputStream; -import alluxio.master.journal.checkpoint.CheckpointType; -import alluxio.proto.journal.File.AddMountPointEntry; -import alluxio.proto.journal.Journal.JournalEntry; -import alluxio.resource.CloseableIterator; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Unit tests for {@link JournalUtils}. - */ -public final class JournalUtilsTest { - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - @Test - public void checkpointAndRestore() throws IOException, InterruptedException { - Journaled journaled = new TestJournaled(0); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - JournalUtils.writeJournalEntryCheckpoint(baos, journaled); - JournalUtils.restoreJournalEntryCheckpoint( - new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())), journaled); - } - - @Test - public void restoreInvalidJournalEntryCheckpoint() throws IOException { - Journaled journaled = new TestJournaled(0); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - // Checkpoint version doesn't match Constants.JOURNAL_ENTRY_CHECKPOINT_VERSION. - CheckpointOutputStream cos = new CheckpointOutputStream(baos, CheckpointType.COMPOUND); - cos.flush(); - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage("Unrecognized checkpoint type"); - JournalUtils.restoreJournalEntryCheckpoint( - new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())), journaled); - } - - @Test - public void checkpointAndRestoreComponents() throws Exception { - List components = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - components.add(new TestJournaled(i)); - } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - JournalUtils.writeToCheckpoint(baos, components); - components.forEach(c -> assertEquals(0, c.getNumEntriesProcessed())); - JournalUtils.restoreFromCheckpoint( - new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())), components); - components.forEach(c -> assertEquals(1, c.getNumEntriesProcessed())); - } - - private static class TestJournaled implements Journaled { - private final CheckpointName mName; - private int mNumEntriesProcessed; - - public TestJournaled(int i) { - mName = CheckpointName.values()[i]; - } - - @Override - public boolean processJournalEntry(JournalEntry entry) { - mNumEntriesProcessed++; - return true; - } - - public int getNumEntriesProcessed() { - return mNumEntriesProcessed; - } - - @Override - public void resetState() { - } - - @Override - public CloseableIterator getJournalEntryIterator() { - return CloseableIterator.noopCloseable(Arrays.asList(JournalEntry.newBuilder() - .setAddMountPoint(AddMountPointEntry.getDefaultInstance()).build()).iterator()); - } - - @Override - public CheckpointName getCheckpointName() { - return mName; - } - } -} diff --git a/core/server/common/src/test/java/alluxio/util/ParallelZipUtilsTest.java b/core/server/common/src/test/java/alluxio/util/ParallelZipUtilsTest.java deleted file mode 100644 index 2c51edbddadb..000000000000 --- a/core/server/common/src/test/java/alluxio/util/ParallelZipUtilsTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.util; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; - -/** - * Units tests for {@link ParallelZipUtils}. - */ -public final class ParallelZipUtilsTest { - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - @Test - public void emptyDir() throws Exception { - Path empty = mFolder.newFolder("emptyDir").toPath(); - - zipUnzipTest(empty); - } - - @Test - public void oneFileDir() throws Exception { - Path dir = mFolder.newFolder("oneFileDir").toPath(); - Path file = dir.resolve("file"); - Files.write(file, "test content".getBytes()); - - zipUnzipTest(dir); - } - - @Test - public void tenFileDir() throws Exception { - Path dir = mFolder.newFolder("tenFileDir").toPath(); - for (int i = 0; i < 10; i++) { - Path file = dir.resolve("file" + i); - Files.write(file, ("test content and a lot of test content" + i).getBytes()); - } - - zipUnzipTest(dir); - } - - @Test - public void emptySubDir() throws Exception { - Path dir = mFolder.newFolder("emptySubDir").toPath(); - Path subDir = dir.resolve("subDir"); - Files.createDirectory(subDir); - - zipUnzipTest(dir); - } - - @Test - public void nested() throws Exception { - Path dir = mFolder.newFolder("emptySubDir").toPath(); - Path current = dir; - for (int i = 0; i < 10; i++) { - Path newDir = current.resolve("dir" + i); - Files.createDirectory(newDir); - current = newDir; - } - Path file = current.resolve("file"); - Files.write(file, "hello world".getBytes()); - - zipUnzipTest(dir); - } - - private void zipUnzipTest(Path path) throws Exception { - String zippedPath = mFolder.newFile("zipped").getPath(); - try (FileOutputStream fos = new FileOutputStream(zippedPath)) { - ParallelZipUtils.compress(path, fos, 5, 5); - } - - Path reconstructed = mFolder.newFolder("unzipped").toPath(); - reconstructed.toFile().delete(); - ParallelZipUtils.decompress(reconstructed, zippedPath, 5); - FileUtil.assertDirectoriesEqual(path, reconstructed); - } -} diff --git a/core/server/master/pom.xml b/core/server/master/pom.xml deleted file mode 100644 index 8fe392d7f5de..000000000000 --- a/core/server/master/pom.xml +++ /dev/null @@ -1,185 +0,0 @@ - - - 4.0.0 - - alluxio-core-server - org.alluxio - 2.10.0-SNAPSHOT - - alluxio-core-server-master - jar - Alluxio Core - Server - Master - Alluxio master service - - - - - ${project.parent.parent.parent.basedir}/build - - - - - - com.google.guava - guava - - - io.dropwizard.metrics - metrics-core - - - io.swagger - swagger-annotations - - - javax.servlet - javax.servlet-api - - - org.apache.commons - commons-lang3 - - - org.eclipse.jetty - jetty-servlet - - - org.glassfish.jersey.containers - jersey-container-servlet-core - - - org.glassfish.jersey.core - jersey-server - - - org.glassfish.jersey.inject - jersey-hk2 - - - org.hdrhistogram - HdrHistogram - - - org.rocksdb - rocksdbjni - - - org.glassfish.jersey.media - jersey-media-json-jackson - provided - - - it.unimi.dsi - fastutil-core - - - - - org.alluxio - alluxio-core-common - ${project.version} - - - org.alluxio - alluxio-core-transport - ${project.version} - - - org.alluxio - alluxio-core-client-fs - ${project.version} - - - org.alluxio - alluxio-core-server-common - ${project.version} - - - org.alluxio - alluxio-job-client - ${project.version} - - - - - com.google.guava - guava-testlib - test - - - - - org.alluxio - alluxio-core-common - ${project.version} - test-jar - test - - - org.alluxio - alluxio-underfs-local - ${project.version} - test - - - org.apache.httpcomponents - httpclient - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - - - false - com.github.kongchen - swagger-maven-plugin - - - - false - http - [Alluxio Master Hostname] - /api/v1 - - v1 - Alluxio Master REST API Documentation - - The Alluxio Master is the central metadata service of the Alluxio System. - - - alluxio.master.meta.AlluxioMasterRestServiceHandler - ${project.parent.parent.parent.basedir}/templates/strapdown.html.hbs - ${project.parent.parent.parent.basedir}/generated/master/index.html - ${project.parent.parent.parent.basedir}/generated/master/swagger-ui - - - - - - - diff --git a/core/server/master/src/main/java/alluxio/master/AlluxioSecondaryMaster.java b/core/server/master/src/main/java/alluxio/master/AlluxioSecondaryMaster.java deleted file mode 100644 index 760bca34cc49..000000000000 --- a/core/server/master/src/main/java/alluxio/master/AlluxioSecondaryMaster.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master; - -import alluxio.Process; -import alluxio.ProcessUtils; -import alluxio.RuntimeConstants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.journal.JournalSystem; -import alluxio.master.journal.JournalUtils; -import alluxio.underfs.MasterUfsManager; -import alluxio.util.CommonUtils; -import alluxio.util.CommonUtils.ProcessType; -import alluxio.util.WaitForOptions; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeoutException; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * The secondary Alluxio master that replays journal logs and writes checkpoints. - */ -@NotThreadSafe -public final class AlluxioSecondaryMaster implements Process { - private static final Logger LOG = LoggerFactory.getLogger(AlluxioSecondaryMaster.class); - private final MasterRegistry mRegistry = new MasterRegistry(); - private final JournalSystem mJournalSystem = new JournalSystem.Builder() - .setLocation(JournalUtils.getJournalLocation()).build(ProcessType.MASTER); - private final CountDownLatch mLatch = new CountDownLatch(1); - - private volatile boolean mRunning = false; - - /** - * Creates a {@link AlluxioSecondaryMaster}. - */ - AlluxioSecondaryMaster() { - String baseDir = Configuration.getString(PropertyKey.SECONDARY_MASTER_METASTORE_DIR); - // Create masters. - MasterUtils.createMasters(mRegistry, CoreMasterContext.newBuilder() - .setJournalSystem(mJournalSystem) - .setPrimarySelector(new AlwaysStandbyPrimarySelector()) - .setSafeModeManager(new DefaultSafeModeManager()) - .setBackupManager(new BackupManager(mRegistry)) - .setBlockStoreFactory(MasterUtils.getBlockStoreFactory(baseDir)) - .setInodeStoreFactory(MasterUtils.getInodeStoreFactory(baseDir)) - .setStartTimeMs(System.currentTimeMillis()) - .setPort(Configuration.getInt(PropertyKey.MASTER_RPC_PORT)) - .setUfsManager(new MasterUfsManager()) - .build()); - // Check that journals of each service have been formatted. - if (!mJournalSystem.isFormatted()) { - throw new RuntimeException( - String.format("Journal %s has not been formatted!", JournalUtils.getJournalLocation())); - } - } - - @Override - public void start() throws Exception { - mJournalSystem.start(); - mRunning = true; - mLatch.await(); - mJournalSystem.stop(); - mRegistry.close(); - mRunning = false; - } - - @Override - public void stop() throws Exception { - mLatch.countDown(); - } - - @Override - public boolean waitForReady(int timeoutMs) { - try { - CommonUtils.waitFor("Secondary master to start", () -> mRunning, - WaitForOptions.defaults().setTimeoutMs(timeoutMs)); - return true; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } catch (TimeoutException e) { - return false; - } - } - - @Override - public String toString() { - return "Alluxio secondary master"; - } - - /** - * Starts the secondary Alluxio master. - * - * @param args command line arguments, should be empty - */ - // TODO(peis): Move the non-static methods into AlluxioSecondaryMasterProcess for consistency. - public static void main(String[] args) { - if (args.length != 0) { - LOG.info("java -cp {} {}", RuntimeConstants.ALLUXIO_JAR, - AlluxioSecondaryMaster.class.getCanonicalName()); - System.exit(-1); - } - - AlluxioSecondaryMaster master = new AlluxioSecondaryMaster(); - ProcessUtils.run(master); - } -} diff --git a/core/server/master/src/main/java/alluxio/master/block/DefaultBlockMaster.java b/core/server/master/src/main/java/alluxio/master/block/DefaultBlockMaster.java deleted file mode 100644 index 7a44abbb0611..000000000000 --- a/core/server/master/src/main/java/alluxio/master/block/DefaultBlockMaster.java +++ /dev/null @@ -1,1746 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.block; - -import alluxio.Constants; -import alluxio.DefaultStorageTierAssoc; -import alluxio.Server; -import alluxio.StorageTierAssoc; -import alluxio.annotation.SuppressFBWarnings; -import alluxio.client.block.options.GetWorkerReportOptions; -import alluxio.client.block.options.GetWorkerReportOptions.WorkerRange; -import alluxio.clock.SystemClock; -import alluxio.collections.ConcurrentHashSet; -import alluxio.collections.IndexDefinition; -import alluxio.collections.IndexedSet; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.BlockInfoException; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.status.InvalidArgumentException; -import alluxio.exception.status.NotFoundException; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.Command; -import alluxio.grpc.CommandType; -import alluxio.grpc.ConfigProperty; -import alluxio.grpc.GetRegisterLeasePRequest; -import alluxio.grpc.GrpcService; -import alluxio.grpc.GrpcUtils; -import alluxio.grpc.RegisterWorkerPOptions; -import alluxio.grpc.RegisterWorkerPRequest; -import alluxio.grpc.ServiceType; -import alluxio.grpc.StorageList; -import alluxio.grpc.WorkerLostStorageInfo; -import alluxio.heartbeat.HeartbeatContext; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.heartbeat.HeartbeatThread; -import alluxio.master.CoreMaster; -import alluxio.master.CoreMasterContext; -import alluxio.master.block.meta.MasterWorkerInfo; -import alluxio.master.block.meta.WorkerMetaLockSection; -import alluxio.master.journal.JournalContext; -import alluxio.master.journal.checkpoint.CheckpointName; -import alluxio.master.metastore.BlockMetaStore; -import alluxio.master.metastore.BlockMetaStore.Block; -import alluxio.master.metrics.MetricsMaster; -import alluxio.metrics.Metric; -import alluxio.metrics.MetricInfo; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proto.journal.Block.BlockContainerIdGeneratorEntry; -import alluxio.proto.journal.Block.BlockInfoEntry; -import alluxio.proto.journal.Block.DeleteBlockEntry; -import alluxio.proto.journal.Journal.JournalEntry; -import alluxio.proto.meta.Block.BlockLocation; -import alluxio.proto.meta.Block.BlockMeta; -import alluxio.resource.CloseableIterator; -import alluxio.resource.LockResource; -import alluxio.util.CommonUtils; -import alluxio.util.IdUtils; -import alluxio.util.ThreadFactoryUtils; -import alluxio.util.executor.ExecutorServiceFactories; -import alluxio.util.executor.ExecutorServiceFactory; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.Address; -import alluxio.wire.BlockInfo; -import alluxio.wire.RegisterLease; -import alluxio.wire.WorkerInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.Striped; -import it.unimi.dsi.fastutil.longs.LongOpenHashSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.UnknownHostException; -import java.time.Clock; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This block master manages the metadata for all the blocks and block workers in Alluxio. - */ -@NotThreadSafe // TODO(jiri): make thread-safe (c.f. ALLUXIO-1664) -public class DefaultBlockMaster extends CoreMaster implements BlockMaster { - private static final Set> DEPS = - ImmutableSet.of(MetricsMaster.class); - - /** - * The number of container ids to 'reserve' before having to journal container id state. This - * allows the master to return container ids within the reservation, without having to write to - * the journal. - */ - private final long mContainerIdReservationSize = Configuration.getInt( - PropertyKey.MASTER_CONTAINER_ID_RESERVATION_SIZE); - - /** The only valid key for {@link #mWorkerInfoCache}. */ - private static final String WORKER_INFO_CACHE_KEY = "WorkerInfoKey"; - - private final ExecutorService mContainerIdDetector = Executors - .newSingleThreadExecutor( - ThreadFactoryUtils.build("default-block-master-container-id-detection-%d", true)); - - private volatile boolean mContainerIdDetectorIsIdle = true; - - // Worker metadata management. - private static final IndexDefinition ID_INDEX = - IndexDefinition.ofUnique(MasterWorkerInfo::getId); - - private static final IndexDefinition ADDRESS_INDEX = - IndexDefinition.ofUnique(MasterWorkerInfo::getWorkerAddress); - - /** - * Mapping between all possible storage level aliases and their ordinal position. This mapping - * forms a total ordering on all storage level aliases in the system, and must be consistent - * across masters. - */ - private static final StorageTierAssoc MASTER_STORAGE_TIER_ASSOC = - new DefaultStorageTierAssoc( - PropertyKey.MASTER_TIERED_STORE_GLOBAL_LEVELS, - PropertyKey.Template.MASTER_TIERED_STORE_GLOBAL_LEVEL_ALIAS); - - private static final Logger LOG = LoggerFactory.getLogger(DefaultBlockMaster.class); - - /** - * Concurrency and locking in the BlockMaster - * - * The block master uses concurrent data structures to allow non-conflicting concurrent access. - * This means each piece of metadata should be locked individually. There are two types of - * metadata in the {@link DefaultBlockMaster}: block metadata and worker metadata. - * - * The worker metadata is represented by the {@link MasterWorkerInfo} object. - * See javadoc of {@link MasterWorkerInfo} for details. - * - * To modify or read a modifiable piece of worker metadata, the {@link MasterWorkerInfo} for the - * worker must be locked following the instructions in {@link MasterWorkerInfo}. - * For block metadata, the id of the block must be locked. - * This will protect the internal integrity of the block and worker metadata. - * - * A worker's relevant locks must be held to - * - Check/Update the worker register status - * - Read/Update the worker usage - * - Read/Update the worker present/to-be-removed blocks - * - Any combinations of the above - * - * A block's lock must be held to - * - Perform any BlockStore operations on the block - * - Add or remove the block from mLostBlocks - * - * Lock ordering must be preserved in order to prevent deadlock. If both worker and block - * metadata must be locked at the same time, the worker metadata must be locked before the block - * metadata. When the locks are released, they must be released in the opposite order. - * - * Locking on the worker metadata are managed by - * {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)}. - * This guarantees when multiple parts of the worker metadata are accessed/updated, - * the locks are acquired and released in order. - * See javadoc of {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} for - * example usages. - * - * It should not be the case that multiple worker metadata must be locked at the same time, or - * multiple block metadata must be locked at the same time. Operations involving different workers - * or different blocks should be able to be performed independently. - */ - - /** - * 10k locks balances between keeping a small memory footprint and avoiding unnecessary lock - * contention. Each stripe is around 100 bytes, so this takes about 1MB. Block locking critical - * sections are short, so it is acceptable to occasionally have conflicts where two different - * blocks want to lock the same stripe. - */ - private final Striped mBlockLocks = Striped.lock(10_000); - /** Manages block metadata and block locations. */ - private final BlockMetaStore mBlockMetaStore; - - /** Keeps track of blocks which are no longer in Alluxio storage. */ - private final ConcurrentHashSet mLostBlocks = new ConcurrentHashSet<>(64, 0.90f, 64); - - /** This state must be journaled. */ - @GuardedBy("itself") - private final BlockContainerIdGenerator mBlockContainerIdGenerator = - new BlockContainerIdGenerator(); - - /** Keeps track of workers which are in communication with the master. */ - private final IndexedSet mWorkers = - new IndexedSet<>(ID_INDEX, ADDRESS_INDEX); - /** Keeps track of workers which are no longer in communication with the master. */ - private final IndexedSet mLostWorkers = - new IndexedSet<>(ID_INDEX, ADDRESS_INDEX); - /** Worker is not visualable until registration completes. */ - private final IndexedSet mTempWorkers = - new IndexedSet<>(ID_INDEX, ADDRESS_INDEX); - /** - * Keeps track of workers which have been decommissioned. - * For we need to distinguish the lost worker accidentally and the decommissioned worker manually. - */ - private final IndexedSet mDecommissionedWorkers = - new IndexedSet<>(ID_INDEX, ADDRESS_INDEX); - - /** - * Tracks the open register streams. - * A stream will be closed if it is completed, aborted due to an error, - * or recycled due to inactivity by {@link WorkerRegisterStreamGCExecutor}. - */ - private final Map mActiveRegisterContexts = - new ConcurrentHashMap<>(); - - /** Listeners to call when lost workers are found. */ - private final List> mLostWorkerFoundListeners - = new ArrayList<>(); - - /** Listeners to call when workers are lost. */ - private final List> mWorkerLostListeners = new ArrayList<>(); - - /** Listeners to call when workers are delete. */ - private final List> mWorkerDeleteListeners = new ArrayList<>(); - - /** Listeners to call when a new worker registers. */ - private final List>> mWorkerRegisteredListeners - = new ArrayList<>(); - - /** Handle to the metrics master. */ - private final MetricsMaster mMetricsMaster; - - /** - * The service that detects lost worker nodes, and tries to restart the failed workers. - * We store it here so that it can be accessed from tests. - */ - @SuppressFBWarnings("URF_UNREAD_FIELD") - - /* The value of the 'next container id' last journaled. */ - @GuardedBy("mBlockContainerIdGenerator") - private volatile long mJournaledNextContainerId = 0; - - /** - * A loading cache for worker info list, refresh periodically. - * This cache only has a single key {@link #WORKER_INFO_CACHE_KEY}. - */ - private final LoadingCache> mWorkerInfoCache; - - private final RegisterLeaseManager mRegisterLeaseManager = new RegisterLeaseManager(); - - /** - * Creates a new instance of {@link DefaultBlockMaster}. - * - * @param metricsMaster the metrics master - * @param masterContext the context for Alluxio master - */ - DefaultBlockMaster(MetricsMaster metricsMaster, CoreMasterContext masterContext) { - this(metricsMaster, masterContext, new SystemClock(), - ExecutorServiceFactories.cachedThreadPool(Constants.BLOCK_MASTER_NAME)); - } - - private DefaultBlockMaster(MetricsMaster metricsMaster, CoreMasterContext masterContext, - Clock clock, ExecutorServiceFactory executorServiceFactory, BlockMetaStore blockMetaStore) { - super(masterContext, clock, executorServiceFactory); - Preconditions.checkNotNull(metricsMaster, "metricsMaster"); - - mBlockMetaStore = blockMetaStore; - mMetricsMaster = metricsMaster; - Metrics.registerGauges(this); - - mWorkerInfoCache = CacheBuilder.newBuilder() - .refreshAfterWrite(Configuration - .getMs(PropertyKey.MASTER_WORKER_INFO_CACHE_REFRESH_TIME), TimeUnit.MILLISECONDS) - .build(new CacheLoader>() { - @Override - public List load(String key) { - return constructWorkerInfoList(); - } - }); - - MetricsSystem.registerGaugeIfAbsent(MetricKey.MASTER_LOST_BLOCK_COUNT.getName(), - this::getLostBlocksCount); - MetricsSystem.registerCachedGaugeIfAbsent(MetricKey.MASTER_TO_REMOVE_BLOCK_COUNT.getName(), - this::getToRemoveBlockCount, 30, TimeUnit.SECONDS); - } - - /** - * Creates a new instance of {@link DefaultBlockMaster}. - * Used for tests where we manually control the clock. - * - * @param metricsMaster the metrics master - * @param masterContext the context for Alluxio master - * @param clock the clock to use for determining the time - * @param executorServiceFactory a factory for creating the executor service to use for running - * maintenance threads - */ - @VisibleForTesting - public DefaultBlockMaster(MetricsMaster metricsMaster, CoreMasterContext masterContext, - Clock clock, ExecutorServiceFactory executorServiceFactory) { - this(metricsMaster, masterContext, clock, executorServiceFactory, - masterContext.getBlockStoreFactory().get()); - } - - @Override - public String getName() { - return Constants.BLOCK_MASTER_NAME; - } - - @Override - public Map getServices() { - Map services = new HashMap<>(); - services.put(ServiceType.BLOCK_MASTER_CLIENT_SERVICE, - new GrpcService(new BlockMasterClientServiceHandler(this))); - services.put(ServiceType.BLOCK_MASTER_WORKER_SERVICE, - new GrpcService(new BlockMasterWorkerServiceHandler(this))); - return services; - } - - @Override - public boolean processJournalEntry(JournalEntry entry) { - // TODO(gene): A better way to process entries besides a huge switch? - if (entry.hasBlockContainerIdGenerator()) { - mJournaledNextContainerId = (entry.getBlockContainerIdGenerator()).getNextContainerId(); - mBlockContainerIdGenerator.setNextContainerId((mJournaledNextContainerId)); - } else if (entry.hasDeleteBlock()) { - mBlockMetaStore.removeBlock(entry.getDeleteBlock().getBlockId()); - } else if (entry.hasBlockInfo()) { - BlockInfoEntry blockInfoEntry = entry.getBlockInfo(); - long length = blockInfoEntry.getLength(); - Optional block = mBlockMetaStore.getBlock(blockInfoEntry.getBlockId()); - if (block.isPresent()) { - long oldLen = block.get().getLength(); - if (oldLen != Constants.UNKNOWN_SIZE) { - LOG.warn("Attempting to update block length ({}) to a different length ({}).", oldLen, - length); - return true; - } - } - mBlockMetaStore.putBlock(blockInfoEntry.getBlockId(), - BlockMeta.newBuilder().setLength(blockInfoEntry.getLength()).build()); - } else { - return false; - } - return true; - } - - @Override - public void resetState() { - mBlockMetaStore.clear(); - mJournaledNextContainerId = 0; - mBlockContainerIdGenerator.setNextContainerId(0); - } - - @Override - public CheckpointName getCheckpointName() { - return CheckpointName.BLOCK_MASTER; - } - - @Override - public CloseableIterator getJournalEntryIterator() { - CloseableIterator blockStoreIterator = mBlockMetaStore.getCloseableIterator(); - Iterator journalIterator = new Iterator() { - @Override - public boolean hasNext() { - return blockStoreIterator.hasNext(); - } - - @Override - public JournalEntry next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - Block block = blockStoreIterator.next(); - BlockInfoEntry blockInfoEntry = - BlockInfoEntry.newBuilder().setBlockId(block.getId()) - .setLength(block.getMeta().getLength()).build(); - return JournalEntry.newBuilder().setBlockInfo(blockInfoEntry).build(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException("BlockMaster#Iterator#remove is not supported."); - } - }; - - CloseableIterator journalCloseableIterator = - CloseableIterator.create(journalIterator, (whatever) -> blockStoreIterator.close()); - - return CloseableIterator.concat( - CloseableIterator.noopCloseable( - CommonUtils.singleElementIterator(getContainerIdJournalEntry())), - journalCloseableIterator); - } - - /** - * Periodically checks the open worker register streams. - * If a stream has been active for a while, close the stream, recycle resources and locks, - * and propagate an error to the worker side. - */ - public class WorkerRegisterStreamGCExecutor implements HeartbeatExecutor { - private final long mTimeout = Configuration.global() - .getMs(PropertyKey.MASTER_WORKER_REGISTER_STREAM_RESPONSE_TIMEOUT); - - @Override - public void heartbeat() { - AtomicInteger removedSessions = new AtomicInteger(0); - mActiveRegisterContexts.entrySet().removeIf((entry) -> { - WorkerRegisterContext context = entry.getValue(); - final long clockTime = mClock.millis(); - final long lastActivityTime = context.getLastActivityTimeMs(); - final long staleTime = clockTime - lastActivityTime; - if (staleTime < mTimeout) { - return false; - } - String msg = String.format( - "ClockTime: %d, LastActivityTime: %d. Worker %d register stream hanging for %sms!" - + " Tune up %s if this is undesired.", - clockTime, lastActivityTime, context.getWorkerInfo().getId(), staleTime, - PropertyKey.MASTER_WORKER_REGISTER_STREAM_RESPONSE_TIMEOUT); - Exception e = new TimeoutException(msg); - try { - context.closeWithError(e); - } catch (Throwable t) { - t.addSuppressed(e); - LOG.error("Failed to close an open register stream for worker {}. " - + "The stream has been open for {}ms.", context.getWorkerId(), staleTime, t); - // Do not remove the entry so this will be retried - return false; - } - removedSessions.getAndDecrement(); - return true; - }); - if (removedSessions.get() > 0) { - LOG.info("Removed {} stale worker registration streams", removedSessions.get()); - } - } - - @Override - public void close() { - // Nothing to clean up - } - } - - @Override - public void start(Boolean isLeader) throws IOException { - super.start(isLeader); - if (isLeader) { - getExecutorService().submit(new HeartbeatThread( - HeartbeatContext.MASTER_LOST_WORKER_DETECTION, new LostWorkerDetectionHeartbeatExecutor(), - (int) Configuration.getMs(PropertyKey.MASTER_LOST_WORKER_DETECTION_INTERVAL), - Configuration.global(), mMasterContext.getUserState())); - } - - // This periodically scans all open register streams and closes hanging ones - getExecutorService().submit(new HeartbeatThread( - HeartbeatContext.MASTER_WORKER_REGISTER_SESSION_CLEANER, - new WorkerRegisterStreamGCExecutor(), - (int) Configuration.getMs(PropertyKey.MASTER_WORKER_REGISTER_STREAM_RESPONSE_TIMEOUT), - Configuration.global(), mMasterContext.getUserState())); - } - - @Override - public void stop() throws IOException { - super.stop(); - } - - @Override - public void close() throws IOException { - super.close(); - mBlockMetaStore.close(); - - mContainerIdDetector.shutdown(); - try { - mContainerIdDetector.awaitTermination(5000, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - LOG.warn("Container id detection executor did not shut down in a timely manner: {}", - e.toString()); - } - } - - @Override - public int getWorkerCount() { - return mWorkers.size(); - } - - @Override - public int getLostWorkerCount() { - return mLostWorkers.size(); - } - - @Override - public int getDecommissionedWorkerCount() { - return mDecommissionedWorkers.size(); - } - - @Override - public long getCapacityBytes() { - long ret = 0; - for (MasterWorkerInfo worker : mWorkers) { - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.USAGE), true)) { - ret += worker.getCapacityBytes(); - } - } - return ret; - } - - @Override - public long getUniqueBlockCount() { - return mBlockMetaStore.size(); - } - - @Override - public long getBlockReplicaCount() { - long ret = 0; - for (MasterWorkerInfo worker : mWorkers) { - ret += worker.getBlockCount(); - } - return ret; - } - - @Override - public StorageTierAssoc getGlobalStorageTierAssoc() { - return MASTER_STORAGE_TIER_ASSOC; - } - - @Override - public long getUsedBytes() { - long ret = 0; - for (MasterWorkerInfo worker : mWorkers) { - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.USAGE), true)) { - ret += worker.getUsedBytes(); - } - } - return ret; - } - - @Override - public List getWorkerInfoList() throws UnavailableException { - if (mSafeModeManager.isInSafeMode()) { - throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage()); - } - try { - return mWorkerInfoCache.get(WORKER_INFO_CACHE_KEY); - } catch (ExecutionException e) { - throw new UnavailableException("Unable to get worker info list from cache", e); - } - } - - private List constructWorkerInfoList() { - List workerInfoList = new ArrayList<>(mWorkers.size()); - for (MasterWorkerInfo worker : mWorkers) { - // extractWorkerInfo handles the locking internally - workerInfoList.add(extractWorkerInfo(worker, - GetWorkerReportOptions.WorkerInfoField.ALL, true)); - } - return workerInfoList; - } - - @Override - public List getLostWorkersInfoList() throws UnavailableException { - if (mSafeModeManager.isInSafeMode()) { - throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage()); - } - List workerInfoList = new ArrayList<>(mLostWorkers.size()); - for (MasterWorkerInfo worker : mLostWorkers) { - // extractWorkerInfo handles the locking internally - workerInfoList.add(extractWorkerInfo(worker, - GetWorkerReportOptions.WorkerInfoField.ALL, false)); - } - workerInfoList.sort(new WorkerInfo.LastContactSecComparator()); - return workerInfoList; - } - - @Override - public void removeDecommissionedWorker(long workerId) throws NotFoundException { - MasterWorkerInfo worker = getWorker(workerId); - Preconditions.checkNotNull(mDecommissionedWorkers - .getFirstByField(ADDRESS_INDEX, worker.getWorkerAddress())); - processFreedWorker(worker); - } - - @Override - public Set getWorkerAddresses() throws UnavailableException { - if (mSafeModeManager.isInSafeMode()) { - throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage()); - } - Set workerAddresses = new HashSet<>(mWorkers.size()); - for (MasterWorkerInfo worker : mWorkers) { - // worker net address is unmodifiable after initialization, no locking is needed - workerAddresses.add(worker.getWorkerAddress()); - } - return workerAddresses; - } - - @Override - public List getWorkerReport(GetWorkerReportOptions options) - throws UnavailableException, InvalidArgumentException { - if (mSafeModeManager.isInSafeMode()) { - throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage()); - } - - Set selectedLiveWorkers = new HashSet<>(); - Set selectedLostWorkers = new HashSet<>(); - Set selectedDecommissionedWorkers = new HashSet<>(); - WorkerRange workerRange = options.getWorkerRange(); - switch (workerRange) { - case ALL: - selectedLiveWorkers.addAll(mWorkers); - selectedLostWorkers.addAll(mLostWorkers); - selectedDecommissionedWorkers.addAll(mDecommissionedWorkers); - break; - case LIVE: - selectedLiveWorkers.addAll(mWorkers); - break; - case LOST: - selectedLostWorkers.addAll(mLostWorkers); - break; - case DECOMMISSIONED: - selectedDecommissionedWorkers.addAll(mDecommissionedWorkers); - break; - case SPECIFIED: - Set addresses = options.getAddresses(); - Set workerNames = new HashSet<>(); - - selectedLiveWorkers = selectInfoByAddress(addresses, mWorkers, workerNames); - selectedLostWorkers = selectInfoByAddress(addresses, mLostWorkers, workerNames); - selectedDecommissionedWorkers = selectInfoByAddress(addresses, - mDecommissionedWorkers, workerNames); - - if (!addresses.isEmpty()) { - String info = String.format("Unrecognized worker names: %s%n" - + "Supported worker names: %s%n", - addresses, workerNames); - throw new InvalidArgumentException(info); - } - break; - default: - throw new InvalidArgumentException("Unrecognized worker range: " + workerRange); - } - - List workerInfoList = new ArrayList<>( - selectedLiveWorkers.size() + selectedLostWorkers.size() - + selectedDecommissionedWorkers.size()); - for (MasterWorkerInfo worker : selectedLiveWorkers) { - // extractWorkerInfo handles the locking internally - workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), true)); - } - for (MasterWorkerInfo worker : selectedLostWorkers) { - // extractWorkerInfo handles the locking internally - workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), false)); - } - for (MasterWorkerInfo worker : selectedDecommissionedWorkers) { - // extractWorkerInfo handles the locking internally - workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), false)); - } - return workerInfoList; - } - - /** - * Locks the {@link MasterWorkerInfo} properly and convert it to a {@link WorkerInfo}. - */ - private WorkerInfo extractWorkerInfo(MasterWorkerInfo worker, - Set fieldRange, boolean isLiveWorker) { - try (LockResource r = worker.lockWorkerMetaForInfo(fieldRange)) { - return worker.generateWorkerInfo(fieldRange, isLiveWorker); - } - } - - @Override - public List getWorkerLostStorage() { - List workerLostStorageList = new ArrayList<>(); - for (MasterWorkerInfo worker : mWorkers) { - try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.USAGE), true)) { - if (worker.hasLostStorage()) { - Map lostStorage = worker.getLostStorage().entrySet() - .stream().collect(Collectors.toMap(Map.Entry::getKey, - e -> StorageList.newBuilder().addAllStorage(e.getValue()).build())); - workerLostStorageList.add(WorkerLostStorageInfo.newBuilder() - .setAddress(GrpcUtils.toProto(worker.getWorkerAddress())) - .putAllLostStorage(lostStorage).build()); - } - } - } - return workerLostStorageList; - } - - @Override - public void removeBlocks(Collection blockIds, boolean delete) throws UnavailableException { - try (JournalContext journalContext = createJournalContext()) { - for (long blockId : blockIds) { - Set workerIds; - try (LockResource r = lockBlock(blockId)) { - Optional block = mBlockMetaStore.getBlock(blockId); - if (!block.isPresent()) { - continue; - } - List locations = mBlockMetaStore.getLocations(blockId); - workerIds = new HashSet<>(locations.size()); - for (BlockLocation loc : locations) { - workerIds.add(loc.getWorkerId()); - } - // Two cases here: - // 1) For delete: delete the block metadata. - // 2) For free: keep the block metadata. mLostBlocks will be changed in - // processWorkerRemovedBlocks - if (delete) { - // Make sure blockId is removed from mLostBlocks when the block metadata is deleted. - // Otherwise blockId in mLostBlock can be dangling index if the metadata is gone. - mLostBlocks.remove(blockId); - mBlockMetaStore.removeBlock(blockId); - JournalEntry entry = JournalEntry.newBuilder() - .setDeleteBlock(DeleteBlockEntry.newBuilder().setBlockId(blockId)).build(); - journalContext.append(entry); - } - } - - // Outside of locking the block. This does not have to be synchronized with the block - // metadata, since it is essentially an asynchronous signal to the worker to remove the - // block. - // TODO(jiacheng): if the block locations are changed (like a new worker is registered - // with the block), the block will not be freed ever. The locking logic in - // workerRegister should be changed to address this race condition. - for (long workerId : workerIds) { - MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId); - if (worker != null) { - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.BLOCKS), false)) { - worker.updateToRemovedBlock(true, blockId); - } - } - } - } - } - } - - @Override - public void decommissionWorker(long workerId) - throws Exception { - //TODO(Tony Sun): added in another pr. - } - - @Override - public void validateBlocks(Function validator, boolean repair) - throws UnavailableException { - long scanLimit = Configuration.getInt(PropertyKey.MASTER_BLOCK_SCAN_INVALID_BATCH_MAX_SIZE); - List invalidBlocks = new ArrayList<>(); - try (CloseableIterator iter = mBlockMetaStore.getCloseableIterator()) { - while (iter.hasNext() && (invalidBlocks.size() < scanLimit || scanLimit < 0)) { - long id = iter.next().getId(); - if (!validator.apply(id)) { - invalidBlocks.add(id); - } - } - } - if (!invalidBlocks.isEmpty()) { - long limit = 100; - List loggedBlocks = invalidBlocks.stream().limit(limit).collect(Collectors.toList()); - LOG.warn("Found {} orphan blocks without corresponding file metadata.", invalidBlocks.size()); - if (invalidBlocks.size() > limit) { - LOG.warn("The first {} orphan blocks include {}.", limit, loggedBlocks); - } else { - LOG.warn("The orphan blocks include {}.", loggedBlocks); - } - if (repair) { - LOG.warn("Deleting {} orphan blocks.", invalidBlocks.size()); - removeBlocks(invalidBlocks, true); - } else { - LOG.warn("Restart Alluxio master with {}=true to delete the blocks and repair the system.", - PropertyKey.Name.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED); - } - } - } - - /** - * @return mJournaledNextContainerId - */ - @Override - public long getJournaledNextContainerId() { - return mJournaledNextContainerId; - } - - /** - * @return a new block container id - */ - @Override - public long getNewContainerId() throws UnavailableException { - long containerId = mBlockContainerIdGenerator.getNewContainerId(); - if (containerId >= (mJournaledNextContainerId - mContainerIdReservationSize / 2)) { - if (containerId >= mJournaledNextContainerId) { - synchronized (mBlockContainerIdGenerator) { - // This container id is not safe with respect to the last journaled container id. - // Therefore, journal the new state of the container id. This implies that when a master - // crashes, the container ids within the reservation which have not been used yet will - // never be used. This is a tradeoff between fully utilizing the container id space, vs. - // improving master scalability. - - // Set the next id to journal with a reservation of container ids, to avoid having to - // write to the journal for ids within the reservation. - long possibleMaxContainerId = mBlockContainerIdGenerator.getNextContainerId(); - if (possibleMaxContainerId >= mJournaledNextContainerId) { - mJournaledNextContainerId = possibleMaxContainerId + mContainerIdReservationSize; - try (JournalContext journalContext = createJournalContext()) { - // This must be flushed while holding the lock on mBlockContainerIdGenerator, in - // order to prevent subsequent calls to return ids that have not been journaled - // and flushed. - journalContext.append(getContainerIdJournalEntry()); - } - } - } - } else { - if (mContainerIdDetectorIsIdle) { - synchronized (mBlockContainerIdGenerator) { - if (mContainerIdDetectorIsIdle) { - mContainerIdDetectorIsIdle = false; - mContainerIdDetector.submit(() -> { - try { - synchronized (mBlockContainerIdGenerator) { - long possibleMaxContainerId = mBlockContainerIdGenerator.getNextContainerId(); - - if (possibleMaxContainerId - >= (mJournaledNextContainerId - mContainerIdReservationSize / 2)) { - mJournaledNextContainerId = possibleMaxContainerId - + mContainerIdReservationSize; - try (JournalContext journalContext = createJournalContext()) { - journalContext.append(getContainerIdJournalEntry()); - } - } - } - } catch (UnavailableException e) { - LOG.error("Container Id Detector failed", e); - } - - mContainerIdDetectorIsIdle = true; - }); - } - } - } - } - } - - return containerId; - } - - /** - * @return a {@link JournalEntry} representing the state of the container id generator - */ - private JournalEntry getContainerIdJournalEntry() { - synchronized (mBlockContainerIdGenerator) { - BlockContainerIdGeneratorEntry blockContainerIdGenerator = - BlockContainerIdGeneratorEntry.newBuilder().setNextContainerId(mJournaledNextContainerId) - .build(); - return JournalEntry.newBuilder().setBlockContainerIdGenerator(blockContainerIdGenerator) - .build(); - } - } - - // TODO(binfan): check the logic is correct or not when commitBlock is a retry - @Override - public void commitBlock(long workerId, long usedBytesOnTier, String tierAlias, - String mediumType, long blockId, long length) - throws NotFoundException, UnavailableException { - LOG.debug("Commit block from workerId: {}, usedBytesOnTier: {}, blockId: {}, length: {}", - workerId, usedBytesOnTier, blockId, length); - - MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId); - // TODO(peis): Check lost workers as well. - if (worker == null) { - throw new NotFoundException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId)); - } - - try (JournalContext journalContext = createJournalContext()) { - // Lock the worker metadata here to preserve the lock order - // The worker metadata must be locked before the blocks - try (LockResource lr = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.USAGE, WorkerMetaLockSection.BLOCKS), false)) { - try (LockResource r = lockBlock(blockId)) { - Optional block = mBlockMetaStore.getBlock(blockId); - if (!block.isPresent() || block.get().getLength() != length) { - if (block.isPresent() && block.get().getLength() != Constants.UNKNOWN_SIZE) { - LOG.warn("Rejecting attempt to change block length from {} to {}", - block.get().getLength(), length); - } else { - mBlockMetaStore.putBlock(blockId, BlockMeta.newBuilder().setLength(length).build()); - BlockInfoEntry blockInfo = - BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build(); - journalContext.append(JournalEntry.newBuilder().setBlockInfo(blockInfo).build()); - } - } - // Update the block metadata with the new worker location. - mBlockMetaStore.addLocation(blockId, BlockLocation.newBuilder() - .setWorkerId(workerId) - .setTier(tierAlias) - .setMediumType(mediumType) - .build()); - // This worker has this block, so it is no longer lost. - mLostBlocks.remove(blockId); - - // Update the worker information for this new block. - // TODO(binfan): when retry commitBlock on master is expected, make sure metrics are not - // double counted. - worker.addBlock(blockId); - worker.updateUsedBytes(tierAlias, usedBytesOnTier); - } - } - - worker.updateLastUpdatedTimeMs(); - } - } - - @Override - public void commitBlockInUFS(long blockId, long length) throws UnavailableException { - LOG.debug("Commit block in ufs. blockId: {}, length: {}", blockId, length); - try (JournalContext journalContext = createJournalContext(); - LockResource r = lockBlock(blockId)) { - if (mBlockMetaStore.getBlock(blockId).isPresent()) { - // Block metadata already exists, so do not need to create a new one. - return; - } - mBlockMetaStore.putBlock(blockId, BlockMeta.newBuilder().setLength(length).build()); - BlockInfoEntry blockInfo = - BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build(); - journalContext.append(JournalEntry.newBuilder().setBlockInfo(blockInfo).build()); - } - } - - @Override - public BlockInfo getBlockInfo(long blockId) throws BlockInfoException, UnavailableException { - return generateBlockInfo(blockId) - .orElseThrow(() -> new BlockInfoException(ExceptionMessage.BLOCK_META_NOT_FOUND, blockId)); - } - - @Override - public List getBlockInfoList(List blockIds) throws UnavailableException { - List ret = new ArrayList<>(blockIds.size()); - for (long blockId : blockIds) { - generateBlockInfo(blockId).ifPresent(ret::add); - } - return ret; - } - - @Override - public Map getTotalBytesOnTiers() { - Map ret = new HashMap<>(); - for (MasterWorkerInfo worker : mWorkers) { - try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.USAGE), true)) { - for (Map.Entry entry : worker.getTotalBytesOnTiers().entrySet()) { - Long total = ret.get(entry.getKey()); - ret.put(entry.getKey(), (total == null ? 0L : total) + entry.getValue()); - } - } - } - return ret; - } - - @Override - public Map getUsedBytesOnTiers() { - Map ret = new HashMap<>(); - for (MasterWorkerInfo worker : mWorkers) { - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.USAGE), true)) { - for (Map.Entry entry : worker.getUsedBytesOnTiers().entrySet()) { - Long used = ret.get(entry.getKey()); - ret.put(entry.getKey(), (used == null ? 0L : used) + entry.getValue()); - } - } - } - return ret; - } - - /** - * Find a worker which is considered lost or just gets its id. - * @param workerNetAddress the address used to find a worker - * @return a {@link MasterWorkerInfo} which is presented in master but not registered, - * or null if not worker is found. - */ - @Nullable - private MasterWorkerInfo findUnregisteredWorker(WorkerNetAddress workerNetAddress) { - for (IndexedSet workers: Arrays.asList(mTempWorkers, - mLostWorkers, mDecommissionedWorkers)) { - MasterWorkerInfo worker = workers.getFirstByField(ADDRESS_INDEX, workerNetAddress); - if (worker != null) { - return worker; - } - } - return null; - } - - /** - * Find a worker which is considered lost or just gets its id. - * @param workerId the id used to find a worker - * @return a {@link MasterWorkerInfo} which is presented in master but not registered, - * or null if not worker is found. - */ - @Nullable - private MasterWorkerInfo findUnregisteredWorker(long workerId) { - for (IndexedSet workers: Arrays.asList(mTempWorkers, - mLostWorkers, mDecommissionedWorkers)) { - MasterWorkerInfo worker = workers.getFirstByField(ID_INDEX, workerId); - if (worker != null) { - return worker; - } - } - return null; - } - - /** - * Re-register a lost worker or complete registration after getting a worker id. - * This method requires no locking on {@link MasterWorkerInfo} because it is only - * reading final fields. - * - * @param workerId the worker id to register - */ - @Nullable - private MasterWorkerInfo recordWorkerRegistration(long workerId) { - for (IndexedSet workers: Arrays.asList(mTempWorkers, - mLostWorkers, mDecommissionedWorkers)) { - MasterWorkerInfo worker = workers.getFirstByField(ID_INDEX, workerId); - if (worker == null) { - continue; - } - - mWorkers.add(worker); - workers.remove(worker); - if (workers == mLostWorkers) { - for (Consumer
function : mLostWorkerFoundListeners) { - // The worker address is final, no need for locking here - function.accept(new Address(worker.getWorkerAddress().getHost(), - worker.getWorkerAddress().getRpcPort())); - } - LOG.warn("A lost worker {} has requested its old id {}.", - worker.getWorkerAddress(), worker.getId()); - } - - return worker; - } - return null; - } - - @Override - public long getWorkerId(WorkerNetAddress workerNetAddress) { - MasterWorkerInfo existingWorker = mWorkers.getFirstByField(ADDRESS_INDEX, workerNetAddress); - if (existingWorker != null) { - // This worker address is already mapped to a worker id. - long oldWorkerId = existingWorker.getId(); - LOG.warn("The worker {} already exists as id {}.", workerNetAddress, oldWorkerId); - return oldWorkerId; - } - - existingWorker = findUnregisteredWorker(workerNetAddress); - if (existingWorker != null) { - return existingWorker.getId(); - } - - // Generate a new worker id. - long workerId = IdUtils.getRandomNonNegativeLong(); - while (!mTempWorkers.add(new MasterWorkerInfo(workerId, workerNetAddress))) { - workerId = IdUtils.getRandomNonNegativeLong(); - } - - LOG.info("getWorkerId(): WorkerNetAddress: {} id: {}", workerNetAddress, workerId); - return workerId; - } - - @Override - public Optional tryAcquireRegisterLease(GetRegisterLeasePRequest request) { - return mRegisterLeaseManager.tryAcquireLease(request); - } - - @Override - public boolean hasRegisterLease(long workerId) { - return mRegisterLeaseManager.hasLease(workerId); - } - - @Override - public void releaseRegisterLease(long workerId) { - mRegisterLeaseManager.releaseLease(workerId); - } - - @Override - public void workerRegister(long workerId, List storageTiers, - Map totalBytesOnTiers, Map usedBytesOnTiers, - Map> currentBlocksOnLocation, - Map lostStorage, RegisterWorkerPOptions options) - throws NotFoundException { - - MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId); - - if (worker == null) { - worker = findUnregisteredWorker(workerId); - } - - if (worker == null) { - throw new NotFoundException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId)); - } - - worker.setBuildVersion(options.getBuildVersion()); - - // Gather all blocks on this worker. - int totalSize = currentBlocksOnLocation.values().stream().mapToInt(List::size).sum(); - Set blocks = new LongOpenHashSet(totalSize); - for (List blockIds : currentBlocksOnLocation.values()) { - blocks.addAll(blockIds); - } - - // Lock all the locks - try (LockResource r = worker.lockWorkerMeta(EnumSet.of( - WorkerMetaLockSection.STATUS, - WorkerMetaLockSection.USAGE, - WorkerMetaLockSection.BLOCKS), false)) { - // Detect any lost blocks on this worker. - Set removedBlocks = worker.register(MASTER_STORAGE_TIER_ASSOC, storageTiers, - totalBytesOnTiers, usedBytesOnTiers, blocks); - processWorkerRemovedBlocks(worker, removedBlocks, false); - processWorkerAddedBlocks(worker, currentBlocksOnLocation); - processWorkerOrphanedBlocks(worker); - worker.addLostStorage(lostStorage); - } - - if (options.getConfigsCount() > 0) { - for (BiConsumer> function : mWorkerRegisteredListeners) { - WorkerNetAddress workerAddress = worker.getWorkerAddress(); - function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), - options.getConfigsList()); - } - } - - recordWorkerRegistration(workerId); - - // Update the TS at the end of the process - worker.updateLastUpdatedTimeMs(); - - // Invalidate cache to trigger new build of worker info list - mWorkerInfoCache.invalidate(WORKER_INFO_CACHE_KEY); - LOG.info("registerWorker(): {}", worker); - } - - @Override - public MasterWorkerInfo getWorker(long workerId) throws NotFoundException { - MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId); - - if (worker == null) { - worker = findUnregisteredWorker(workerId); - } - - if (worker == null) { - throw new NotFoundException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId)); - } - - return worker; - } - - @Override - public void workerRegisterStream(WorkerRegisterContext context, - RegisterWorkerPRequest chunk, boolean isFirstMsg) { - if (isFirstMsg) { - workerRegisterStart(context, chunk); - } else { - workerRegisterBatch(context, chunk); - } - } - - protected void workerRegisterStart(WorkerRegisterContext context, - RegisterWorkerPRequest chunk) { - final List storageTiers = chunk.getStorageTiersList(); - final Map totalBytesOnTiers = chunk.getTotalBytesOnTiersMap(); - final Map usedBytesOnTiers = chunk.getUsedBytesOnTiersMap(); - final Map lostStorage = chunk.getLostStorageMap(); - - final Map> currentBlocksOnLocation = - BlockMasterWorkerServiceHandler.reconstructBlocksOnLocationMap( - chunk.getCurrentBlocksList(), context.getWorkerId()); - RegisterWorkerPOptions options = chunk.getOptions(); - - MasterWorkerInfo workerInfo = context.getWorkerInfo(); - Preconditions.checkState(workerInfo != null, - "No workerInfo metadata found in the WorkerRegisterContext!"); - mActiveRegisterContexts.put(workerInfo.getId(), context); - - // The workerInfo is locked so we can operate on its blocks without race conditions - // We start with assuming all blocks in (mBlocks + mToRemoveBlocks) do not exist. - // With each batch we receive, we mark them not-to-be-removed. - // Eventually what's left in the mToRemove will be the ones that do not exist anymore. - workerInfo.markAllBlocksToRemove(); - workerInfo.updateUsage(MASTER_STORAGE_TIER_ASSOC, storageTiers, - totalBytesOnTiers, usedBytesOnTiers); - processWorkerAddedBlocks(workerInfo, currentBlocksOnLocation); - processWorkerOrphanedBlocks(workerInfo); - workerInfo.addLostStorage(lostStorage); - workerInfo.setBuildVersion(options.getBuildVersion()); - - // TODO(jiacheng): This block can be moved to a non-locked section - if (options.getConfigsCount() > 0) { - for (BiConsumer> function : mWorkerRegisteredListeners) { - WorkerNetAddress workerAddress = workerInfo.getWorkerAddress(); - function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), - options.getConfigsList()); - } - } - } - - protected void workerRegisterBatch(WorkerRegisterContext context, RegisterWorkerPRequest chunk) { - final Map> currentBlocksOnLocation = - BlockMasterWorkerServiceHandler.reconstructBlocksOnLocationMap( - chunk.getCurrentBlocksList(), context.getWorkerId()); - MasterWorkerInfo workerInfo = context.getWorkerInfo(); - Preconditions.checkState(workerInfo != null, - "No workerInfo metadata found in the WorkerRegisterContext!"); - - // Even if we add the BlockLocation before the workerInfo is fully registered, - // it should be fine because the block can be read on this workerInfo. - // If the stream fails in the middle, the blocks recorded on the MasterWorkerInfo - // will be removed by processLostWorker() - processWorkerAddedBlocks(workerInfo, currentBlocksOnLocation); - - processWorkerOrphanedBlocks(workerInfo); - - // Update the TS at the end of the process - workerInfo.updateLastUpdatedTimeMs(); - } - - @Override - public void workerRegisterFinish(WorkerRegisterContext context) { - MasterWorkerInfo workerInfo = context.getWorkerInfo(); - Preconditions.checkState(workerInfo != null, - "No workerInfo metadata found in the WorkerRegisterContext!"); - - // Detect any lost blocks on this workerInfo. - Set removedBlocks; - if (workerInfo.mIsRegistered) { - // This is a re-register of an existing workerInfo. Assume the new block ownership data is - // more up-to-date and update the existing block information. - LOG.info("re-registering an existing workerId: {}", workerInfo.getId()); - - // The toRemoveBlocks field now contains all the updates - // after all the blocks have been processed. - removedBlocks = workerInfo.getToRemoveBlocks(); - } else { - removedBlocks = Collections.emptySet(); - } - LOG.info("Found {} blocks to remove from the workerInfo", removedBlocks.size()); - processWorkerRemovedBlocks(workerInfo, removedBlocks, true); - - // Mark registered successfully - workerInfo.mIsRegistered = true; - recordWorkerRegistration(workerInfo.getId()); - - // Update the TS at the end of the process - workerInfo.updateLastUpdatedTimeMs(); - - // Invalidate cache to trigger new build of workerInfo info list - mWorkerInfoCache.invalidate(WORKER_INFO_CACHE_KEY); - LOG.info("Worker successfully registered: {}", workerInfo); - mActiveRegisterContexts.remove(workerInfo.getId()); - mRegisterLeaseManager.releaseLease(workerInfo.getId()); - } - - @Override - public Command workerHeartbeat(long workerId, Map capacityBytesOnTiers, - Map usedBytesOnTiers, List removedBlockIds, - Map> addedBlocks, - Map lostStorage, - List metrics) { - MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId); - if (worker == null) { - LOG.warn("Could not find worker id: {} for heartbeat.", workerId); - return Command.newBuilder().setCommandType(CommandType.Register).build(); - } - - // Update the TS before the heartbeat so even if the worker heartbeat processing - // is time-consuming or triggers GC, the worker does not get marked as lost - // by the LostWorkerDetectionHeartbeatExecutor - worker.updateLastUpdatedTimeMs(); - - // The address is final, no need for locking - processWorkerMetrics(worker.getWorkerAddress().getHost(), metrics); - - Command workerCommand = null; - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.USAGE, WorkerMetaLockSection.BLOCKS), false)) { - worker.addLostStorage(lostStorage); - - if (capacityBytesOnTiers != null) { - worker.updateCapacityBytes(capacityBytesOnTiers); - } - worker.updateUsedBytes(usedBytesOnTiers); - - // Technically, 'worker' should be confirmed to still be in the data structure. Lost worker - // detection can remove it. However, we are intentionally ignoring this race, since the worker - // will just re-register regardless. - - processWorkerRemovedBlocks(worker, removedBlockIds, false); - processWorkerAddedBlocks(worker, addedBlocks); - Set toRemoveBlocks = worker.getToRemoveBlocks(); - if (toRemoveBlocks.isEmpty()) { - workerCommand = Command.newBuilder().setCommandType(CommandType.Nothing).build(); - } else { - workerCommand = Command.newBuilder().setCommandType(CommandType.Free) - .addAllData(toRemoveBlocks).build(); - } - } - - // Update the TS again - worker.updateLastUpdatedTimeMs(); - - // Should not reach here - Preconditions.checkNotNull(workerCommand, "Worker heartbeat response command is null!"); - - return workerCommand; - } - - @Override - public Clock getClock() { - return mClock; - } - - private void processWorkerMetrics(String hostname, List metrics) { - if (metrics.isEmpty()) { - return; - } - mMetricsMaster.workerHeartbeat(hostname, metrics); - } - - /** - * Updates the worker and block metadata for blocks removed from a worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * An exclusive lock is required. - * - * @param workerInfo The worker metadata object - * @param removedBlockIds A list of block ids removed from the worker - */ - private void processWorkerRemovedBlocks(MasterWorkerInfo workerInfo, - Collection removedBlockIds, boolean sendCommand) { - for (long removedBlockId : removedBlockIds) { - try (LockResource r = lockBlock(removedBlockId)) { - Optional block = mBlockMetaStore.getBlock(removedBlockId); - if (block.isPresent()) { - LOG.debug("Block {} is removed on worker {}.", removedBlockId, workerInfo.getId()); - mBlockMetaStore.removeLocation(removedBlockId, workerInfo.getId()); - if (mBlockMetaStore.getLocations(removedBlockId).size() == 0) { - mLostBlocks.add(removedBlockId); - } - } - // Remove the block even if its metadata has been deleted already. - if (sendCommand) { - workerInfo.scheduleRemoveFromWorker(removedBlockId); - } else { - workerInfo.removeBlockFromWorkerMeta(removedBlockId); - } - } - } - } - - /** - * Updates the worker and block metadata for blocks added to a worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * An exclusive lock is required. - * - * @param workerInfo The worker metadata object - * @param addedBlockIds A mapping from storage tier alias to a list of block ids added - */ - private void processWorkerAddedBlocks(MasterWorkerInfo workerInfo, - Map> addedBlockIds) { - long invalidBlockCount = 0; - for (Map.Entry> entry : addedBlockIds.entrySet()) { - for (long blockId : entry.getValue()) { - try (LockResource r = lockBlock(blockId)) { - Optional block = mBlockMetaStore.getBlock(blockId); - if (block.isPresent()) { - workerInfo.addBlock(blockId); - BlockLocation location = entry.getKey(); - Preconditions.checkState(location.getWorkerId() == workerInfo.getId(), - "BlockLocation has a different workerId %s from the request sender's workerId %s", - location.getWorkerId(), workerInfo.getId()); - mBlockMetaStore.addLocation(blockId, location); - mLostBlocks.remove(blockId); - } else { - invalidBlockCount++; - // The block is not recognized and should therefore be purged from the worker - // The file may have been removed when the worker was lost - workerInfo.scheduleRemoveFromWorker(blockId); - LOG.debug("Invalid block: {} from worker {}.", blockId, - workerInfo.getWorkerAddress().getHost()); - } - } - } - } - if (invalidBlockCount > 0) { - LOG.warn("{} invalid blocks found on worker {} in total", invalidBlockCount, - workerInfo.getWorkerAddress().getHost()); - } - } - - /** - * Checks the blocks on the worker. For blocks not present in Alluxio anymore, - * they will be marked to-be-removed from the worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * A shared lock is required. - * - * @param workerInfo The worker metadata object - */ - private void processWorkerOrphanedBlocks(MasterWorkerInfo workerInfo) { - long orphanedBlockCount = 0; - for (long block : workerInfo.getBlocks()) { - if (!mBlockMetaStore.getBlock(block).isPresent()) { - orphanedBlockCount++; - LOG.debug("Requesting delete for orphaned block: {} from worker {}.", block, - workerInfo.getWorkerAddress().getHost()); - workerInfo.updateToRemovedBlock(true, block); - } - } - if (orphanedBlockCount > 0) { - LOG.warn("{} blocks marked as orphaned from worker {}", orphanedBlockCount, - workerInfo.getWorkerAddress().getHost()); - } - } - - @Override - public boolean isBlockLost(long blockId) { - return mLostBlocks.contains(blockId); - } - - @Override - public Iterator getLostBlocksIterator() { - return mLostBlocks.iterator(); - } - - @Override - public int getLostBlocksCount() { - return mLostBlocks.size(); - } - - private long getToRemoveBlockCount() { - long ret = 0; - for (MasterWorkerInfo worker : mWorkers) { - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.BLOCKS), true)) { - ret += worker.getToRemoveBlockCount(); - } - } - return ret; - } - - /** - * Generates block info, including worker locations, for a block id. - * This requires no locks on the {@link MasterWorkerInfo} because it is only reading - * final fields. - * - * @param blockId a block id - * @return optional block info, empty if the block does not exist - */ - private Optional generateBlockInfo(long blockId) throws UnavailableException { - if (mSafeModeManager.isInSafeMode()) { - throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage()); - } - - BlockMeta block; - List blockLocations; - try (LockResource r = lockBlock(blockId)) { - Optional blockOpt = mBlockMetaStore.getBlock(blockId); - if (!blockOpt.isPresent()) { - return Optional.empty(); - } - block = blockOpt.get(); - blockLocations = new ArrayList<>(mBlockMetaStore.getLocations(blockId)); - } - - // Sort the block locations by their alias ordinal in the master storage tier mapping - blockLocations.sort(Comparator.comparingInt( - o -> MASTER_STORAGE_TIER_ASSOC.getOrdinal(o.getTier()))); - - List locations = new ArrayList<>(blockLocations.size()); - for (BlockLocation location : blockLocations) { - MasterWorkerInfo workerInfo = - mWorkers.getFirstByField(ID_INDEX, location.getWorkerId()); - if (workerInfo != null) { - // worker metadata is intentionally not locked here because: - // - it would be an incorrect order (correct order is lock worker first, then block) - // - only uses getters of final variables - locations.add(new alluxio.wire.BlockLocation().setWorkerId(location.getWorkerId()) - .setWorkerAddress(workerInfo.getWorkerAddress()) - .setTierAlias(location.getTier()).setMediumType(location.getMediumType())); - } - } - return Optional.of( - new BlockInfo().setBlockId(blockId).setLength(block.getLength()).setLocations(locations)); - } - - @Override - public void reportLostBlocks(List blockIds) { - mLostBlocks.addAll(blockIds); - } - - @Override - public Set> getDependencies() { - return DEPS; - } - - /** - * Lost worker periodic check. - */ - public final class LostWorkerDetectionHeartbeatExecutor implements HeartbeatExecutor { - - /** - * Constructs a new {@link LostWorkerDetectionHeartbeatExecutor}. - */ - public LostWorkerDetectionHeartbeatExecutor() {} - - @Override - public void heartbeat() { - long masterWorkerTimeoutMs = Configuration.getMs(PropertyKey.MASTER_WORKER_TIMEOUT_MS); - long masterWorkerDeleteTimeoutMs = - Configuration.getMs(PropertyKey.MASTER_LOST_WORKER_DELETION_TIMEOUT_MS); - for (MasterWorkerInfo worker : mWorkers) { - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.BLOCKS), false)) { - // This is not locking because the field is atomic - final long lastUpdate = mClock.millis() - worker.getLastUpdatedTimeMs(); - if (lastUpdate > masterWorkerTimeoutMs) { - LOG.error("The worker {}({}) timed out after {}ms without a heartbeat!", worker.getId(), - worker.getWorkerAddress(), lastUpdate); - processLostWorker(worker); - } - } - } - for (MasterWorkerInfo worker : mLostWorkers) { - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.BLOCKS), false)) { - final long lastUpdate = mClock.millis() - worker.getLastUpdatedTimeMs(); - if ((lastUpdate - masterWorkerTimeoutMs) > masterWorkerDeleteTimeoutMs) { - LOG.error("The worker {}({}) timed out after {}ms without a heartbeat! " - + "Master will forget about this worker.", worker.getId(), - worker.getWorkerAddress(), lastUpdate); - deleteWorkerMetadata(worker); - } - } - } - } - - @Override - public void close() { - // Nothing to clean up - } - } - - /** - * Forces all workers to be lost. This should only be used for testing. - */ - @VisibleForTesting - public void forgetAllWorkers() { - for (MasterWorkerInfo worker : mWorkers) { - try (LockResource r = worker.lockWorkerMeta( - EnumSet.of(WorkerMetaLockSection.BLOCKS), false)) { - processLostWorker(worker); - } - } - } - - /** - * Updates the metadata for the specified lost worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * An exclusive lock is required. - * - * @param worker the worker metadata - */ - private void processLostWorker(MasterWorkerInfo worker) { - mLostWorkers.add(worker); - mWorkers.remove(worker); - WorkerNetAddress workerAddress = worker.getWorkerAddress(); - for (Consumer
function : mWorkerLostListeners) { - function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort())); - } - // We only remove the blocks from master locations but do not - // mark these blocks to-remove from the worker. - // So if the worker comes back again the blocks are kept. - processWorkerRemovedBlocks(worker, worker.getBlocks(), false); - } - - private void deleteWorkerMetadata(MasterWorkerInfo worker) { - mWorkers.remove(worker); - mLostWorkers.remove(worker); - mTempWorkers.remove(worker); - WorkerNetAddress workerAddress = worker.getWorkerAddress(); - for (Consumer
function : mWorkerDeleteListeners) { - function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort())); - } - } - - private void processFreedWorker(MasterWorkerInfo worker) { - mDecommissionedWorkers.remove(worker); - } - - LockResource lockBlock(long blockId) { - return new LockResource(mBlockLocks.get(blockId)); - } - - /** - * Selects the MasterWorkerInfo from workerInfoSet whose host or related IP address - * exists in addresses. - * - * @param addresses the address set that user passed in - * @param workerInfoSet the MasterWorkerInfo set to select info from - * @param workerNames the supported worker names - */ - private Set selectInfoByAddress(Set addresses, - Set workerInfoSet, Set workerNames) { - return workerInfoSet.stream().filter(info -> { - String host = info.getWorkerAddress().getHost(); - workerNames.add(host); - - String ip = null; - try { - ip = NetworkAddressUtils.resolveIpAddress(host); - workerNames.add(ip); - } catch (UnknownHostException e) { - // The host may already be an IP address - } - - if (addresses.contains(host)) { - addresses.remove(host); - return true; - } - - if (ip != null) { - if (addresses.contains(ip)) { - addresses.remove(ip); - return true; - } - } - return false; - }).collect(Collectors.toSet()); - } - - @Override - public void registerLostWorkerFoundListener(Consumer
function) { - mLostWorkerFoundListeners.add(function); - } - - @Override - public void registerWorkerLostListener(Consumer
function) { - mWorkerLostListeners.add(function); - } - - @Override - public void registerWorkerDeleteListener(Consumer
function) { - mWorkerDeleteListeners.add(function); - } - - @Override - public void registerNewWorkerConfListener(BiConsumer> function) { - mWorkerRegisteredListeners.add(function); - } - - /** - * Class that contains metrics related to BlockMaster. - */ - public static final class Metrics { - /** - * Registers metric gauges. - * - * @param master the block master handle - */ - @VisibleForTesting - public static void registerGauges(final DefaultBlockMaster master) { - MetricsSystem.registerGaugeIfAbsent(MetricKey.CLUSTER_CAPACITY_TOTAL.getName(), - master::getCapacityBytes); - - MetricsSystem.registerGaugeIfAbsent(MetricKey.CLUSTER_CAPACITY_USED.getName(), - master::getUsedBytes); - - MetricsSystem.registerGaugeIfAbsent(MetricKey.CLUSTER_CAPACITY_FREE.getName(), - () -> master.getCapacityBytes() - master.getUsedBytes()); - - MetricsSystem.registerGaugeIfAbsent(MetricKey.MASTER_UNIQUE_BLOCKS.getName(), - master::getUniqueBlockCount); - - MetricsSystem.registerGaugeIfAbsent(MetricKey.MASTER_TOTAL_BLOCK_REPLICA_COUNT.getName(), - master::getBlockReplicaCount); - for (int i = 0; i < master.getGlobalStorageTierAssoc().size(); i++) { - String alias = master.getGlobalStorageTierAssoc().getAlias(i); - // TODO(lu) Add template to dynamically construct metric key - MetricsSystem.registerGaugeIfAbsent( - MetricKey.CLUSTER_CAPACITY_TOTAL.getName() + MetricInfo.TIER + alias, - () -> master.getTotalBytesOnTiers().getOrDefault(alias, 0L)); - - MetricsSystem.registerGaugeIfAbsent( - MetricKey.CLUSTER_CAPACITY_USED.getName() + MetricInfo.TIER + alias, - () -> master.getUsedBytesOnTiers().getOrDefault(alias, 0L)); - MetricsSystem.registerGaugeIfAbsent( - MetricKey.CLUSTER_CAPACITY_FREE.getName() + MetricInfo.TIER + alias, - () -> master.getTotalBytesOnTiers().getOrDefault(alias, 0L) - - master.getUsedBytesOnTiers().getOrDefault(alias, 0L)); - } - - MetricsSystem.registerGaugeIfAbsent(MetricKey.CLUSTER_WORKERS.getName(), - master::getWorkerCount); - MetricsSystem.registerGaugeIfAbsent(MetricKey.CLUSTER_LOST_WORKERS.getName(), - master::getLostWorkerCount); - } - - private Metrics() {} // prevent instantiation - } -} diff --git a/core/server/master/src/main/java/alluxio/master/block/meta/MasterWorkerInfo.java b/core/server/master/src/main/java/alluxio/master/block/meta/MasterWorkerInfo.java deleted file mode 100644 index b8bde41ddd38..000000000000 --- a/core/server/master/src/main/java/alluxio/master/block/meta/MasterWorkerInfo.java +++ /dev/null @@ -1,733 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.block.meta; - -import alluxio.Constants; -import alluxio.StorageTierAssoc; -import alluxio.client.block.options.GetWorkerReportOptions; -import alluxio.client.block.options.GetWorkerReportOptions.WorkerInfoField; -import alluxio.grpc.BuildVersion; -import alluxio.grpc.StorageList; -import alluxio.master.block.DefaultBlockMaster; -import alluxio.resource.LockResource; -import alluxio.util.CommonUtils; -import alluxio.wire.WorkerInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; -import it.unimi.dsi.fastutil.longs.LongOpenHashSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.concurrent.locks.StampedLock; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class embeds all metadata for an Alluxio worker. - * This class is not thread safe, so external locking is required. - * - * There are multiple locks in this object, each guarding a group of metadata. - * - * The metadata fields are separated into a few different groups. - * Each group has its corresponding locking mechanism. - * The {@link MasterWorkerInfo} has the following groups of metadata: - * 1. Metadata like ID, address etc., represented by a {@link StaticWorkerMeta} object. - * This group is thread safe, meaning no locking is required. - * 2. Worker last updated timestamp. This is thread safe, meaning no locking is required. - * 3. Worker register status. This is guarded by a {@link StampedLock#asReadWriteLock()}. - * 4. Worker resource usage, represented by a {@link WorkerUsageMeta} object. - * This is guarded by a {@link StampedLock#asReadWriteLock()}. - * 5. Worker block lists, including the present blocks and blocks to be removed from the worker. - * This is guarded by a {@link StampedLock#asReadWriteLock()}. - * - * When accessing certain fields in this object, external locking is required. - * As listed above, group 1 and 2 are thread safe and do not require external locking. - * Group 3, 4, and 5 require external locking. - * - * Locking can be done with {@link #lockWorkerMeta(EnumSet, boolean)}. - * This method returns a {@link LockResource} which can be managed by try-finally. - * Internally, {@link WorkerMetaLock} is used to manage the corresponding internal locks - * with an order and unlocking them properly on close. - * - * If the fields are only read, shared locks should be acquired and released as below: - * 
- *   try (LockResource r = lockWorkerMeta(
- *       EnumSet.of(WorkerMetaLockSection.USAGE), true)) {
- *     ...
- *   }
- *
- * - * If the fields are updated, exclusive locks should be acquired and released as below: - * 
- *   try (LockResource r = lockWorkerMeta(
- *       EnumSet.of(
- *           WorkerMetaLockSection.STATUS,
- *           WorkerMetaLockSection.USAGE,
- *           WorkerMetaLockSection.BLOCKS)), true)) {
- *     ...
- *   }
- *
- * - * The locks are internally {@link StampedLock} which are NOT reentrant! - * We chose {@link StampedLock} instead of {@link ReentrantReadWriteLock} - * because the latter does not allow the write lock to be grabbed in one thread - * but later released in another thread. - * This is undesirable because when the worker registers in a stream, the thread - * that handles the 1st message will acquire the lock and the thread that handles - * the complete signal will be responsible for releasing the lock. - * In the current gRPC architecture it is impossible to enforce the two threads - * to be the same. - * - * Because then locks are not reentrant, you must be extra careful NOT to - * acquire the lock while holding, because that will result in a deadlock! - * This is especially the case for write locks. - * - * The current write lock holders include the following: - * 1. In {@link DefaultBlockMaster}, the methods related to worker register/heartbeat, - * and block removal/commit. - * 2. In {@link alluxio.master.block.WorkerRegisterContext}, - * to write locks are held throughout the lifecycle. - * 3. In {@link DefaultBlockMaster.LostWorkerDetectionHeartbeatExecutor#heartbeat()} - */ -@NotThreadSafe -public final class MasterWorkerInfo { - private static final Logger LOG = LoggerFactory.getLogger(MasterWorkerInfo.class); - private static final String LIVE_WORKER_STATE = "In Service"; - private static final String LOST_WORKER_STATE = "Out of Service"; - - private static final EnumSet USAGE_INFO_FIELDS = - EnumSet.of(WorkerInfoField.WORKER_CAPACITY_BYTES, - WorkerInfoField.WORKER_CAPACITY_BYTES_ON_TIERS, - WorkerInfoField.WORKER_USED_BYTES, - WorkerInfoField.WORKER_USED_BYTES_ON_TIERS); - - /** Worker's last updated time in ms. */ - private final AtomicLong mLastUpdatedTimeMs; - /** Worker's build version (including version and revision). */ - private final AtomicReference mBuildVersion; - /** Worker metadata, this field is thread safe. */ - private final StaticWorkerMeta mMeta; - - /** If true, the worker is considered registered. */ - @GuardedBy("mStatusLock") - public boolean mIsRegistered; - /** Locks the worker register status. */ - private final ReadWriteLock mStatusLock; - - /** Worker usage data. */ - @GuardedBy("mUsageLock") - private final WorkerUsageMeta mUsage; - /** Locks the worker usage data. */ - private final ReadWriteLock mUsageLock; - - /** Ids of blocks the worker contains. */ - @GuardedBy("mBlockListLock") - private Set mBlocks; - /** Ids of blocks the worker should remove. */ - @GuardedBy("mBlockListLock") - private final Set mToRemoveBlocks; - /** Locks the 2 block sets above. */ - private final ReadWriteLock mBlockListLock; - - /** Stores the mapping from WorkerMetaLockSection to the lock. */ - private final Map mLockTypeToLock; - - /** - * Creates a new instance of {@link MasterWorkerInfo}. - * - * @param id the worker id to use - * @param address the worker address to use - */ - public MasterWorkerInfo(long id, WorkerNetAddress address) { - mMeta = new StaticWorkerMeta(id, address); - mUsage = new WorkerUsageMeta(); - mBlocks = new LongOpenHashSet(); - mToRemoveBlocks = new LongOpenHashSet(); - mLastUpdatedTimeMs = new AtomicLong(CommonUtils.getCurrentMs()); - mBuildVersion = new AtomicReference<>(BuildVersion.getDefaultInstance()); - - // Init all locks - mStatusLock = new StampedLock().asReadWriteLock(); - mUsageLock = new StampedLock().asReadWriteLock(); - mBlockListLock = new StampedLock().asReadWriteLock(); - mLockTypeToLock = ImmutableMap.of( - WorkerMetaLockSection.STATUS, mStatusLock, - WorkerMetaLockSection.USAGE, mUsageLock, - WorkerMetaLockSection.BLOCKS, mBlockListLock); - } - - /** - * Marks the worker as registered, while updating all of its metadata. - * Write locks on {@link MasterWorkerInfo#mStatusLock}, {@link MasterWorkerInfo#mUsageLock} - * and {@link MasterWorkerInfo#mBlockListLock} are required. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with all three lock types specified: - * - * 
-   *   try (LockResource r = worker.lockWorkerMeta(EnumSet.of(
-   *       WorkerMetaLockSection.STATUS,
-   *       WorkerMetaLockSection.USAGE,
-   *       WorkerMetaLockSection.BLOCKS), false)) {
-   *     register(...);
-   *   }
-   *
- * - * @param globalStorageTierAssoc global mapping between storage aliases and ordinal position - * @param storageTierAliases list of storage tier aliases in order of their position in the - * hierarchy - * @param totalBytesOnTiers mapping from storage tier alias to total bytes - * @param usedBytesOnTiers mapping from storage tier alias to used byes - * @param blocks set of block ids on this worker - * @return A Set of blocks removed (or lost) from this worker - */ - public Set register(final StorageTierAssoc globalStorageTierAssoc, - final List storageTierAliases, final Map totalBytesOnTiers, - final Map usedBytesOnTiers, final Set blocks) { - mUsage.updateUsage(globalStorageTierAssoc, storageTierAliases, - totalBytesOnTiers, usedBytesOnTiers); - - Set removedBlocks; - if (mIsRegistered) { - // This is a re-register of an existing worker. Assume the new block ownership data is more - // up-to-date and update the existing block information. - LOG.info("re-registering an existing workerId: {}", mMeta.mId); - - // Compute the difference between the existing block data, and the new data. - removedBlocks = Sets.difference(mBlocks, blocks); - } else { - removedBlocks = Collections.emptySet(); - } - - // Set the new block information. - mBlocks = blocks; - - mIsRegistered = true; - return removedBlocks; - } - - /** - * Adds a block to the worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * An exclusive lock is required. - * - * @param blockId the id of the block to be added - */ - public void addBlock(long blockId) { - mBlocks.add(blockId); - // This step is added because in the beginning of a stream register - // we mark all blocks to be removed - mToRemoveBlocks.remove(blockId); - } - - /** - * Removes a block from the worker. - * This is typically called when we know the block has been removed from the worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * An exclusive lock is required. - * - * @param blockId the id of the block to be removed - */ - public void removeBlockFromWorkerMeta(long blockId) { - mBlocks.remove(blockId); - mToRemoveBlocks.remove(blockId); - } - - /** - * Remove the block from the worker metadata and add to the to-remove list. - * The next worker heartbeat will issue the remove command to the worker - * so the block is deleted later. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * An exclusive lock is required. - * - * @param blockId the block ID - */ - public void scheduleRemoveFromWorker(long blockId) { - mBlocks.remove(blockId); - mToRemoveBlocks.add(blockId); - } - - /** - * Adds new worker lost storage paths. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * An exclusive lock is required. - * - * @param lostStorage the lost storage to add - */ - public void addLostStorage(Map lostStorage) { - for (Map.Entry entry : lostStorage.entrySet()) { - List paths = mUsage.mLostStorage.getOrDefault(entry.getKey(), new ArrayList<>()); - paths.addAll(entry.getValue().getStorageList()); - mUsage.mLostStorage.put(entry.getKey(), paths); - } - } - - /** - * Gets the selected field information for this worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMetaForInfo(Set)}. - * The required locks will be determined internally based on the fields. - * - * @param fieldRange the client selected fields - * @param isLiveWorker the worker is live or not - * @return generated worker information - */ - public WorkerInfo generateWorkerInfo(Set fieldRange, boolean isLiveWorker) { - WorkerInfo info = new WorkerInfo(); - for (WorkerInfoField field : fieldRange) { - switch (field) { - case ADDRESS: - info.setAddress(mMeta.mWorkerAddress); - break; - case BLOCK_COUNT: - info.setBlockCount(getBlockCount()); - break; - case WORKER_CAPACITY_BYTES: - info.setCapacityBytes(mUsage.mCapacityBytes); - break; - case WORKER_CAPACITY_BYTES_ON_TIERS: - info.setCapacityBytesOnTiers(mUsage.mTotalBytesOnTiers); - break; - case ID: - info.setId(mMeta.mId); - break; - case LAST_CONTACT_SEC: - info.setLastContactSec((int) ((CommonUtils.getCurrentMs() - - mLastUpdatedTimeMs.get()) / Constants.SECOND_MS)); - break; - case START_TIME_MS: - info.setStartTimeMs(mMeta.mStartTimeMs); - break; - case STATE: - if (isLiveWorker) { - info.setState(LIVE_WORKER_STATE); - } else { - info.setState(LOST_WORKER_STATE); - } - break; - case WORKER_USED_BYTES: - info.setUsedBytes(mUsage.mUsedBytes); - break; - case WORKER_USED_BYTES_ON_TIERS: - info.setUsedBytesOnTiers(mUsage.mUsedBytesOnTiers); - break; - case BUILD_VERSION: - BuildVersion v = mBuildVersion.get(); - info.setVersion(v.getVersion()); - info.setRevision(v.getRevision()); - break; - default: - LOG.warn("Unrecognized worker info field: " + field); - } - } - return info; - } - - /** - * {@link StaticWorkerMeta} is thread safe so the value can be read without locking. - * - * @return the worker's address - */ - public WorkerNetAddress getWorkerAddress() { - return mMeta.mWorkerAddress; - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * A shared lock is required. - * - * @return the available space of the worker in bytes - */ - public long getAvailableBytes() { - return mUsage.getAvailableBytes(); - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * A shared lock is required. - * - * This returns a copy so the lock can be released when this method returns. - * - * @return ids of all blocks the worker contains - */ - public Set getBlocks() { - return new LongOpenHashSet(mBlocks); - } - - /** - * Return the block count of this worker. - * - * @return the block count of this worker - */ - public int getBlockCount() { - return mBlocks.size(); - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * A shared lock is required. - * - * @return the capacity of the worker in bytes - */ - public long getCapacityBytes() { - return mUsage.mCapacityBytes; - } - - /** - * No locking required. - * - * @return the id of the worker - */ - public long getId() { - return mMeta.mId; - } - - /** - * No locking required. - * - * @return the last updated time of the worker in ms - */ - public long getLastUpdatedTimeMs() { - return mLastUpdatedTimeMs.get(); - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * A shared lock is required. - * - * This returns a copy so the lock can be released after this method returns. - * - * @return ids of blocks the worker should remove - */ - public Set getToRemoveBlocks() { - return new LongOpenHashSet(mToRemoveBlocks); - } - - /** - * @return used space of the worker in bytes - */ - public long getUsedBytes() { - return mUsage.mUsedBytes; - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * A shared lock is required. - * - * @return the total bytes on each storage tier - */ - public Map getTotalBytesOnTiers() { - return mUsage.mTotalBytesOnTiers; - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * A shared lock is required. - * - * @return the used bytes on each storage tier - */ - public Map getUsedBytesOnTiers() { - return mUsage.mUsedBytesOnTiers; - } - - /** - * No locking required. - * - * @return the start time in milliseconds - */ - public long getStartTime() { - return mMeta.mStartTimeMs; - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#STATUS} specified. - * A shared lock is required. - * - * @return whether the worker has been registered yet - */ - public boolean isRegistered() { - return mIsRegistered; - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * A shared lock is required. - * - * @return the free bytes on each storage tier - */ - public Map getFreeBytesOnTiers() { - Map freeCapacityBytes = new HashMap<>(); - for (Map.Entry entry : mUsage.mTotalBytesOnTiers.entrySet()) { - freeCapacityBytes.put(entry.getKey(), - entry.getValue() - mUsage.mUsedBytesOnTiers.get(entry.getKey())); - } - return freeCapacityBytes; - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * A shared lock is required. - * - * This returns a copy so the lock can be released after the map is returned. - * - * @return the map from tier alias to lost storage paths in this worker - */ - public Map> getLostStorage() { - return new HashMap<>(mUsage.mLostStorage); - } - - /** - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * A shared lock is required. - * - * @return true if this worker has lost storage, false otherwise - */ - public boolean hasLostStorage() { - return mUsage.mLostStorage.size() > 0; - } - - @Override - // TODO(jiacheng): Read lock on the conversion - public String toString() { - BuildVersion buildVersion = mBuildVersion.get(); - return MoreObjects.toStringHelper(this) - .add("id", mMeta.mId) - .add("workerAddress", mMeta.mWorkerAddress) - .add("capacityBytes", mUsage.mCapacityBytes) - .add("usedBytes", mUsage.mUsedBytes) - .add("lastUpdatedTimeMs", mLastUpdatedTimeMs.get()) - // We only show the number of blocks unless it is for DEBUG logs - .add("blocks", LOG.isDebugEnabled() ? mBlocks : CommonUtils.summarizeCollection(mBlocks)) - .add("lostStorage", mUsage.mLostStorage) - .add("version", buildVersion.getVersion()) - .add("revision", buildVersion.getRevision()).toString(); - } - - /** - * Updates the last updated time of the worker in ms. - * No locking is required. - */ - public void updateLastUpdatedTimeMs() { - mLastUpdatedTimeMs.set(CommonUtils.getCurrentMs()); - } - - /** - * Adds or removes a block from the to-be-removed blocks set of the worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * An exclusive lock is required. - * - * @param add true if to add, to remove otherwise - * @param blockId the id of the block to be added or removed - */ - public void updateToRemovedBlock(boolean add, long blockId) { - if (add) { - if (mBlocks.contains(blockId)) { - mToRemoveBlocks.add(blockId); - } - } else { - mToRemoveBlocks.remove(blockId); - } - } - - /** - * Sets the capacity of the worker in bytes. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * An exclusive lock is required. - * - * @param capacityBytesOnTiers used bytes on each storage tier - */ - public void updateCapacityBytes(Map capacityBytesOnTiers) { - long capacityBytes = 0; - mUsage.mTotalBytesOnTiers = capacityBytesOnTiers; - for (long t : mUsage.mTotalBytesOnTiers.values()) { - capacityBytes += t; - } - mUsage.mCapacityBytes = capacityBytes; - } - - /** - * Sets the used space of the worker in bytes. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * An exclusive lock is required. - * - * @param usedBytesOnTiers used bytes on each storage tier - */ - public void updateUsedBytes(Map usedBytesOnTiers) { - long usedBytes = 0; - mUsage.mUsedBytesOnTiers = new HashMap<>(usedBytesOnTiers); - for (long t : mUsage.mUsedBytesOnTiers.values()) { - usedBytes += t; - } - mUsage.mUsedBytes = usedBytes; - } - - /** - * Sets the used space of the worker in bytes. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * An exclusive lock is required. - * - * @param tierAlias alias of storage tier - * @param usedBytesOnTier used bytes on certain storage tier - */ - public void updateUsedBytes(String tierAlias, long usedBytesOnTier) { - mUsage.mUsedBytes += usedBytesOnTier - mUsage.mUsedBytesOnTiers.get(tierAlias); - mUsage.mUsedBytesOnTiers.put(tierAlias, usedBytesOnTier); - } - - ReadWriteLock getLock(WorkerMetaLockSection lockType) { - return mLockTypeToLock.get(lockType); - } - - /** - * Locks the corresponding locks on the metadata groups. - * See the javadoc for this class for example usage. - * - * The locks will be acquired in order and later released in the opposite order. - * The locks can either be shared or exclusive. - * The isShared flag will apply to all the locks acquired here. - * - * This returns a {@link LockResource} which can be managed by a try-finally block. - * See javadoc for {@link WorkerMetaLock} for more details about the internals. - * - * @param lockTypes the locks - * @param isShared if false, the locking is exclusive - * @return a {@link LockResource} of the {@link WorkerMetaLock} - */ - public LockResource lockWorkerMeta(EnumSet lockTypes, boolean isShared) { - return new LockResource(new WorkerMetaLock(lockTypes, isShared, this)); - } - - /** - * Returns the number of blocks that should be removed from the worker. - * - * @return the count - */ - @VisibleForTesting - public int getToRemoveBlockCount() { - return mToRemoveBlocks.size(); - } - - /** - * Updates the worker storage usage. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#USAGE} specified. - * An exclusive lock is required. - * - * @param globalStorageTierAssoc storage tier setup from configuration - * @param storageTiers the storage tiers - * @param totalBytesOnTiers the capacity of each tier - * @param usedBytesOnTiers the current usage of each tier - */ - public void updateUsage(StorageTierAssoc globalStorageTierAssoc, List storageTiers, - Map totalBytesOnTiers, Map usedBytesOnTiers) { - mUsage.updateUsage(globalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers); - } - - /** - * Marks all the blocks on the worker to be removed. - * This is called at the beginning of a register stream, where we do not know what blocks - * are no longer on the worker in {@link #mBlocks}. - * First all blocks will be marked to-be-removed, then in the stream when we see a block list, - * those blocks will be added to {@link #mBlocks} and removed from {@link #mToRemoveBlocks}. - * In this way, at the end of the stream, {@link #mToRemoveBlocks} contains only the blocks - * that no longer exist on the worker. - * - * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * with {@link WorkerMetaLockSection#BLOCKS} specified. - * An exclusive lock is required. - */ - public void markAllBlocksToRemove() { - mToRemoveBlocks.addAll(mBlocks); - } - - /** - * Finds the read locks necessary for required worker information. - * Locks the corresponding read locks for the specified worker information fields. - * This is a wrapper of {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)} - * - * @param fieldRange a set of {@link WorkerInfoField} - * @return a {@link LockResource} of the {@link WorkerMetaLock} - */ - public LockResource lockWorkerMetaForInfo( - Set fieldRange) { - EnumSet lockTypes = EnumSet.noneOf(WorkerMetaLockSection.class); - if (fieldRange.contains(GetWorkerReportOptions.WorkerInfoField.BLOCK_COUNT)) { - lockTypes.add(WorkerMetaLockSection.BLOCKS); - } - if (fieldRange.stream().anyMatch(USAGE_INFO_FIELDS::contains)) { - lockTypes.add(WorkerMetaLockSection.USAGE); - } - return lockWorkerMeta(lockTypes, true); - } - - /** - * Sets the build version of the worker. - * BuildVersion is reported by the worker in the register request. - * - * @param buildVersion the {@link BuildVersion} of the worker - */ - public void setBuildVersion(BuildVersion buildVersion) { - mBuildVersion.set(buildVersion); - } - - /** - * Get the build version of the worker. - * This is used to monitor cluster status when performing rolling upgrades. - * - * @return the {@link BuildVersion} of the worker - */ - public BuildVersion getBuildVersion() { - return mBuildVersion.get(); - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/AccessTimeUpdater.java b/core/server/master/src/main/java/alluxio/master/file/AccessTimeUpdater.java deleted file mode 100644 index 80b71258fdb6..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/AccessTimeUpdater.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.UnavailableException; -import alluxio.master.file.meta.Inode; -import alluxio.master.file.meta.InodeTree; -import alluxio.master.journal.JournalContext; -import alluxio.master.journal.JournalSystem; -import alluxio.master.journal.sink.JournalSink; -import alluxio.proto.journal.File.UpdateInodeEntry; -import alluxio.proto.journal.Journal; -import alluxio.resource.LockResource; -import alluxio.util.ThreadFactoryUtils; -import alluxio.util.ThreadUtils; - -import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class handles the update of inode last access time. - */ -@NotThreadSafe -final class AccessTimeUpdater implements JournalSink { - private static final Logger LOG = LoggerFactory.getLogger(AccessTimeUpdater.class); - - private final long mFlushInterval; - private final long mUpdatePrecision; - private final long mShutdownTimeout; - - private final FileSystemMaster mFileSystemMaster; - private final InodeTree mInodeTree; - - /** Keep track of all inodes that need access time update. */ - private final ConcurrentHashMap mAccessTimeUpdates; - private ScheduledExecutorService mExecutorService = null; - private final AtomicBoolean mUpdateScheduled = new AtomicBoolean(); - - /** - * Constructs a new {@link AccessTimeUpdater}. - */ - public AccessTimeUpdater(FileSystemMaster fileSystemMaster, InodeTree inodeTree, - JournalSystem journalSystem) { - this(fileSystemMaster, inodeTree, journalSystem, - Configuration.getMs(PropertyKey.MASTER_FILE_ACCESS_TIME_JOURNAL_FLUSH_INTERVAL), - Configuration.getMs(PropertyKey.MASTER_FILE_ACCESS_TIME_UPDATE_PRECISION), - Configuration.getMs(PropertyKey.MASTER_FILE_ACCESS_TIME_UPDATER_SHUTDOWN_TIMEOUT)); - } - - /** - * Constructs a new {@link AccessTimeUpdater} with time configurations. - */ - @VisibleForTesting - public AccessTimeUpdater(FileSystemMaster fileSystemMaster, InodeTree inodeTree, - JournalSystem journalSystem, long flushInterval, long updatePrecision, long shutdownTimeout) { - mFileSystemMaster = fileSystemMaster; - mInodeTree = inodeTree; - mAccessTimeUpdates = new ConcurrentHashMap<>(); - mFlushInterval = flushInterval; - mUpdatePrecision = updatePrecision; - mShutdownTimeout = shutdownTimeout; - journalSystem.addJournalSink(mFileSystemMaster, this); - } - - public void start() { - start(mFlushInterval > 0 ? Executors.newSingleThreadScheduledExecutor( - ThreadFactoryUtils.build("AccessTimeUpdater-%d", true)) : null); - } - - @VisibleForTesting - public synchronized void start(ScheduledExecutorService executorService) { - if (mExecutorService != null && mExecutorService != executorService - && !mExecutorService.isShutdown()) { - stop(); - } - mExecutorService = executorService; - } - - @Override - public void beforeShutdown() { - if (mExecutorService != null) { - flushUpdates(); - } - } - - public synchronized void stop() { - if (mExecutorService != null) { - ThreadUtils.shutdownAndAwaitTermination(mExecutorService, mShutdownTimeout); - } - } - - /** - * Update inode last access time. Requires at least read lock acquired for the inode. - * - * @param context the journal context - * @param inode the target inode - * @param opTimeMs the last access time to set on the inode - */ - public void updateAccessTime(JournalContext context, Inode inode, long opTimeMs) { - if (opTimeMs - inode.getLastAccessTimeMs() > mUpdatePrecision) { - try (LockResource lr = mInodeTree.getInodeLockManager().lockUpdate(inode.getId())) { - if (mExecutorService != null) { - // journal update asynchronously - UpdateInodeEntry entry = mInodeTree.updateInodeAccessTimeNoJournal(inode.getId(), - opTimeMs); - scheduleJournalUpdate(entry); - } else { - mInodeTree.updateInode(context, UpdateInodeEntry.newBuilder() - .setId(inode.getId()) - .setLastAccessTimeMs(opTimeMs) - .build()); - } - } - } - } - - private void scheduleJournalUpdate(UpdateInodeEntry entry) { - mAccessTimeUpdates.put(entry.getId(), entry.getLastAccessTimeMs()); - if (mUpdateScheduled.compareAndSet(false, true)) { - mExecutorService.schedule(this::flushScheduledUpdates, mFlushInterval, TimeUnit.MILLISECONDS); - } - } - - private void flushScheduledUpdates() { - mUpdateScheduled.set(false); - flushUpdates(); - } - - private void flushUpdates() { - try (JournalContext context = mFileSystemMaster.createJournalContext()) { - for (Iterator> iterator = mAccessTimeUpdates.entrySet().iterator(); - iterator.hasNext();) { - Map.Entry inodeEntry = iterator.next(); - iterator.remove(); - UpdateInodeEntry entry = UpdateInodeEntry.newBuilder() - .setId(inodeEntry.getKey()) - .setLastAccessTimeMs(inodeEntry.getValue()) - .build(); - context.append(Journal.JournalEntry.newBuilder().setUpdateInode(entry) - .build()); - } - } catch (UnavailableException e) { - LOG.debug("Failed to flush access time updates.", e); - } - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/BlockIntegrityChecker.java b/core/server/master/src/main/java/alluxio/master/file/BlockIntegrityChecker.java deleted file mode 100644 index 24334a592eb0..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/BlockIntegrityChecker.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.heartbeat.HeartbeatExecutor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Heartbeat executor for validating inode and block integrity. - */ -public final class BlockIntegrityChecker implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(BlockIntegrityChecker.class); - - private final FileSystemMaster mFileSystemMaster; - private final boolean mRepair; - - /** - * Constructs a block integrity checker based on the given {@link FileSystemMaster}. - * - * @param fsm the master to check - */ - public BlockIntegrityChecker(FileSystemMaster fsm) { - mFileSystemMaster = fsm; - mRepair = Configuration - .getBoolean(PropertyKey.MASTER_PERIODIC_BLOCK_INTEGRITY_CHECK_REPAIR); - } - - @Override - public void heartbeat() { - try { - mFileSystemMaster.validateInodeBlocks(mRepair); - } catch (Exception e) { - LOG.error("Failed to run periodic block integrity check.", e); - } - } - - @Override - public void close() { - // Nothing to clean up. - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/InodeTtlChecker.java b/core/server/master/src/main/java/alluxio/master/file/InodeTtlChecker.java deleted file mode 100644 index 76796c39b1fa..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/InodeTtlChecker.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.exception.FileDoesNotExistException; -import alluxio.grpc.DeletePOptions; -import alluxio.grpc.FreePOptions; -import alluxio.grpc.TtlAction; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.master.ProtobufUtils; -import alluxio.master.file.contexts.DeleteContext; -import alluxio.master.file.contexts.FreeContext; -import alluxio.master.file.meta.Inode; -import alluxio.master.file.meta.InodeTree; -import alluxio.master.file.meta.InodeTree.LockPattern; -import alluxio.master.file.meta.LockedInodePath; -import alluxio.master.file.meta.TtlBucket; -import alluxio.master.file.meta.TtlBucketList; -import alluxio.master.journal.JournalContext; -import alluxio.master.journal.NoopJournalContext; -import alluxio.proto.journal.File.UpdateInodeEntry; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Set; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class represents the executor for periodic inode ttl check. - */ -@NotThreadSafe -final class InodeTtlChecker implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(InodeTtlChecker.class); - - private final FileSystemMaster mFileSystemMaster; - private final InodeTree mInodeTree; - private final TtlBucketList mTtlBuckets; - - /** - * Constructs a new {@link InodeTtlChecker}. - */ - public InodeTtlChecker(FileSystemMaster fileSystemMaster, InodeTree inodeTree) { - mFileSystemMaster = fileSystemMaster; - mInodeTree = inodeTree; - mTtlBuckets = inodeTree.getTtlBuckets(); - } - - @Override - public void heartbeat() throws InterruptedException { - Set expiredBuckets = mTtlBuckets.getExpiredBuckets(System.currentTimeMillis()); - for (TtlBucket bucket : expiredBuckets) { - for (Inode inode : bucket.getInodes()) { - // Throw if interrupted. - if (Thread.interrupted()) { - throw new InterruptedException("InodeTtlChecker interrupted."); - } - AlluxioURI path = null; - try (LockedInodePath inodePath = - mInodeTree.lockFullInodePath( - inode.getId(), LockPattern.READ, NoopJournalContext.INSTANCE) - ) { - path = inodePath.getUri(); - } catch (FileDoesNotExistException e) { - // The inode has already been deleted, nothing needs to be done. - continue; - } catch (Exception e) { - LOG.error("Exception trying to clean up {} for ttl check: {}", inode.toString(), - e.toString()); - } - if (path != null) { - try { - TtlAction ttlAction = inode.getTtlAction(); - LOG.info("Path {} TTL has expired, performing action {}", path.getPath(), ttlAction); - switch (ttlAction) { - case FREE: - // public free method will lock the path, and check WRITE permission required at - // parent of file - if (inode.isDirectory()) { - mFileSystemMaster.free(path, FreeContext - .mergeFrom(FreePOptions.newBuilder().setForced(true).setRecursive(true))); - } else { - mFileSystemMaster.free(path, - FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(true))); - } - try (JournalContext journalContext = mFileSystemMaster.createJournalContext()) { - // Reset state - mInodeTree.updateInode(journalContext, UpdateInodeEntry.newBuilder() - .setId(inode.getId()) - .setTtl(Constants.NO_TTL) - .setTtlAction(ProtobufUtils.toProtobuf(TtlAction.DELETE)) - .build()); - } - mTtlBuckets.remove(inode); - break; - case DELETE:// Default if not set is DELETE - // public delete method will lock the path, and check WRITE permission required at - // parent of file - if (inode.isDirectory()) { - mFileSystemMaster.delete(path, - DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true))); - } else { - mFileSystemMaster.delete(path, DeleteContext.defaults()); - } - break; - default: - LOG.error("Unknown ttl action {}", ttlAction); - } - } catch (Exception e) { - LOG.error("Exception trying to clean up {} for ttl check", inode, e); - } - } - } - } - mTtlBuckets.removeBuckets(expiredBuckets); - } - - @Override - public void close() { - // Nothing to clean up - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/LostFileDetector.java b/core/server/master/src/main/java/alluxio/master/file/LostFileDetector.java deleted file mode 100644 index 3eadbc4a5c25..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/LostFileDetector.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file; - -import alluxio.exception.FileDoesNotExistException; -import alluxio.exception.status.UnavailableException; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.master.block.BlockId; -import alluxio.master.block.BlockMaster; -import alluxio.master.file.meta.Inode; -import alluxio.master.file.meta.InodeTree; -import alluxio.master.file.meta.InodeTree.LockPattern; -import alluxio.master.file.meta.LockedInodePath; -import alluxio.master.file.meta.PersistenceState; -import alluxio.master.journal.JournalContext; -import alluxio.master.journal.NoopJournalContext; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proto.journal.File.UpdateInodeEntry; -import alluxio.util.IdUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Lost files periodic check. - */ -@NotThreadSafe -final class LostFileDetector implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(LostFileDetector.class); - private final FileSystemMaster mFileSystemMaster; - private final InodeTree mInodeTree; - private final BlockMaster mBlockMaster; - - /** - * Constructs a new {@link LostFileDetector}. - */ - public LostFileDetector(FileSystemMaster fileSystemMaster, BlockMaster blockMaster, - InodeTree inodeTree) { - mFileSystemMaster = fileSystemMaster; - mInodeTree = inodeTree; - mBlockMaster = blockMaster; - MetricsSystem.registerCachedGaugeIfAbsent(MetricKey.MASTER_LOST_FILE_COUNT.getName(), - () -> mFileSystemMaster.getLostFiles().size()); - } - - @Override - public void heartbeat() throws InterruptedException { - Iterator iter = mBlockMaster.getLostBlocksIterator(); - Set markedFiles = new HashSet<>(); - while (iter.hasNext()) { - if (Thread.interrupted()) { - throw new InterruptedException("LostFileDetector interrupted."); - } - long blockId = iter.next(); - long containerId = BlockId.getContainerId(blockId); - long fileId = IdUtils.createFileId(containerId); - if (markedFiles.contains(fileId)) { - iter.remove(); - continue; - } - boolean markAsLost = false; - try ( - LockedInodePath inodePath = - mInodeTree.lockFullInodePath(fileId, LockPattern.READ, NoopJournalContext.INSTANCE) - ) { - Inode inode = inodePath.getInode(); - if (inode.getPersistenceState() != PersistenceState.PERSISTED) { - markAsLost = true; - } - } catch (FileDoesNotExistException e) { - LOG.debug("Exception trying to get inode from inode tree", e); - iter.remove(); - continue; - } - - if (markAsLost) { - // update the state - try (JournalContext journalContext = mFileSystemMaster.createJournalContext(); - LockedInodePath inodePath = - mInodeTree.lockFullInodePath(fileId, LockPattern.WRITE_INODE, journalContext)) { - Inode inode = inodePath.getInode(); - if (inode.getPersistenceState() != PersistenceState.PERSISTED) { - mInodeTree.updateInode(journalContext, - UpdateInodeEntry.newBuilder().setId(inode.getId()) - .setPersistenceState(PersistenceState.LOST.name()).build()); - markedFiles.add(fileId); - } - iter.remove(); - } catch (FileDoesNotExistException e) { - LOG.debug("Failed to mark file {} as lost. The inode does not exist anymore.", - fileId, e); - iter.remove(); - } catch (UnavailableException e) { - LOG.warn("Failed to mark files LOST because the journal is not available. " - + "{} files are affected: {}", - markedFiles.size(), markedFiles, e); - break; - } - } - } - } - - @Override - public void close() { - // Nothing to clean up - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/UfsCleaner.java b/core/server/master/src/main/java/alluxio/master/file/UfsCleaner.java deleted file mode 100644 index bc9ab0ab6ef4..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/UfsCleaner.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file; - -import alluxio.heartbeat.HeartbeatExecutor; - -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Under file system periodic clean. - */ -@NotThreadSafe -final class UfsCleaner implements HeartbeatExecutor { - private final FileSystemMaster mFileSystemMaster; - - /** - * Constructs a new {@link UfsCleaner}. - */ - public UfsCleaner(FileSystemMaster fileSystemMaster) { - mFileSystemMaster = fileSystemMaster; - } - - @Override - public void heartbeat() { - mFileSystemMaster.cleanupUfs(); - } - - @Override - public void close() {} -} diff --git a/core/server/master/src/main/java/alluxio/master/file/activesync/ActiveSyncManager.java b/core/server/master/src/main/java/alluxio/master/file/activesync/ActiveSyncManager.java deleted file mode 100644 index 1d038633fbb3..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/activesync/ActiveSyncManager.java +++ /dev/null @@ -1,780 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file.activesync; - -import alluxio.AlluxioURI; -import alluxio.ProcessUtils; -import alluxio.SyncInfo; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.InvalidPathException; -import alluxio.heartbeat.HeartbeatContext; -import alluxio.heartbeat.HeartbeatThread; -import alluxio.master.file.FileSystemMaster; -import alluxio.master.file.RpcContext; -import alluxio.master.file.meta.MountTable; -import alluxio.master.file.meta.options.MountInfo; -import alluxio.master.journal.JournalContext; -import alluxio.master.journal.Journaled; -import alluxio.master.journal.checkpoint.CheckpointName; -import alluxio.proto.journal.File; -import alluxio.proto.journal.File.AddSyncPointEntry; -import alluxio.proto.journal.File.RemoveSyncPointEntry; -import alluxio.proto.journal.Journal; -import alluxio.proto.journal.Journal.JournalEntry; -import alluxio.resource.CloseableIterator; -import alluxio.resource.CloseableResource; -import alluxio.resource.LockResource; -import alluxio.retry.ExponentialTimeBoundedRetry; -import alluxio.retry.RetryPolicy; -import alluxio.retry.RetryUtils; -import alluxio.security.user.ServerUserState; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.util.ThreadFactoryUtils; -import alluxio.util.io.PathUtils; -import alluxio.wire.SyncPointInfo; - -import com.google.common.collect.Iterators; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.text.MessageFormat; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Supplier; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Manager for the Active UFS sync process. - * - * There are several threads cooperating to make the active sync process happen. - * 1. An active polling thread that polls HDFS for change events and aggregates these events. - * 2. A heartbeat thread that wakes up periodically to consume the aggregated events, and perform - * syncing if necessary. - * 3. For initial syncing, we launch a future to perform initial syncing asynchronously. This is - * stored in mSyncPathStatus. - */ -@NotThreadSafe -public class ActiveSyncManager implements Journaled { - private static final Logger LOG = LoggerFactory.getLogger(ActiveSyncManager.class); - - // a reference to the mount table - private final MountTable mMountTable; - // a list of sync points - private final List mSyncPathList; - // a map which maps mount id to a thread polling that UFS - private final Map> mPollerMap; - // a map which maps each mount id to a list of paths being actively synced on mountpoint - private final Map> mFilterMap; - // a map which maps mount id to the latest txid synced on that mount point - private final Map mStartingTxIdMap; - // Future.isDone = INITIALLY_SYNCED, !Future.isDone = SYNCING - // Future == null => NOT_INITIALLY_SYNCED - private final Map> mSyncPathStatus; - // a lock which protects the above data structures - private final Lock mLock; - // a reference to FSM - private final FileSystemMaster mFileSystemMaster; - // a local executor service used to launch polling threads - private final ThreadPoolExecutor mExecutorService; - private boolean mStarted; - private final Supplier mRetryPolicy = () -> - ExponentialTimeBoundedRetry.builder() - .withMaxDuration(Duration - .ofMillis(Configuration.getMs( - PropertyKey.MASTER_UFS_ACTIVE_SYNC_RETRY_TIMEOUT))) - .withInitialSleep(Duration.ofMillis(100)) - .withMaxSleep(Duration.ofSeconds(60)) - .build(); - - /** - * Constructs a Active Sync Manager. - * - * @param mountTable mount table - * @param fileSystemMaster file system master - */ - public ActiveSyncManager(MountTable mountTable, FileSystemMaster fileSystemMaster) { - mMountTable = mountTable; - mPollerMap = new ConcurrentHashMap<>(); - mFilterMap = new ConcurrentHashMap<>(); - mStartingTxIdMap = new ConcurrentHashMap<>(); - mSyncPathList = new CopyOnWriteArrayList<>(); - mFileSystemMaster = fileSystemMaster; - mSyncPathStatus = new ConcurrentHashMap<>(); - - // A lock used to protect the state stored in the above maps and lists - mLock = new ReentrantLock(); - // Executor Service for active syncing - mExecutorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), - Runtime.getRuntime().availableProcessors(), - 1, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), - ThreadFactoryUtils.build("ActiveSyncManager-%d", false)); - mExecutorService.allowCoreThreadTimeOut(true); - } - - /** - * Gets the lock protecting the syncManager. - * - * @return syncmanager lock - */ - public Lock getLock() { - return mLock; - } - - /** - * Gets the retry policy. - * - * @return retry policy - */ - public RetryPolicy getRetryPolicy() { - return mRetryPolicy.get(); - } - - /** - * @param syncPoint the uri to check - * @return true if the given path is a sync point - */ - public boolean isSyncPoint(AlluxioURI syncPoint) { - return mSyncPathList.contains(syncPoint); - } - - /** - * Check if a URI is actively synced. - * - * @param path path to check - * @return true if a URI is being actively synced - */ - public boolean isUnderSyncPoint(AlluxioURI path) { - for (AlluxioURI syncedPath : mSyncPathList) { - try { - if (PathUtils.hasPrefix(path.getPath(), syncedPath.getPath()) - && mMountTable.getMountPoint(path).equals(mMountTable.getMountPoint(syncedPath))) { - return true; - } - } catch (InvalidPathException e) { - return false; - } - } - return false; - } - - /** - * Start the polling threads. - */ - public void start() throws IOException { - mStarted = true; - // Initialize UFS states - for (AlluxioURI syncPoint : mSyncPathList) { - MountTable.Resolution resolution; - try { - resolution = mMountTable.resolve(syncPoint); - } catch (InvalidPathException e) { - LOG.info("Invalid Path encountered during start up of ActiveSyncManager, " - + "path {}, exception {}", syncPoint, e); - continue; - } - - try (CloseableResource ufsResource = resolution.acquireUfsResource()) { - if (!ufsResource.get().supportsActiveSync()) { - throw new UnsupportedOperationException("Active Sync is not supported on this UFS type: " - + ufsResource.get().getUnderFSType()); - } - ufsResource.get().startSync(resolution.getUri()); - } - } - // attempt to restart from a past txid, if this fails, it will result in MissingEventException - // therefore forces a sync - for (Map.Entry> entry : mFilterMap.entrySet()) { - long mountId = entry.getKey(); - long txId = mStartingTxIdMap.getOrDefault(mountId, SyncInfo.INVALID_TXID); - if (!entry.getValue().isEmpty()) { - launchPollingThread(mountId, txId); - } - - try { - if ((txId == SyncInfo.INVALID_TXID) && Configuration.getBoolean( - PropertyKey.MASTER_UFS_ACTIVE_SYNC_INITIAL_SYNC_ENABLED)) { - mExecutorService.submit( - () -> entry.getValue().parallelStream().forEach( - syncPoint -> { - MountTable.Resolution resolution; - try { - resolution = mMountTable.resolve(syncPoint); - } catch (InvalidPathException e) { - LOG.info("Invalid Path encountered during start up of ActiveSyncManager, " - + "path {}, exception {}", syncPoint, e); - return; - } - startInitialFullSync(syncPoint, resolution); - } - )); - } - } catch (Exception e) { - LOG.warn("exception encountered during initial sync: {}", e.toString()); - } - } - } - - /** - * Launches polling thread on a particular mount point with starting txId. - * - * @param mountId launch polling thread on a mount id - * @param txId specifies the transaction id to initialize the pollling thread - */ - public void launchPollingThread(long mountId, long txId) { - LOG.debug("launch polling thread for mount id {}, txId {}", mountId, txId); - if (!mPollerMap.containsKey(mountId)) { - UfsManager.UfsClient ufsClient = mMountTable.getUfsClient(mountId); - if (ufsClient == null) { - LOG.warn("Mount id {} does not exist", mountId); - return; - } - try (CloseableResource ufsResource = ufsClient.acquireUfsResource()) { - ufsResource.get().startActiveSyncPolling(txId); - } catch (IOException e) { - LOG.warn("IO Exception trying to launch Polling thread: {}", e.toString()); - } - ActiveSyncer syncer = new ActiveSyncer(mFileSystemMaster, this, mMountTable, mountId); - Future future = getExecutor().submit( - new HeartbeatThread(HeartbeatContext.MASTER_ACTIVE_UFS_SYNC, - syncer, (int) Configuration.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INTERVAL), - Configuration.global(), ServerUserState.global())); - mPollerMap.put(mountId, future); - } - } - - /** - * Apply {@link AddSyncPointEntry} and journal the entry. - * - * @param context journal context - * @param entry addSyncPoint entry - */ - public void applyAndJournal(Supplier context, AddSyncPointEntry entry) { - LOG.info("Apply startSync {}", entry.getSyncpointPath()); - try { - apply(entry); - context.get().append(Journal.JournalEntry.newBuilder().setAddSyncPoint(entry).build()); - } catch (Throwable t) { - ProcessUtils.fatalError(LOG, t, "Failed to apply %s", entry); - throw t; // fatalError will usually system.exit - } - } - - /** - * Apply {@link RemoveSyncPointEntry} and journal the entry. - * - * @param context journal context - * @param entry removeSyncPoint entry - */ - public void applyAndJournal(Supplier context, RemoveSyncPointEntry entry) { - try { - apply(entry); - context.get().append(Journal.JournalEntry.newBuilder().setRemoveSyncPoint(entry).build()); - } catch (Throwable t) { - ProcessUtils.fatalError(LOG, t, "Failed to apply %s", entry); - throw t; // fatalError will usually system.exit - } - } - - /** - * Start active sync on a URI and journal the add entry. - * - * @param rpcContext the master rpc or no-op context - * @param syncPoint sync point to be start - */ - public void startSyncAndJournal(RpcContext rpcContext, AlluxioURI syncPoint) - throws InvalidPathException { - try (LockResource r = new LockResource(mLock)) { - MountTable.Resolution resolution = mMountTable.resolve(syncPoint); - long mountId = resolution.getMountId(); - try (CloseableResource ufsResource = resolution.acquireUfsResource()) { - if (!ufsResource.get().supportsActiveSync()) { - throw new UnsupportedOperationException( - "Active Syncing is not supported on this UFS type: " - + ufsResource.get().getUnderFSType()); - } - } - - if (isUnderSyncPoint(syncPoint)) { - throw new InvalidPathException("URI " + syncPoint + " is already a sync point"); - } - AddSyncPointEntry addSyncPoint = - AddSyncPointEntry.newBuilder() - .setSyncpointPath(syncPoint.toString()) - .setMountId(mountId) - .build(); - applyAndJournal(rpcContext, addSyncPoint); - try { - startSyncInternal(syncPoint, resolution); - } catch (Throwable e) { - LOG.warn("Start sync failed on {}", syncPoint, e); - // revert state; - RemoveSyncPointEntry removeSyncPoint = - File.RemoveSyncPointEntry.newBuilder() - .setSyncpointPath(syncPoint.toString()).build(); - applyAndJournal(rpcContext, removeSyncPoint); - recoverFromStartSync(syncPoint, resolution.getMountId()); - throw e; - } - } - } - - /** - * Stop active sync on a mount id. - * - * @param mountId mountId to stop active sync - */ - public void stopSyncForMount(long mountId) throws InvalidPathException { - LOG.info("Stop sync for mount id {}", mountId); - if (mFilterMap.containsKey(mountId)) { - List toBeDeleted = new ArrayList<>(mFilterMap.get(mountId)); - for (AlluxioURI uri : toBeDeleted) { - stopSyncAndJournal(RpcContext.NOOP, uri); - } - } - } - - /** - * Stop active sync on a URI and journal the remove entry. - * - * @param rpcContext the master rpc or no-op context - * @param syncPoint sync point to be stopped - */ - public void stopSyncAndJournal(RpcContext rpcContext, AlluxioURI syncPoint) - throws InvalidPathException { - if (!isSyncPoint(syncPoint)) { - throw new InvalidPathException(String.format("%s is not a sync point", syncPoint)); - } - try (LockResource r = new LockResource(mLock)) { - MountTable.Resolution resolution = mMountTable.resolve(syncPoint); - LOG.debug("stop syncPoint {}", syncPoint.getPath()); - final long mountId = resolution.getMountId(); - RemoveSyncPointEntry removeSyncPoint = File.RemoveSyncPointEntry.newBuilder() - .setSyncpointPath(syncPoint.toString()) - .setMountId(mountId) - .build(); - applyAndJournal(rpcContext, removeSyncPoint); - try { - stopSyncInternal(syncPoint); - } catch (Throwable e) { - LOG.warn("Stop sync failed on {}", syncPoint, e); - // revert state; - AddSyncPointEntry addSyncPoint = - File.AddSyncPointEntry.newBuilder() - .setSyncpointPath(syncPoint.toString()).build(); - applyAndJournal(rpcContext, addSyncPoint); - recoverFromStopSync(syncPoint); - } - } - } - - /** - * Get the filter list associated with mount Id. - * - * @param mountId mountId - * @return a list of URIs (sync points) associated with that mount id - */ - public List getFilterList(long mountId) { - return mFilterMap.get(mountId); - } - - /** - * Get the sync point list. - * - * @return a list of URIs (sync points) - */ - public List getSyncPathList() { - List returnList = new ArrayList<>(); - for (AlluxioURI uri: mSyncPathList) { - SyncPointInfo.SyncStatus status; - Future syncStatus = mSyncPathStatus.get(uri); - if (syncStatus == null) { - status = SyncPointInfo.SyncStatus.NOT_INITIALLY_SYNCED; - } else if (syncStatus.isDone()) { - status = SyncPointInfo.SyncStatus.INITIALLY_SYNCED; - } else { - status = SyncPointInfo.SyncStatus.SYNCING; - } - returnList.add(new SyncPointInfo(uri, status)); - } - return returnList; - } - - private Iterator getSyncPathIterator() { - final Iterator it = mSyncPathList.iterator(); - return new Iterator() { - private AlluxioURI mEntry = null; - - @Override - public boolean hasNext() { - if (mEntry != null) { - return true; - } - if (it.hasNext()) { - mEntry = it.next(); - return true; - } - return false; - } - - @Override - public Journal.JournalEntry next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - String syncPointPath = mEntry.getPath(); - long mountId = -1; - while (mountId == -1) { - try { - syncPointPath = mEntry.getPath(); - String mountPoint = mMountTable.getMountPoint(mEntry); - MountInfo mountInfo = mMountTable.getMountTable().get(mountPoint); - mountId = mountInfo.getMountId(); - } catch (InvalidPathException e) { - LOG.info("Path resolution failed for {}, exception {}", syncPointPath, e); - mEntry = null; - if (!hasNext()) { - throw new NoSuchElementException(); - } - } - } - mEntry = null; - - File.AddSyncPointEntry addSyncPointEntry = - File.AddSyncPointEntry.newBuilder() - .setSyncpointPath(syncPointPath) - .setMountId(mountId) - .build(); - - return Journal.JournalEntry.newBuilder().setAddSyncPoint(addSyncPointEntry).build(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException( - "ActiveSyncManager#Iterator#remove is not supported."); - } - }; - } - - private void apply(RemoveSyncPointEntry removeSyncPoint) { - AlluxioURI syncPoint = new AlluxioURI(removeSyncPoint.getSyncpointPath()); - long mountId = removeSyncPoint.getMountId(); - - try (LockResource r = new LockResource(mLock)) { - LOG.info("SyncPoint stopped {}", syncPoint.getPath()); - - if (mFilterMap.containsKey(mountId)) { - List list = mFilterMap.get(mountId); - if (list != null) { - list.remove(syncPoint); - } - mSyncPathList.remove(syncPoint); - } else { - mSyncPathList.remove(syncPoint); - // We should not be in this situation - throw new RuntimeException( - String.format("mountId for the syncPoint %s not found in the filterMap", - syncPoint)); - } - } - } - - private void apply(AddSyncPointEntry addSyncPoint) { - AlluxioURI syncPoint = new AlluxioURI(addSyncPoint.getSyncpointPath()); - long mountId = addSyncPoint.getMountId(); - - LOG.info("SyncPoint added {}, mount id {}", syncPoint.getPath(), mountId); - // Add the new sync point to the filter map - if (mFilterMap.containsKey(mountId)) { - mFilterMap.get(mountId).add(syncPoint); - } else { - ArrayList list = new ArrayList<>(); - list.add(syncPoint); - mFilterMap.put(mountId, list); - } - // Add to the sync point list - mSyncPathList.add(syncPoint); - } - - /** - * Continue to start sync after we have journaled the operation. - * - * @param syncPoint the sync point that we are trying to start - * @param resolution the mount table resolution of sync point - */ - private void startSyncInternal(AlluxioURI syncPoint, MountTable.Resolution resolution) { - startInitialFullSync(syncPoint, resolution); - launchPollingThread(resolution.getMountId(), SyncInfo.INVALID_TXID); - } - - /** - * Start a thread for an initial full sync. - * - * @param syncPoint the sync point - * @param resolution the mount table resolution - */ - private void startInitialFullSync(AlluxioURI syncPoint, MountTable.Resolution resolution) { - try (CloseableResource ufsResource = resolution.acquireUfsResource()) { - Future syncFuture = mExecutorService.submit( - () -> { - try { - // Notify ufs polling thread to keep track of events related to specified uri - ufsResource.get().startSync(resolution.getUri()); - // Start the initial metadata sync between the ufs and alluxio for the specified uri - if (Configuration.getBoolean( - PropertyKey.MASTER_UFS_ACTIVE_SYNC_INITIAL_SYNC_ENABLED)) { - - RetryUtils.retry("active sync during start", - () -> mFileSystemMaster.activeSyncMetadata(syncPoint, - null, getExecutor()), getRetryPolicy()); - } - } catch (IOException e) { - LOG.info(MessageFormat.format( - "IOException encountered during initial syncing of sync point {0}", - resolution.getUri()), e); - } - }); - mSyncPathStatus.put(syncPoint, syncFuture); - } - } - - /** - * Clean up tasks to stop sync point after we have journaled. - * - * @param syncPoint the sync point to stop - * @throws InvalidPathException throw an invalid path exception - */ - private void stopSyncInternal(AlluxioURI syncPoint) throws InvalidPathException { - MountTable.Resolution resolution = mMountTable.resolve(syncPoint); - // Remove initial sync thread - Future syncFuture = mSyncPathStatus.remove(syncPoint); - if (syncFuture != null) { - syncFuture.cancel(true); - } - - long mountId = resolution.getMountId(); - if (mFilterMap.containsKey(mountId) && mFilterMap.get(mountId).isEmpty()) { - Future future = mPollerMap.remove(mountId); - if (future != null) { - future.cancel(true); - } - } - - // Tell UFS to stop monitoring the path - try (CloseableResource ufs = resolution.acquireUfsResource()) { - ufs.get().stopSync(resolution.getUri()); - } catch (IOException e) { - LOG.info("Ufs IOException for uri {}, exception is {}", syncPoint, e); - } - - // Stop active sync polling on a particular UFS if it is the last sync point - if (mFilterMap.containsKey(mountId) && mFilterMap.get(mountId).isEmpty()) { - // syncPoint removed was the last syncPoint for the mountId - mFilterMap.remove(mountId); - try (CloseableResource ufs = resolution.acquireUfsResource()) { - ufs.get().stopActiveSyncPolling(); - } catch (IOException e) { - LOG.warn("Encountered IOException when trying to stop polling thread: {}", e.toString()); - } - } - } - - private Iterator getTxIdIterator() { - final Iterator> it = mStartingTxIdMap.entrySet().iterator(); - return new Iterator() { - private Map.Entry mEntry = null; - - @Override - public boolean hasNext() { - if (mEntry != null) { - return true; - } - if (it.hasNext()) { - mEntry = it.next(); - return true; - } - return false; - } - - @Override - public Journal.JournalEntry next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - long mountId = mEntry.getKey(); - long txId = mEntry.getValue(); - mEntry = null; - - File.ActiveSyncTxIdEntry txIdEntry = - File.ActiveSyncTxIdEntry.newBuilder().setMountId(mountId) - .setTxId(txId).build(); - return Journal.JournalEntry.newBuilder().setActiveSyncTxId(txIdEntry).build(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException( - "ActiveSyncManager#Iterator#remove is not supported."); - } - }; - } - - /** - * Set the transaction id for a particular mountId. - * - * @param mountId mount id - * @param txId transaction id - */ - public void setTxId(long mountId, long txId) { - mStartingTxIdMap.put(mountId, txId); - } - - /** - * Get SyncManager Executor. - * - * @return an executor for active syncing - */ - public ExecutorService getExecutor() { - return mExecutorService; - } - - /** - * Stops the sync manager and any outstanding threads, does not change the sync points. - * - * This stops four things in the following order. - * 1. Stop any outstanding initial sync futures for the sync points. (syncFuture.cancel) - * 2. Stop the heartbeat thread that periodically wakes up to process events that have been - * recorded for the past heartbeat interval. - * 3. Tell the polling thread to stop monitoring the path for events - * 4. Stop the thread that is polling HDFS for events - */ - public void stop() { - if (!mStarted) { - return; - } - mStarted = false; - for (AlluxioURI syncPoint : mSyncPathList) { - try { - stopSyncInternal(syncPoint); - } catch (InvalidPathException e) { - LOG.warn("stop: InvalidPathException resolving syncPoint {}, exception {}", - syncPoint, e); - return; - } - } - } - - /** - * Recover from a stop sync operation. - * - * @param uri uri to stop sync - */ - public void recoverFromStopSync(AlluxioURI uri) { - if (mSyncPathStatus.containsKey(uri)) { - // nothing to recover from, since the syncPathStatus still contains syncPoint - return; - } - try { - // the init sync thread has been removed, to reestablish sync, we need to sync again - MountTable.Resolution resolution = mMountTable.resolve(uri); - startInitialFullSync(uri, resolution); - launchPollingThread(resolution.getMountId(), SyncInfo.INVALID_TXID); - } catch (Throwable t) { - LOG.warn("Recovering from stop syncing failed: {}", t.toString()); - } - } - - /** - * Recover from start sync operation. - * - * @param uri uri to start sync - * @param mountId mount id of the uri - */ - public void recoverFromStartSync(AlluxioURI uri, long mountId) { - // if the init sync has been launched, we need to stop it - if (mSyncPathStatus.containsKey(uri)) { - Future syncFuture = mSyncPathStatus.remove(uri); - if (syncFuture != null) { - syncFuture.cancel(true); - } - } - - // if the polling thread has been launched, we need to stop it - mFilterMap.remove(mountId); - Future future = mPollerMap.remove(mountId); - if (future != null) { - future.cancel(true); - } - } - - @Override - public boolean processJournalEntry(JournalEntry entry) { - if (entry.hasAddSyncPoint()) { - apply(entry.getAddSyncPoint()); - return true; - } else if (entry.hasRemoveSyncPoint()) { - apply(entry.getRemoveSyncPoint()); - return true; - } else if (entry.hasActiveSyncTxId()) { - File.ActiveSyncTxIdEntry activeSyncTxId = entry.getActiveSyncTxId(); - setTxId(activeSyncTxId.getMountId(), activeSyncTxId.getTxId()); - return true; - } - return false; - } - - /** - * {@inheritDoc} - * - * It clears all sync points, and stops the polling thread. - */ - @Override - public void resetState() { - for (long mountId : new HashSet<>(mFilterMap.keySet())) { - try { - // stops sync point under this mount point. Note this clears the sync point and - // stops associated polling threads. - stopSyncForMount(mountId); - } catch (InvalidPathException e) { - LOG.info("Exception resetting mountId {}, exception: {}", mountId, e); - } - } - } - - @Override - public CheckpointName getCheckpointName() { - return CheckpointName.ACTIVE_SYNC_MANAGER; - } - - @Override - public CloseableIterator getJournalEntryIterator() { - return CloseableIterator.noopCloseable( - Iterators.concat(getSyncPathIterator(), getTxIdIterator())); - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/activesync/ActiveSyncer.java b/core/server/master/src/main/java/alluxio/master/file/activesync/ActiveSyncer.java deleted file mode 100644 index 666da9434682..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/activesync/ActiveSyncer.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file.activesync; - -import alluxio.AlluxioURI; -import alluxio.SyncInfo; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.master.file.FileSystemMaster; -import alluxio.master.file.meta.MountTable; -import alluxio.resource.CloseableResource; -import alluxio.retry.RetryUtils; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.util.LogUtils; - -import com.google.common.base.Throwables; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Periodically sync the files for a particular mount point. - */ -@NotThreadSafe -public class ActiveSyncer implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(ActiveSyncer.class); - private final FileSystemMaster mFileSystemMaster; - private final ActiveSyncManager mSyncManager; - private final MountTable mMountTable; - private final long mMountId; - private final AlluxioURI mMountUri; - private final Queue> mSyncTasks; - - /** - * Constructs a new {@link ActiveSyncer}. - * - * @param fileSystemMaster fileSystem master - * @param syncManager sync manager - * @param mountTable mount table - * @param mountId mount id - */ - public ActiveSyncer(FileSystemMaster fileSystemMaster, ActiveSyncManager syncManager, - MountTable mountTable, long mountId) { - mFileSystemMaster = fileSystemMaster; - mSyncManager = syncManager; - mMountTable = mountTable; - mMountId = mountId; - mMountUri = Objects.requireNonNull(mMountTable.getMountInfo(mMountId)).getAlluxioUri(); - mSyncTasks = new LinkedBlockingQueue<>(32); - } - - @Override - public void heartbeat() { - LOG.debug("start sync heartbeat for {} with mount id {}", mMountUri, mMountId); - // Remove any previously completed sync tasks - mSyncTasks.removeIf(Future::isDone); - - List filterList = mSyncManager.getFilterList(mMountId); - - if (filterList == null || filterList.isEmpty()) { - return; - } - - try { - UfsManager.UfsClient ufsclient = Objects.requireNonNull(mMountTable.getUfsClient(mMountId)); - try (CloseableResource ufsResource = ufsclient.acquireUfsResource()) { - UnderFileSystem ufs = ufsResource.get(); - if (!ufs.supportsActiveSync()) { - return; - } - SyncInfo syncInfo = ufs.getActiveSyncInfo(); - // This returns a list of ufsUris that we need to sync. - Set ufsSyncPoints = syncInfo.getSyncPoints(); - // Parallelize across sync points - CompletableFuture[] tasksPerSync = new CompletableFuture[ufsSyncPoints.size()]; - int idx = 0; - for (AlluxioURI ufsUri : ufsSyncPoints) { - tasksPerSync[idx] = CompletableFuture.supplyAsync(() -> { - processSyncPoint(ufsUri, syncInfo); - return syncInfo.getTxId(); - }, mSyncManager.getExecutor()); - idx++; - } - // Journal the latest processed txId - CompletableFuture syncTask = - CompletableFuture.allOf(tasksPerSync) - .thenRunAsync(() -> mFileSystemMaster - .recordActiveSyncTxid(syncInfo.getTxId(), mMountId), - mSyncManager.getExecutor()); - int attempts = 0; - while (!mSyncTasks.offer(syncTask)) { - if (Thread.currentThread().isInterrupted()) { - break; - } - // We should only enter the loop if the task queue is full. This would occur if all - // sync tasks in the last 32 / (heartbeats/minute) minutes have not completed. - for (CompletableFuture f : mSyncTasks) { - try { - long waitTime = Configuration.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INTERVAL) - / mSyncTasks.size(); - f.get(waitTime, TimeUnit.MILLISECONDS); - mSyncTasks.remove(f); - break; - } catch (TimeoutException e) { - LOG.trace("sync task did not complete during heartbeat. Attempt: {}", attempts); - } catch (InterruptedException | ExecutionException e) { - LogUtils.warnWithException(LOG, "Failed while waiting on task to add new task to " - + "head of queue", e); - if (Thread.currentThread().isInterrupted()) { - Thread.currentThread().interrupt(); - } - } - attempts++; - } - } - } - } catch (IOException e) { - LOG.warn("IOException " + Throwables.getStackTraceAsString(e)); - } - } - - @Override - public void close() { - for (CompletableFuture syncTask : mSyncTasks) { - syncTask.cancel(true); - } - } - - /** - * Process a single sync point. - * - * @param ufsUri ufs URI for the sync point - * @param syncInfo active sync info for mount - */ - private void processSyncPoint(AlluxioURI ufsUri, SyncInfo syncInfo) { - AlluxioURI alluxioUri = mMountTable.reverseResolve(ufsUri).getUri(); - if (alluxioUri == null) { - LOG.warn("Unable to reverse resolve ufsUri {}", ufsUri); - return; - } - try { - if (syncInfo.isForceSync()) { - LOG.debug("force full sync {}", ufsUri); - RetryUtils.retry("Full Sync", () -> { - mFileSystemMaster.activeSyncMetadata(alluxioUri, null, mSyncManager.getExecutor()); - }, mSyncManager.getRetryPolicy()); - } else { - LOG.debug("incremental sync {}", ufsUri); - RetryUtils.retry("Incremental Sync", () -> { - mFileSystemMaster.activeSyncMetadata(alluxioUri, - syncInfo.getChangedFiles(ufsUri).stream() - .map((uri) -> Objects.requireNonNull(mMountTable.reverseResolve(uri)).getUri()) - .collect(Collectors.toSet()), - mSyncManager.getExecutor()); - }, mSyncManager.getRetryPolicy()); - } - } catch (IOException e) { - LOG.warn("Failed to submit active sync job to master: ufsUri {}, syncPoint {} ", ufsUri, - alluxioUri, e); - } - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/meta/TtlBucket.java b/core/server/master/src/main/java/alluxio/master/file/meta/TtlBucket.java deleted file mode 100644 index eda16f7dc6a8..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/meta/TtlBucket.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file.meta; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; - -import com.google.common.base.Objects; - -import java.util.Collection; -import java.util.concurrent.ConcurrentHashMap; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A bucket with all inodes whose ttl value lies in the bucket's time interval. The bucket's time - * interval starts at a certain time and lasts for - * {@link PropertyKey#MASTER_TTL_CHECKER_INTERVAL_MS}. - */ -@ThreadSafe -public final class TtlBucket implements Comparable { - /** - * The time interval of this bucket is the same as ttl checker's interval. - * - * This field is intentionally not final so that tests can change the value. - */ - private static long sTtlIntervalMs = - Configuration.getMs(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS); - /** - * Each bucket has a time to live interval, this value is the start of the interval, interval - * value is the same as the configuration of {@link PropertyKey#MASTER_TTL_CHECKER_INTERVAL_MS}. - */ - private final long mTtlIntervalStartTimeMs; - /** - * A collection of inodes whose ttl value is in the range of this bucket's interval. The mapping - * is from inode id to inode. - */ - private final ConcurrentHashMap mInodes; - - /** - * Creates a new instance of {@link TtlBucket}. - * - * @param startTimeMs the start time to use - */ - public TtlBucket(long startTimeMs) { - mTtlIntervalStartTimeMs = startTimeMs; - mInodes = new ConcurrentHashMap<>(); - } - - /** - * @return the ttl interval start time in milliseconds - */ - public long getTtlIntervalStartTimeMs() { - return mTtlIntervalStartTimeMs; - } - - /** - * - * @return the ttl interval end time in milliseconds - */ - public long getTtlIntervalEndTimeMs() { - return mTtlIntervalStartTimeMs + sTtlIntervalMs; - } - - /** - * @return the ttl interval in milliseconds - */ - public static long getTtlIntervalMs() { - return sTtlIntervalMs; - } - - /** - * @return the set of all inodes in the bucket backed by the internal set, changes made to the - * returned set will be shown in the internal set, and vice versa - */ - public Collection getInodes() { - return mInodes.values(); - } - - /** - * Adds a inode to the bucket. - * - * @param inode the inode to be added - */ - public void addInode(Inode inode) { - mInodes.put(inode.getId(), inode); - } - - /** - * Removes a inode from the bucket. - * - * @param inode the inode to be removed - */ - public void removeInode(InodeView inode) { - mInodes.remove(inode.getId()); - } - - /** - * Compares this bucket's TTL interval start time to that of another bucket. - * - * @param ttlBucket the bucket to be compared to - * @return 0 when return values of {@link #getTtlIntervalStartTimeMs()} from the two buckets are - * the same, -1 when that value of current instance is smaller, otherwise, 1 - */ - @Override - public int compareTo(TtlBucket ttlBucket) { - long startTime1 = getTtlIntervalStartTimeMs(); - long startTime2 = ttlBucket.getTtlIntervalStartTimeMs(); - return Long.compare(startTime1, startTime2); - } - - /** - * Compares to a specific object. - * - * @param o the object to compare - * @return true if object is also {@link TtlBucket} and represents the same TtlIntervalStartTime - */ - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof TtlBucket)) { - return false; - } - TtlBucket that = (TtlBucket) o; - return mTtlIntervalStartTimeMs == that.mTtlIntervalStartTimeMs; - } - - /** - * Returns the hash code for the {@link TtlBucket}. - * - * @return The hash code value for this {@link TtlBucket} - */ - @Override - public int hashCode() { - return Objects.hashCode(mTtlIntervalStartTimeMs); - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/meta/TtlBucketList.java b/core/server/master/src/main/java/alluxio/master/file/meta/TtlBucketList.java deleted file mode 100644 index a815117e725a..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/meta/TtlBucketList.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file.meta; - -import alluxio.Constants; -import alluxio.master.journal.checkpoint.CheckpointInputStream; -import alluxio.master.journal.checkpoint.CheckpointName; -import alluxio.master.journal.checkpoint.CheckpointOutputStream; -import alluxio.master.journal.checkpoint.CheckpointType; -import alluxio.master.journal.checkpoint.Checkpointed; -import alluxio.master.metastore.ReadOnlyInodeStore; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.EOFException; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentSkipListSet; -import javax.annotation.Nullable; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A list of non-empty {@link TtlBucket}s sorted by ttl interval start time of each bucket. - *

- * Two adjacent buckets may not have adjacent intervals since there may be no inodes with ttl value - * in the skipped intervals. - */ -@ThreadSafe -public final class TtlBucketList implements Checkpointed { - private static final Logger LOG = LoggerFactory.getLogger(TtlBucketList.class); - - /** - * List of buckets sorted by interval start time. SkipList is used for O(logn) insertion and - * retrieval, see {@link ConcurrentSkipListSet}. - */ - private final ConcurrentSkipListSet mBucketList; - private final ReadOnlyInodeStore mInodeStore; - - /** - * Creates a new list of {@link TtlBucket}s. - * - * @param inodeStore the inode store - */ - public TtlBucketList(ReadOnlyInodeStore inodeStore) { - mInodeStore = inodeStore; - mBucketList = new ConcurrentSkipListSet<>(); - } - - /** - * Gets the bucket in the list that contains the inode. - * - * @param inode the inode to be contained - * @return the bucket containing the inode, or null if no such bucket exists - */ - @Nullable - private TtlBucket getBucketContaining(InodeView inode) { - if (inode.getTtl() == Constants.NO_TTL) { - // no bucket will contain a inode with NO_TTL. - return null; - } - - long ttlEndTimeMs = inode.getCreationTimeMs() + inode.getTtl(); - // Gets the last bucket with interval start time less than or equal to the inode's life end - // time. - TtlBucket bucket = mBucketList.floor(new TtlBucket(ttlEndTimeMs)); - if (bucket == null || bucket.getTtlIntervalEndTimeMs() < ttlEndTimeMs - || (bucket.getTtlIntervalEndTimeMs() == ttlEndTimeMs - && TtlBucket.getTtlIntervalMs() != 0)) { - // 1. There is no bucket in the list, or - // 2. All buckets' interval start time is larger than the inode's life end time, or - // 3. No bucket actually contains ttlEndTimeMs in its interval. - return null; - } - - return bucket; - } - - /** - * Inserts an inode to the appropriate bucket where its ttl end time lies in the - * bucket's interval, if no appropriate bucket exists, a new bucket will be created to contain - * this inode, if ttl value is {@link Constants#NO_TTL}, the inode won't be inserted to any - * buckets and nothing will happen. - * - * @param inode the inode to be inserted - */ - public void insert(Inode inode) { - if (inode.getTtl() == Constants.NO_TTL) { - return; - } - - TtlBucket bucket; - while (true) { - bucket = getBucketContaining(inode); - if (bucket != null) { - break; - } - long ttlEndTimeMs = inode.getCreationTimeMs() + inode.getTtl(); - // No bucket contains the inode, so a new bucket should be added with an appropriate interval - // start. Assume the list of buckets have continuous intervals, and the first interval starts - // at 0, then ttlEndTimeMs should be in number (ttlEndTimeMs / interval) interval, so the - // start time of this interval should be (ttlEndTimeMs / interval) * interval. - long interval = TtlBucket.getTtlIntervalMs(); - bucket = new TtlBucket(interval == 0 ? ttlEndTimeMs : ttlEndTimeMs / interval * interval); - if (mBucketList.add(bucket)) { - break; - } - // If we reach here, it means the same bucket has been concurrently inserted by another - // thread. - } - // TODO(zhouyufa): Consider the concurrent situation that the bucket is expired and processed by - // the InodeTtlChecker, then adding the inode into the bucket is meaningless since the bucket - // will not be accessed again. (c.f. ALLUXIO-2821) - bucket.addInode(inode); - } - - /** - * Removes a inode from the bucket containing it if the inode is in one of the buckets, otherwise, - * do nothing. - * - *

- * Assume that no inode in the buckets has ttl value that equals {@link Constants#NO_TTL}. - * If a inode with valid ttl value is inserted to the buckets and its ttl value is going to be set - * to {@link Constants#NO_TTL} later, be sure to remove the inode from the buckets first. - * - * @param inode the inode to be removed - */ - public void remove(InodeView inode) { - TtlBucket bucket = getBucketContaining(inode); - if (bucket != null) { - bucket.removeInode(inode); - } - } - - /** - * Retrieves buckets whose ttl interval has expired before the specified time, that is, the - * bucket's interval start time should be less than or equal to (specified time - ttl interval). - * The returned set is backed by the internal set. - * - * @param time the expiration time - * @return a set of expired buckets or an empty set if no buckets have expired - */ - public Set getExpiredBuckets(long time) { - return mBucketList.headSet(new TtlBucket(time - TtlBucket.getTtlIntervalMs()), true); - } - - /** - * Removes all buckets in the set. - * - * @param buckets a set of buckets to be removed - */ - public void removeBuckets(Set buckets) { - mBucketList.removeAll(buckets); - } - - @Override - public CheckpointName getCheckpointName() { - return CheckpointName.TTL_BUCKET_LIST; - } - - @Override - public void writeToCheckpoint(OutputStream output) throws IOException, InterruptedException { - CheckpointOutputStream cos = new CheckpointOutputStream(output, CheckpointType.LONGS); - for (TtlBucket bucket : mBucketList) { - for (Inode inode : bucket.getInodes()) { - cos.writeLong(inode.getId()); - } - } - } - - @Override - public void restoreFromCheckpoint(CheckpointInputStream input) throws IOException { - mBucketList.clear(); - Preconditions.checkState(input.getType() == CheckpointType.LONGS, - "Unexpected checkpoint type: %s", input.getType()); - while (true) { - try { - long id = input.readLong(); - Optional inode = mInodeStore.get(id); - if (inode.isPresent()) { - insert(inode.get()); - } else { - LOG.error("Failed to find inode for id {}", id); - } - } catch (EOFException e) { - break; - } - } - } -} diff --git a/core/server/master/src/main/java/alluxio/master/file/replication/ReplicationChecker.java b/core/server/master/src/main/java/alluxio/master/file/replication/ReplicationChecker.java deleted file mode 100644 index 44e801dc29d9..000000000000 --- a/core/server/master/src/main/java/alluxio/master/file/replication/ReplicationChecker.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file.replication; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.client.job.JobMasterClientPool; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.BlockInfoException; -import alluxio.exception.FileDoesNotExistException; -import alluxio.exception.JobDoesNotExistException; -import alluxio.exception.status.ResourceExhaustedException; -import alluxio.exception.status.UnavailableException; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.job.plan.replicate.DefaultReplicationHandler; -import alluxio.job.plan.replicate.ReplicationHandler; -import alluxio.job.wire.Status; -import alluxio.master.SafeModeManager; -import alluxio.master.block.BlockMaster; -import alluxio.master.file.meta.InodeFile; -import alluxio.master.file.meta.InodeTree; -import alluxio.master.file.meta.InodeTree.LockPattern; -import alluxio.master.file.meta.LockedInodePath; -import alluxio.master.file.meta.PersistenceState; -import alluxio.master.journal.NoopJournalContext; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.util.logging.SamplingLogger; -import alluxio.wire.BlockInfo; -import alluxio.wire.BlockLocation; - -import com.google.common.collect.HashBiMap; -import com.google.common.collect.ImmutableSet; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.commons.lang3.tuple.Triple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.annotation.concurrent.ThreadSafe; - -/** - * The executor to check block replication level periodically and handle over-replicated and - * under-replicated blocks correspondingly. - */ -@ThreadSafe -public final class ReplicationChecker implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(ReplicationChecker.class); - private static final Logger SAMPLING_LOG = new SamplingLogger(LOG, 10L * Constants.MINUTE_MS); - - /** Maximum number of active jobs to be submitted to the job service. **/ - private final int mMaxActiveJobs; - - /** Handler to the inode tree. */ - private final InodeTree mInodeTree; - /** Handler to the block master. */ - private final BlockMaster mBlockMaster; - /** Handler for adjusting block replication level. */ - private final ReplicationHandler mReplicationHandler; - /** Manager of master safe mode state. */ - private final SafeModeManager mSafeModeManager; - - private final HashBiMap mActiveJobToInodeID; - - private enum Mode { - EVICT, - REPLICATE - } - - /** - * Constructs a new {@link ReplicationChecker} using default (job service) handler to replicate - * and evict blocks. - * - * @param inodeTree inode tree of the filesystem master - * @param blockMaster block master - * @param safeModeManager manager of master safe mode state - * @param jobMasterClientPool job master client pool - */ - public ReplicationChecker(InodeTree inodeTree, BlockMaster blockMaster, - SafeModeManager safeModeManager, JobMasterClientPool jobMasterClientPool) { - this(inodeTree, blockMaster, safeModeManager, - new DefaultReplicationHandler(jobMasterClientPool)); - } - - /** - * Constructs a new {@link ReplicationChecker} with specified replicate and evict handlers (for - * unit testing). - * - * @param inodeTree inode tree of the filesystem master - * @param blockMaster block master - * @param safeModeManager manager of master safe mode state - * @param replicationHandler handler to replicate blocks - */ - public ReplicationChecker(InodeTree inodeTree, BlockMaster blockMaster, - SafeModeManager safeModeManager, ReplicationHandler replicationHandler) { - mInodeTree = inodeTree; - mBlockMaster = blockMaster; - mSafeModeManager = safeModeManager; - mReplicationHandler = replicationHandler; - - // Do not use more than 10% of the job service - mMaxActiveJobs = Math.max(1, - (int) (Configuration.getLong(PropertyKey.JOB_MASTER_JOB_CAPACITY) * 0.1)); - mActiveJobToInodeID = HashBiMap.create(); - MetricsSystem.registerCachedGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.MASTER_REPLICA_MGMT_ACTIVE_JOB_SIZE.getName()), - mActiveJobToInodeID::size); - } - - private boolean shouldRun() { - // In unit tests there may not be workers, but we still want the ReplicationChecker to execute - if (Configuration.getBoolean(PropertyKey.TEST_MODE)) { - return true; - } - if (mSafeModeManager.isInSafeMode() || mBlockMaster.getWorkerCount() == 0) { - LOG.debug("Skip the ReplicationChecker in safe mode and when there are no workers"); - return false; - } - return true; - } - - /** - * {@inheritDoc} - * - * During this heartbeat, this class will check: - *

- * (1) Is there any block from the pinned files becomes under replicated (i.e., the number of - * existing copies is smaller than the target replication min for this file, possibly due to node - * failures), and schedule replicate jobs to increase the replication level when found; - * - * (2) Is there any blocks over replicated, schedule evict jobs to reduce the replication level. - */ - @Override - public void heartbeat() throws InterruptedException { - if (!shouldRun()) { - return; - } - final Set activeJobIds = new HashSet<>(); - try { - if (!mActiveJobToInodeID.isEmpty()) { - final List activeEvictJobIds = - mReplicationHandler.findJobs("Evict", - ImmutableSet.of(Status.RUNNING, Status.CREATED)); - final List activeMoveJobIds = - mReplicationHandler.findJobs("Move", - ImmutableSet.of(Status.RUNNING, Status.CREATED)); - final List activeReplicateJobIds = - mReplicationHandler.findJobs("Replicate", - ImmutableSet.of(Status.RUNNING, Status.CREATED)); - - activeJobIds.addAll(activeEvictJobIds); - activeJobIds.addAll(activeMoveJobIds); - activeJobIds.addAll(activeReplicateJobIds); - mActiveJobToInodeID.keySet().removeIf(jobId -> !activeJobIds.contains(jobId)); - } - } catch (IOException e) { - // It is possible the job master process is not answering rpcs, - // log but do not throw the exception - // which will kill the replication checker thread. - LOG.debug("Failed to contact job master to get updated list of replication jobs", e); - } - - Set inodes; - - // Check the set of files that could possibly be under-replicated - inodes = mInodeTree.getPinIdSet(); - check(inodes, mReplicationHandler, Mode.REPLICATE); - - // Check the set of files that could possibly be over-replicated - inodes = mInodeTree.getReplicationLimitedFileIds(); - check(inodes, mReplicationHandler, Mode.EVICT); - - // Check the set of files that could possibly be mis-replicated - inodes = mInodeTree.getPinIdSet(); - checkMisreplicated(inodes, mReplicationHandler); - } - - @Override - public void close() { - // Nothing to clean up - } - - /** - * Find a set of from and to locations for our moveBlock operation. - * After the move, the file will be in the desired pinned medium at least minReplication times. - * - * @param file the file to scan for misplaced blocks - * @param blockInfo blockInfo containing information about the block locations - * @return a map that maps from workerHost to desired medium - */ - private Map findMisplacedBlock( - InodeFile file, BlockInfo blockInfo) { - Set pinnedMediumTypes = file.getMediumTypes(); - if (pinnedMediumTypes.isEmpty()) { - // nothing needs to be moved - return Collections.emptyMap(); - } - Map movement = new HashMap<>(); - // at least pinned to one medium type - String firstPinnedMedium = pinnedMediumTypes.iterator().next(); - int minReplication = file.getReplicationMin(); - int correctReplication = 0; - List candidates = new ArrayList<>(); - for (BlockLocation loc: blockInfo.getLocations()) { - if (pinnedMediumTypes.contains(loc.getMediumType())) { - correctReplication++; - } else { - candidates.add(loc.getWorkerAddress().getHost()); - } - } - if (correctReplication >= minReplication) { - // there are more than minReplication in the right location - return Collections.emptyMap(); - } else { - int toMove = minReplication - correctReplication; - for (String candidate : candidates) { - // We are only addressing pinning to a single medium case, in the future we might - // address pinning to multiple medium types. - movement.put(candidate, firstPinnedMedium); - toMove--; - if (toMove == 0) { - return movement; - } - } - } - return movement; - } - - private void checkMisreplicated(Set inodes, ReplicationHandler handler) - throws InterruptedException { - for (long inodeId : inodes) { - if (mActiveJobToInodeID.size() >= mMaxActiveJobs) { - return; - } - if (mActiveJobToInodeID.containsValue(inodeId)) { - continue; - } - // Throw if interrupted. - if (Thread.interrupted()) { - throw new InterruptedException("ReplicationChecker interrupted."); - } - try (LockedInodePath inodePath = - mInodeTree.lockFullInodePath(inodeId, LockPattern.READ, NoopJournalContext.INSTANCE) - ) { - InodeFile file = inodePath.getInodeFile(); - for (long blockId : file.getBlockIds()) { - BlockInfo blockInfo = null; - try { - blockInfo = mBlockMaster.getBlockInfo(blockId); - } catch (BlockInfoException e) { - // Cannot find this block in Alluxio from BlockMaster, possibly persisted in UFS - } catch (UnavailableException e) { - // The block master is not available, wait for the next heartbeat - LOG.warn("The block master is not available: {}", e.toString()); - return; - } - if (blockInfo == null) { - // no block info available, we simply log and return; - LOG.warn("Block info is null"); - return; - } - - for (Map.Entry entry - : findMisplacedBlock(file, blockInfo).entrySet()) { - try { - final long jobId = - handler.migrate(inodePath.getUri(), blockId, entry.getKey(), entry.getValue()); - mActiveJobToInodeID.put(jobId, inodeId); - } catch (Exception e) { - LOG.warn( - "Unexpected exception encountered when starting a migration job (uri={}," - + " block ID={}, workerHost= {}) : {}", - inodePath.getUri(), blockId, entry.getKey(), e.toString()); - LOG.debug("Exception: ", e); - } - } - } - } catch (FileDoesNotExistException e) { - LOG.warn("Failed to check replication level for inode id {} : {}", inodeId, e.toString()); - } - } - } - - private Set check(Set inodes, ReplicationHandler handler, Mode mode) - throws InterruptedException { - Set processedFileIds = new HashSet<>(); - for (long inodeId : inodes) { - if (mActiveJobToInodeID.size() >= mMaxActiveJobs) { - return processedFileIds; - } - if (mActiveJobToInodeID.containsValue(inodeId)) { - continue; - } - Set> requests = new HashSet<>(); - // Throw if interrupted. - if (Thread.interrupted()) { - throw new InterruptedException("ReplicationChecker interrupted."); - } - // TODO(binfan): calling lockFullInodePath locks the entire path from root to the target - // file and may increase lock contention in this tree. Investigate if we could avoid - // locking the entire path but just the inode file since this access is read-only. - try (LockedInodePath inodePath = mInodeTree.lockFullInodePath( - inodeId, LockPattern.READ, NoopJournalContext.INSTANCE) - ) { - InodeFile file = inodePath.getInodeFile(); - for (long blockId : file.getBlockIds()) { - BlockInfo blockInfo = null; - try { - blockInfo = mBlockMaster.getBlockInfo(blockId); - } catch (BlockInfoException e) { - // Cannot find this block in Alluxio from BlockMaster, possibly persisted in UFS - } catch (UnavailableException e) { - // The block master is not available, wait for the next heartbeat - LOG.warn("The block master is not available: {}", e.toString()); - return processedFileIds; - } - int currentReplicas = (blockInfo == null) ? 0 : blockInfo.getLocations().size(); - switch (mode) { - case EVICT: - int maxReplicas = file.getReplicationMax(); - if (file.getPersistenceState() == PersistenceState.TO_BE_PERSISTED - && file.getReplicationDurable() > maxReplicas) { - maxReplicas = file.getReplicationDurable(); - } - if (currentReplicas > maxReplicas) { - requests.add(new ImmutableTriple<>(inodePath.getUri(), blockId, - maxReplicas)); - } - break; - case REPLICATE: - int minReplicas = file.getReplicationMin(); - if (file.getPersistenceState() == PersistenceState.TO_BE_PERSISTED - && file.getReplicationDurable() > minReplicas) { - minReplicas = file.getReplicationDurable(); - } - if (currentReplicas < minReplicas) { - // if this file is not persisted and block master thinks it is lost, no effort made - if (!file.isPersisted() && mBlockMaster.isBlockLost(blockId)) { - continue; - } - requests.add(new ImmutableTriple<>(inodePath.getUri(), blockId, - minReplicas)); - } - break; - default: - LOG.warn("Unexpected replication mode {}.", mode); - } - } - } catch (FileDoesNotExistException e) { - LOG.warn("Failed to check replication level for inode id {} : {}", inodeId, e.toString()); - } - - for (Triple entry : requests) { - AlluxioURI uri = entry.getLeft(); - long blockId = entry.getMiddle(); - int numReplicas = entry.getRight(); - try { - long jobId; - switch (mode) { - case EVICT : - case REPLICATE: - jobId = handler.setReplica(uri, blockId, numReplicas); - break; - default: - throw new RuntimeException(String.format("Unexpected replication mode %s.", mode)); - } - processedFileIds.add(inodeId); - mActiveJobToInodeID.put(jobId, inodeId); - } catch (JobDoesNotExistException | ResourceExhaustedException e) { - LOG.warn("The job service is busy, will retry later. {}", e.toString()); - return processedFileIds; - } catch (UnavailableException e) { - LOG.warn("Unable to complete the replication check: {}, will retry later.", e.toString()); - return processedFileIds; - } catch (Exception e) { - SAMPLING_LOG.warn( - "Unexpected exception encountered when starting a {} job (uri={}," - + " block ID={}, num replicas={}) : {}", - mode, uri, blockId, numReplicas, e.toString()); - LOG.debug("Job service unexpected exception: ", e); - } - } - } - return processedFileIds; - } -} diff --git a/core/server/master/src/main/java/alluxio/master/journal/JournalMasterClientServiceHandler.java b/core/server/master/src/main/java/alluxio/master/journal/JournalMasterClientServiceHandler.java deleted file mode 100644 index 37da2fcf39d8..000000000000 --- a/core/server/master/src/main/java/alluxio/master/journal/JournalMasterClientServiceHandler.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal; - -import alluxio.RpcUtils; -import alluxio.grpc.GetNodeStatePRequest; -import alluxio.grpc.GetNodeStatePResponse; -import alluxio.grpc.GetQuorumInfoPRequest; -import alluxio.grpc.GetQuorumInfoPResponse; -import alluxio.grpc.GetTransferLeaderMessagePRequest; -import alluxio.grpc.GetTransferLeaderMessagePResponse; -import alluxio.grpc.JournalMasterClientServiceGrpc; -import alluxio.grpc.RemoveQuorumServerPRequest; -import alluxio.grpc.RemoveQuorumServerPResponse; -import alluxio.grpc.ResetPrioritiesPRequest; -import alluxio.grpc.ResetPrioritiesPResponse; -import alluxio.grpc.TransferLeadershipPRequest; -import alluxio.grpc.TransferLeadershipPResponse; - -import io.grpc.stub.StreamObserver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is a gRPC handler for journal master RPCs invoked by an Alluxio client. - */ -public class JournalMasterClientServiceHandler - extends JournalMasterClientServiceGrpc.JournalMasterClientServiceImplBase { - private static final Logger LOG = - LoggerFactory.getLogger(JournalMasterClientServiceHandler.class); - - private final JournalMaster mJournalMaster; - - /** - * Creates gRPC service handler for JobMaster service. - * - * @param journalMaster the journal master - */ - public JournalMasterClientServiceHandler(JournalMaster journalMaster) { - mJournalMaster = journalMaster; - } - - @Override - public void getQuorumInfo(GetQuorumInfoPRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, mJournalMaster::getQuorumInfo, "getQuorumInfo", "request=%s", - responseObserver, request); - } - - @Override - public void removeQuorumServer(RemoveQuorumServerPRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, () -> { - mJournalMaster.removeQuorumServer(request.getServerAddress()); - return RemoveQuorumServerPResponse.getDefaultInstance(); - }, "removeQuorumServer", "request=%s", responseObserver, request); - } - - @Override - public void transferLeadership(TransferLeadershipPRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, () -> { - String transferId = mJournalMaster.transferLeadership(request.getServerAddress()); - return TransferLeadershipPResponse.newBuilder().setTransferId(transferId).build(); - }, "transferLeadership", "request=%s", responseObserver, request); - } - - @Override - public void resetPriorities(ResetPrioritiesPRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, () -> { - mJournalMaster.resetPriorities(); - return ResetPrioritiesPResponse.getDefaultInstance(); - }, "resetPriorities", "request=%s", responseObserver, request); - } - - @Override - public void getTransferLeaderMessage(GetTransferLeaderMessagePRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, () -> mJournalMaster.getTransferLeaderMessage(request.getTransferId()), - "GetTransferLeaderMessage", "request=%s", responseObserver, request); - } - - @Override - public void getNodeState(GetNodeStatePRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, mJournalMaster::getNodeState, - "GetNodeState", "request=%s", responseObserver, request); - } -} diff --git a/core/server/master/src/main/java/alluxio/master/meta/DefaultMetaMaster.java b/core/server/master/src/main/java/alluxio/master/meta/DefaultMetaMaster.java deleted file mode 100644 index 8b1631c4f43e..000000000000 --- a/core/server/master/src/main/java/alluxio/master/meta/DefaultMetaMaster.java +++ /dev/null @@ -1,716 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.meta; - -import alluxio.ClientContext; -import alluxio.Constants; -import alluxio.Server; -import alluxio.clock.SystemClock; -import alluxio.collections.IndexDefinition; -import alluxio.collections.IndexedSet; -import alluxio.conf.Configuration; -import alluxio.conf.ConfigurationValueOptions; -import alluxio.conf.PropertyKey; -import alluxio.conf.Source; -import alluxio.exception.AlluxioException; -import alluxio.exception.status.NotFoundException; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.BackupPOptions; -import alluxio.grpc.BackupPRequest; -import alluxio.grpc.BackupStatusPRequest; -import alluxio.grpc.GetConfigurationPOptions; -import alluxio.grpc.GrpcService; -import alluxio.grpc.MetaCommand; -import alluxio.grpc.RegisterMasterPOptions; -import alluxio.grpc.Scope; -import alluxio.grpc.ServiceType; -import alluxio.heartbeat.HeartbeatContext; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.heartbeat.HeartbeatThread; -import alluxio.master.CoreMaster; -import alluxio.master.CoreMasterContext; -import alluxio.master.MasterClientContext; -import alluxio.master.StateLockOptions; -import alluxio.master.backup.BackupLeaderRole; -import alluxio.master.backup.BackupRole; -import alluxio.master.backup.BackupWorkerRole; -import alluxio.master.block.BlockMaster; -import alluxio.master.journal.JournalContext; -import alluxio.master.journal.JournalType; -import alluxio.master.journal.checkpoint.CheckpointName; -import alluxio.master.meta.checkconf.ConfigurationChecker; -import alluxio.master.meta.checkconf.ConfigurationStore; -import alluxio.proto.journal.Journal; -import alluxio.proto.journal.Meta; -import alluxio.resource.CloseableIterator; -import alluxio.underfs.UfsManager; -import alluxio.util.ConfigurationUtils; -import alluxio.util.IdUtils; -import alluxio.util.OSUtils; -import alluxio.util.ThreadFactoryUtils; -import alluxio.util.executor.ExecutorServiceFactories; -import alluxio.util.executor.ExecutorServiceFactory; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.Address; -import alluxio.wire.BackupStatus; -import alluxio.wire.ConfigCheckReport; -import alluxio.wire.ConfigHash; - -import com.google.common.collect.ImmutableSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.text.MessageFormat; -import java.time.Clock; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Executors; -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * The default meta master. - */ -@NotThreadSafe -public final class DefaultMetaMaster extends CoreMaster implements MetaMaster { - private static final Logger LOG = LoggerFactory.getLogger(DefaultMetaMaster.class); - private static final Set> DEPS = ImmutableSet.of(BlockMaster.class); - - // Master metadata management. - private static final IndexDefinition ID_INDEX = - IndexDefinition.ofUnique(MasterInfo::getId); - - private static final IndexDefinition ADDRESS_INDEX = - IndexDefinition.ofUnique(MasterInfo::getAddress); - - /** Core master context. */ - private final CoreMasterContext mCoreMasterContext; - - /** The clock to use for determining the time. */ - private final Clock mClock = new SystemClock(); - - /** The master configuration store. */ - private final ConfigurationStore mMasterConfigStore = new ConfigurationStore(); - /** The worker configuration store. */ - private final ConfigurationStore mWorkerConfigStore = new ConfigurationStore(); - /** The server-side configuration checker. */ - private final ConfigurationChecker mConfigChecker = - new ConfigurationChecker(mMasterConfigStore, mWorkerConfigStore); - - /** Keeps track of standby masters which are in communication with the leader master. */ - private final IndexedSet mMasters = - new IndexedSet<>(ID_INDEX, ADDRESS_INDEX); - /** Keeps track of standby masters which are no longer in communication with the leader master. */ - private final IndexedSet mLostMasters = - new IndexedSet<>(ID_INDEX, ADDRESS_INDEX); - - /** The connect address for the rpc server. */ - private final InetSocketAddress mRpcConnectAddress - = NetworkAddressUtils.getConnectAddress(NetworkAddressUtils.ServiceType.MASTER_RPC, - Configuration.global()); - - /** Indicates if newer version is available. */ - private boolean mNewerVersionAvailable; - - /** The address of this master. */ - private final Address mMasterAddress; - - /** The manager of all ufs. */ - private final UfsManager mUfsManager; - - /** The metadata daily backup. */ - private DailyMetadataBackup mDailyBackup; - - /** Path level properties. */ - private final PathProperties mPathProperties; - - /** Persisted state for MetaMaster. */ - private final State mState; - - /** Value to be used for the cluster ID when not assigned. */ - public static final String INVALID_CLUSTER_ID = "INVALID_CLUSTER_ID"; - - /** Used to manage backup role. */ - private BackupRole mBackupRole; - - @Nullable - private final JournalSpaceMonitor mJournalSpaceMonitor; - - /** - * Journaled state for MetaMaster. - */ - @NotThreadSafe - public static final class State implements alluxio.master.journal.Journaled { - /** A unique ID to identify the cluster. */ - private String mClusterID = INVALID_CLUSTER_ID; - - /** - * @return the cluster ID - */ - public String getClusterID() { - return mClusterID; - } - - @Override - public CheckpointName getCheckpointName() { - return CheckpointName.CLUSTER_INFO; - } - - @Override - public boolean processJournalEntry(Journal.JournalEntry entry) { - if (entry.hasClusterInfo()) { - mClusterID = entry.getClusterInfo().getClusterId(); - return true; - } - return false; - } - - /** - * @param ctx the journal context - * @param clusterId the clusterId journal clusterId - */ - public void applyAndJournal(java.util.function.Supplier ctx, String clusterId) { - applyAndJournal(ctx, - Journal.JournalEntry.newBuilder() - .setClusterInfo(Meta.ClusterInfoEntry.newBuilder().setClusterId(clusterId).build()) - .build()); - } - - @Override - public void resetState() { - mClusterID = INVALID_CLUSTER_ID; - } - - @Override - public CloseableIterator getJournalEntryIterator() { - if (mClusterID.equals(INVALID_CLUSTER_ID)) { - return CloseableIterator.noopCloseable(Collections.emptyIterator()); - } - return CloseableIterator.noopCloseable(Collections.singleton(Journal.JournalEntry.newBuilder() - .setClusterInfo(Meta.ClusterInfoEntry.newBuilder().setClusterId(mClusterID).build()) - .build()).iterator()); - } - } - - /** - * Creates a new instance of {@link DefaultMetaMaster}. - * - * @param blockMaster a block master handle - * @param masterContext the context for Alluxio master - */ - DefaultMetaMaster(BlockMaster blockMaster, CoreMasterContext masterContext) { - this(blockMaster, masterContext, - ExecutorServiceFactories.cachedThreadPool(Constants.META_MASTER_NAME)); - } - - /** - * Creates a new instance of {@link DefaultMetaMaster}. - * - * @param blockMaster a block master handle - * @param masterContext the context for Alluxio master - * @param executorServiceFactory a factory for creating the executor service to use for running - * maintenance threads - */ - DefaultMetaMaster(BlockMaster blockMaster, CoreMasterContext masterContext, - ExecutorServiceFactory executorServiceFactory) { - super(masterContext, new SystemClock(), executorServiceFactory); - mCoreMasterContext = masterContext; - mMasterAddress = - new Address().setHost(Configuration.getOrDefault(PropertyKey.MASTER_HOSTNAME, - mRpcConnectAddress.getHostName())) - .setRpcPort(mPort); - /* Handle to the block master. */ - blockMaster.registerLostWorkerFoundListener(mWorkerConfigStore::lostNodeFound); - blockMaster.registerWorkerLostListener(mWorkerConfigStore::handleNodeLost); - blockMaster.registerNewWorkerConfListener(mWorkerConfigStore::registerNewConf); - blockMaster.registerWorkerDeleteListener(mWorkerConfigStore::handleNodeDelete); - - mUfsManager = masterContext.getUfsManager(); - - mPathProperties = new PathProperties(); - mState = new State(); - if (Configuration.getEnum(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.class) - .equals(JournalType.EMBEDDED) && OSUtils.isLinux()) { - mJournalSpaceMonitor = new JournalSpaceMonitor(Configuration.global()); - } else { - mJournalSpaceMonitor = null; - } - } - - @Override - public Map getServices() { - Map services = new HashMap<>(); - services.put(ServiceType.META_MASTER_CONFIG_SERVICE, - new GrpcService(new MetaMasterConfigurationServiceHandler(this)).disableAuthentication()); - services.put(ServiceType.META_MASTER_CLIENT_SERVICE, - new GrpcService(new MetaMasterClientServiceHandler(this))); - services.put(ServiceType.META_MASTER_MASTER_SERVICE, - new GrpcService(new MetaMasterMasterServiceHandler(this))); - // Add backup role services. - services.putAll(mBackupRole.getRoleServices()); - services.putAll(mJournalSystem.getJournalServices()); - return services; - } - - @Override - public String getName() { - return Constants.META_MASTER_NAME; - } - - @Override - public Set> getDependencies() { - return DEPS; - } - - @Override - public void start(Boolean isPrimary) throws IOException { - super.start(isPrimary); - mWorkerConfigStore.reset(); - mMasterConfigStore.reset(); - if (isPrimary) { - // Add the configuration of the current leader master - mMasterConfigStore.registerNewConf(mMasterAddress, - Configuration.getConfiguration(Scope.MASTER)); - - // The service that detects lost standby master nodes - getExecutorService().submit(new HeartbeatThread( - HeartbeatContext.MASTER_LOST_MASTER_DETECTION, - new LostMasterDetectionHeartbeatExecutor(), - (int) Configuration.getMs(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL), - Configuration.global(), mMasterContext.getUserState())); - getExecutorService().submit( - new HeartbeatThread(HeartbeatContext.MASTER_LOG_CONFIG_REPORT_SCHEDULING, - new LogConfigReportHeartbeatExecutor(), - (int) Configuration - .getMs(PropertyKey.MASTER_LOG_CONFIG_REPORT_HEARTBEAT_INTERVAL), - Configuration.global(), mMasterContext.getUserState())); - - if (Configuration.getBoolean(PropertyKey.MASTER_DAILY_BACKUP_ENABLED)) { - mDailyBackup = new DailyMetadataBackup(this, Executors.newSingleThreadScheduledExecutor( - ThreadFactoryUtils.build("DailyMetadataBackup-%d", true)), mUfsManager); - mDailyBackup.start(); - } - if (mJournalSpaceMonitor != null) { - getExecutorService().submit(new HeartbeatThread( - HeartbeatContext.MASTER_JOURNAL_SPACE_MONITOR, mJournalSpaceMonitor, - Configuration.getMs(PropertyKey.MASTER_JOURNAL_SPACE_MONITOR_INTERVAL), - Configuration.global(), mMasterContext.getUserState())); - } - if (mState.getClusterID().equals(INVALID_CLUSTER_ID)) { - try (JournalContext context = createJournalContext()) { - String clusterID = java.util.UUID.randomUUID().toString(); - mState.applyAndJournal(context, clusterID); - LOG.info("Created new cluster ID {}", clusterID); - } - if (Configuration.getBoolean(PropertyKey.MASTER_UPDATE_CHECK_ENABLED) - && !Configuration.getBoolean(PropertyKey.TEST_MODE)) { - getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_UPDATE_CHECK, - new UpdateChecker(this), - (int) Configuration.getMs(PropertyKey.MASTER_UPDATE_CHECK_INTERVAL), - Configuration.global(), mMasterContext.getUserState())); - } - } else { - LOG.info("Detected existing cluster ID {}", mState.getClusterID()); - } - mBackupRole = new BackupLeaderRole(mCoreMasterContext); - } else { - if (ConfigurationUtils.isHaMode(Configuration.global())) { - // Standby master should setup MetaMasterSync to communicate with the leader master - RetryHandlingMetaMasterMasterClient metaMasterClient = - new RetryHandlingMetaMasterMasterClient(MasterClientContext - .newBuilder(ClientContext.create(Configuration.global())).build()); - getExecutorService().submit(new HeartbeatThread(HeartbeatContext.META_MASTER_SYNC, - new MetaMasterSync(mMasterAddress, metaMasterClient), - (int) Configuration.getMs(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL), - Configuration.global(), mMasterContext.getUserState())); - LOG.info("Standby master with address {} starts sending heartbeat to leader master.", - mMasterAddress); - } - // Enable worker role if backup delegation is enabled. - if (Configuration.getBoolean(PropertyKey.MASTER_BACKUP_DELEGATION_ENABLED)) { - mBackupRole = new BackupWorkerRole(mCoreMasterContext); - } - } - } - - @Override - public void stop() throws IOException { - if (mDailyBackup != null) { - mDailyBackup.stop(); - mDailyBackup = null; - } - if (mBackupRole != null) { - mBackupRole.close(); - mBackupRole = null; - } - super.stop(); - } - - /** - * Overrides current backup role and forces the master to take a local backup. - * @return the {@link BackupStatus} - * @throws AlluxioException if it encounters issues triggering the backup - */ - public BackupStatus takeEmergencyBackup() throws AlluxioException { - mBackupRole = new BackupLeaderRole(mCoreMasterContext); - BackupPRequest request = BackupPRequest.newBuilder() - .setOptions(BackupPOptions.newBuilder() - .setAllowLeader(true) - .setBypassDelegation(true) - .setRunAsync(false) - .build()) - .build(); - return backup(request, StateLockOptions.defaults()); - } - - @Override - public BackupStatus backup(BackupPRequest request, StateLockOptions stateLockOptions) - throws AlluxioException { - return mBackupRole.backup(request, stateLockOptions); - } - - @Override - public BackupStatus getBackupStatus(BackupStatusPRequest statusPRequest) throws AlluxioException { - return mBackupRole.getBackupStatus(statusPRequest); - } - - @Override - public String checkpoint() throws IOException { - mJournalSystem.checkpoint(mMasterContext.getStateLockManager()); - return NetworkAddressUtils.getConnectHost(NetworkAddressUtils.ServiceType.MASTER_RPC, - Configuration.global()); - } - - @Override - public ConfigCheckReport getConfigCheckReport() { - return mConfigChecker.getConfigCheckReport(); - } - - @Override - public alluxio.wire.Configuration getConfiguration(GetConfigurationPOptions options) { - // NOTE(cc): there is no guarantee that the returned cluster and path configurations are - // consistent snapshot of the system's state at a certain time, the path configuration might - // be in a newer state. But it's guaranteed that the hashes are respectively correspondent to - // the properties. - alluxio.wire.Configuration.Builder builder = alluxio.wire.Configuration.newBuilder(); - - if (!options.getIgnoreClusterConf()) { - for (PropertyKey key : Configuration.keySet()) { - if (key.isBuiltIn()) { - Source source = Configuration.getSource(key); - Object value = Configuration.getOrDefault(key, null, - ConfigurationValueOptions.defaults().useDisplayValue(true) - .useRawValue(options.getRawValue())); - builder.addClusterProperty(key.getName(), value, source); - } - } - // NOTE(cc): assumes that Configuration is read-only when master is running, otherwise, - // the following hash might not correspond to the above cluster configuration. - builder.setClusterConfHash(Configuration.hash()); - } - - if (!options.getIgnorePathConf()) { - PathPropertiesView pathProperties = mPathProperties.snapshot(); - pathProperties.getProperties().forEach((path, properties) -> - properties.forEach((key, value) -> - builder.addPathProperty(path, key, value))); - builder.setPathConfHash(pathProperties.getHash()); - } - - return builder.build(); - } - - @Override - public ConfigHash getConfigHash() { - return new ConfigHash(Configuration.hash(), mPathProperties.hash()); - } - - @Override - public Optional getJournalSpaceMonitor() { - return Optional.ofNullable(mJournalSpaceMonitor); - } - - @Override - public void setPathConfiguration(String path, Map properties) - throws UnavailableException { - try (JournalContext ctx = createJournalContext()) { - mPathProperties.add(ctx, path, properties); - } - } - - @Override - public void removePathConfiguration(String path, Set keys) - throws UnavailableException { - try (JournalContext ctx = createJournalContext()) { - mPathProperties.remove(ctx, path, keys); - } - } - - @Override - public void removePathConfiguration(String path) throws UnavailableException { - try (JournalContext ctx = createJournalContext()) { - mPathProperties.removeAll(ctx, path); - } - } - - @Override - public void setNewerVersionAvailable(boolean available) { - mNewerVersionAvailable = available; - } - - @Override - public boolean getNewerVersionAvailable() { - return mNewerVersionAvailable; - } - - @Override - public List

getMasterAddresses() { - return mMasterConfigStore.getLiveNodeAddresses(); - } - - @Override - public List
getWorkerAddresses() { - return mWorkerConfigStore.getLiveNodeAddresses(); - } - - @Override - public alluxio.wire.MasterInfo[] getMasterInfos() { - alluxio.wire.MasterInfo[] masterInfos = new alluxio.wire.MasterInfo[mMasters.size()]; - int indexNum = 0; - for (MasterInfo master : mMasters) { - masterInfos[indexNum] = new alluxio.wire.MasterInfo(master.getId(), - master.getAddress(), master.getLastUpdatedTimeMs()); - indexNum++; - } - return masterInfos; - } - - @Override - public alluxio.wire.MasterInfo[] getLostMasterInfos() { - alluxio.wire.MasterInfo[] masterInfos = new alluxio.wire.MasterInfo[mLostMasters.size()]; - int indexNum = 0; - for (MasterInfo master : mLostMasters) { - masterInfos[indexNum] = new alluxio.wire.MasterInfo(master.getId(), - master.getAddress(), master.getLastUpdatedTimeMs()); - indexNum++; - } - return masterInfos; - } - - @Override - public long getMasterId(Address address) { - MasterInfo existingMaster = mMasters.getFirstByField(ADDRESS_INDEX, address); - if (existingMaster != null) { - // This master address is already mapped to a master id. - long oldMasterId = existingMaster.getId(); - LOG.warn("The master {} already exists as id {}.", address, oldMasterId); - return oldMasterId; - } - - MasterInfo lostMaster = mLostMasters.getFirstByField(ADDRESS_INDEX, address); - if (lostMaster != null) { - // This is one of the lost masters - mMasterConfigStore.lostNodeFound(lostMaster.getAddress()); - synchronized (lostMaster) { - final long lostMasterId = lostMaster.getId(); - LOG.warn("A lost master {} has requested its old id {}.", address, lostMasterId); - - // Update the timestamp of the master before it is considered an active master. - lostMaster.updateLastUpdatedTimeMs(); - mMasters.add(lostMaster); - mLostMasters.remove(lostMaster); - return lostMasterId; - } - } - - // Generate a new master id. - long masterId = IdUtils.getRandomNonNegativeLong(); - while (!mMasters.add(new MasterInfo(masterId, address))) { - masterId = IdUtils.getRandomNonNegativeLong(); - } - - LOG.info("getMasterId(): MasterAddress: {} id: {}", address, masterId); - return masterId; - } - - @Override - public InetSocketAddress getRpcAddress() { - return mRpcConnectAddress; - } - - @Override - public long getStartTimeMs() { - return mStartTimeMs; - } - - @Override - public long getUptimeMs() { - return System.currentTimeMillis() - mStartTimeMs; - } - - @Override - public int getWebPort() { - return Configuration.getInt(PropertyKey.MASTER_WEB_PORT); - } - - @Override - public boolean isInSafeMode() { - return mSafeModeManager.isInSafeMode(); - } - - @Override - public MetaCommand masterHeartbeat(long masterId) { - MasterInfo master = mMasters.getFirstByField(ID_INDEX, masterId); - if (master == null) { - LOG.warn("Could not find master id: {} for heartbeat.", masterId); - return MetaCommand.MetaCommand_Register; - } - - master.updateLastUpdatedTimeMs(); - return MetaCommand.MetaCommand_Nothing; - } - - @Override - public void masterRegister(long masterId, RegisterMasterPOptions options) - throws NotFoundException { - MasterInfo master = mMasters.getFirstByField(ID_INDEX, masterId); - if (master == null) { - throw new NotFoundException( - MessageFormat.format("No master with masterId {0,number,#} is found", masterId)); - } - - master.updateLastUpdatedTimeMs(); - - mMasterConfigStore.registerNewConf(master.getAddress(), options.getConfigsList()); - - LOG.info("registerMaster(): master: {}", master); - } - - @Override - public CheckpointName getCheckpointName() { - return CheckpointName.META_MASTER; - } - - @Override - public String getClusterID() { - return mState.getClusterID(); - } - - @Override - public CloseableIterator getJournalEntryIterator() { - return CloseableIterator.concat(mPathProperties.getJournalEntryIterator(), - mState.getJournalEntryIterator()); - } - - @Override - public boolean processJournalEntry(Journal.JournalEntry entry) { - return mState.processJournalEntry(entry) || mPathProperties.processJournalEntry(entry); - } - - @Override - public void resetState() { - mState.resetState(); - mPathProperties.resetState(); - } - - @Override - public Map updateConfiguration(Map propertiesMap) { - Map result = new HashMap<>(); - int successCount = 0; - for (Map.Entry entry : propertiesMap.entrySet()) { - try { - PropertyKey key = PropertyKey.fromString(entry.getKey()); - if (Configuration.getBoolean(PropertyKey.CONF_DYNAMIC_UPDATE_ENABLED) - && key.isDynamic()) { - Object oldValue = Configuration.get(key); - Object value = key.parseValue(entry.getValue()); - Configuration.set(key, value, Source.RUNTIME); - result.put(entry.getKey(), true); - successCount++; - LOG.info("Property {} has been updated to \"{}\" from \"{}\"", - key.getName(), entry.getValue(), oldValue); - } else { - LOG.warn("Update a non-dynamic property {} is not allowed", key.getName()); - result.put(entry.getKey(), false); - } - } catch (Exception e) { - result.put(entry.getKey(), false); - LOG.error("Failed to update property {} to {}", entry.getKey(), entry.getValue(), e); - } - } - LOG.debug("Update {} properties, succeed {}.", propertiesMap.size(), successCount); - return result; - } - - /** - * Lost master periodic check. - */ - private final class LostMasterDetectionHeartbeatExecutor implements HeartbeatExecutor { - - /** - * Constructs a new {@link LostMasterDetectionHeartbeatExecutor}. - */ - public LostMasterDetectionHeartbeatExecutor() { - } - - @Override - public void heartbeat() { - long masterTimeoutMs = Configuration.getMs(PropertyKey.MASTER_HEARTBEAT_TIMEOUT); - for (MasterInfo master : mMasters) { - synchronized (master) { - final long lastUpdate = mClock.millis() - master.getLastUpdatedTimeMs(); - if (lastUpdate > masterTimeoutMs) { - LOG.error("The master {}({}) timed out after {}ms without a heartbeat!", master.getId(), - master.getAddress(), lastUpdate); - mLostMasters.add(master); - mMasters.remove(master); - mMasterConfigStore.handleNodeLost(master.getAddress()); - } - } - } - } - - @Override - public void close() { - // Nothing to clean up - } - } - - /** - * Periodically log the config check report. - */ - private final class LogConfigReportHeartbeatExecutor implements HeartbeatExecutor { - private volatile boolean mFirst = true; - - @Override - public void heartbeat() { - // Skip the first heartbeat since it happens before servers have time to register their - // configurations. - if (mFirst) { - mFirst = false; - } else { - mConfigChecker.logConfigReport(); - } - } - - @Override - public void close() { - // Nothing to clean up - } - } -} diff --git a/core/server/master/src/main/java/alluxio/master/meta/MasterInfo.java b/core/server/master/src/main/java/alluxio/master/meta/MasterInfo.java deleted file mode 100644 index 5b7742620886..000000000000 --- a/core/server/master/src/main/java/alluxio/master/meta/MasterInfo.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.meta; - -import alluxio.wire.Address; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; - -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Master information. - */ -@NotThreadSafe -public final class MasterInfo { - /** Master's address. */ - private final Address mAddress; - /** The id of the master. */ - private final long mId; - /** Master's last updated time in ms. */ - private long mLastUpdatedTimeMs; - - /** - * Creates a new instance of {@link MasterInfo}. - * - * @param id the master id to use - * @param address the master address to use - */ - public MasterInfo(long id, Address address) { - mAddress = Preconditions.checkNotNull(address, "address"); - mId = id; - mLastUpdatedTimeMs = System.currentTimeMillis(); - } - - /** - * @return the master's address - */ - public Address getAddress() { - return mAddress; - } - - /** - * @return the id of the master - */ - public long getId() { - return mId; - } - - /** - * @return the last updated time of the master in ms - */ - public long getLastUpdatedTimeMs() { - return mLastUpdatedTimeMs; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("id", mId).add("address", mAddress) - .add("lastUpdatedTimeMs", mLastUpdatedTimeMs).toString(); - } - - /** - * Updates the last updated time of the master in ms. - */ - public void updateLastUpdatedTimeMs() { - mLastUpdatedTimeMs = System.currentTimeMillis(); - } -} diff --git a/core/server/master/src/main/java/alluxio/master/meta/RetryHandlingMetaMasterMasterClient.java b/core/server/master/src/main/java/alluxio/master/meta/RetryHandlingMetaMasterMasterClient.java deleted file mode 100644 index 3f155c9185b5..000000000000 --- a/core/server/master/src/main/java/alluxio/master/meta/RetryHandlingMetaMasterMasterClient.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.meta; - -import alluxio.AbstractMasterClient; -import alluxio.Constants; -import alluxio.grpc.ConfigProperty; -import alluxio.grpc.GetMasterIdPRequest; -import alluxio.grpc.MasterHeartbeatPRequest; -import alluxio.grpc.MetaCommand; -import alluxio.grpc.MetaMasterMasterServiceGrpc; -import alluxio.grpc.RegisterMasterPOptions; -import alluxio.grpc.RegisterMasterPRequest; -import alluxio.grpc.ServiceType; -import alluxio.master.MasterClientContext; -import alluxio.wire.Address; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A wrapper for the gRPC client to interact with the primary meta master, - * used by Alluxio standby masters. - */ -@ThreadSafe -public final class RetryHandlingMetaMasterMasterClient extends AbstractMasterClient { - private static final Logger LOG = - LoggerFactory.getLogger(RetryHandlingMetaMasterMasterClient.class); - private MetaMasterMasterServiceGrpc.MetaMasterMasterServiceBlockingStub mClient = null; - - /** - * Creates a instance of {@link RetryHandlingMetaMasterMasterClient}. - * - * @param conf master client configuration - */ - public RetryHandlingMetaMasterMasterClient(MasterClientContext conf) { - super(conf); - } - - @Override - protected ServiceType getRemoteServiceType() { - return ServiceType.META_MASTER_MASTER_SERVICE; - } - - @Override - protected String getServiceName() { - return Constants.META_MASTER_MASTER_SERVICE_NAME; - } - - @Override - protected long getServiceVersion() { - return Constants.META_MASTER_MASTER_SERVICE_VERSION; - } - - @Override - protected void afterConnect() { - mClient = MetaMasterMasterServiceGrpc.newBlockingStub(mChannel); - } - - /** - * Returns a master id for a master address. - * - * @param address the address to get a master id for - * @return a master id - */ - public long getId(final Address address) throws IOException { - return retryRPC(() -> mClient - .getMasterId(GetMasterIdPRequest.newBuilder().setMasterAddress(address.toProto()).build()) - .getMasterId(), LOG, "GetId", "address=%s", address); - } - - /** - * Sends a heartbeat to the leader master. Standby masters periodically execute this method - * so that the leader master knows they are still running. - * - * @param masterId the master id - * @return whether this master should re-register - */ - public MetaCommand heartbeat(final long masterId) throws IOException { - return retryRPC(() -> mClient - .masterHeartbeat(MasterHeartbeatPRequest.newBuilder().setMasterId(masterId).build()) - .getCommand(), LOG, "Heartbeat", "masterId=%d", masterId); - } - - /** - * Registers with the leader master. - * - * @param masterId the master id of the standby master registering - * @param configList the configuration of this master - */ - public void register(final long masterId, final List configList) - throws IOException { - retryRPC(() -> { - mClient.registerMaster(RegisterMasterPRequest.newBuilder().setMasterId(masterId) - .setOptions(RegisterMasterPOptions.newBuilder().addAllConfigs(configList).build()) - .build()); - return null; - }, LOG, "Register", "masterId=%d,configList=%s", masterId, configList); - } -} diff --git a/core/server/master/src/main/java/alluxio/master/meta/UpdateChecker.java b/core/server/master/src/main/java/alluxio/master/meta/UpdateChecker.java deleted file mode 100644 index d7d75f837014..000000000000 --- a/core/server/master/src/main/java/alluxio/master/meta/UpdateChecker.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.meta; - -import alluxio.ProjectConstants; -import alluxio.check.UpdateCheck; -import alluxio.heartbeat.HeartbeatExecutor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Periodically Alluxio version update check. - */ -@NotThreadSafe -public final class UpdateChecker implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(UpdateChecker.class); - private static final String NUM_WORKER_INFO_FORMAT = "numWorkers:%s"; - - private final MetaMaster mMetaMaster; - - /** - * Creates a new instance of {@link UpdateChecker}. - * - * @param metaMaster the meta master - */ - public UpdateChecker(DefaultMetaMaster metaMaster) { - mMetaMaster = metaMaster; - } - - /** - * Heartbeat for the periodic update check. - */ - @Override - public void heartbeat() { - try { - List additionalInfo = new ArrayList<>(); - int clusterSize = mMetaMaster.getWorkerAddresses().size(); - additionalInfo.add(String.format(NUM_WORKER_INFO_FORMAT, - // TODO(lu) use -1 here since we cannot distinguish - // no worker vs cluster not ready (still registering) cases - clusterSize > 0 ? clusterSize : -1)); - String latestVersion = - UpdateCheck.getLatestVersion(mMetaMaster.getClusterID(), additionalInfo, - 3000, 3000, 3000); - if (!ProjectConstants.VERSION.equals(latestVersion)) { - LOG.info("The latest version (" + latestVersion + ") is not the same " - + "as the current version (" + ProjectConstants.VERSION + "). To upgrade " - + "visit https://www.alluxio.io/download/."); - mMetaMaster.setNewerVersionAvailable(true); - } - } catch (Throwable t) { - LOG.debug("Unable to check for updates:", t); - } - } - - @Override - public void close() {} -} diff --git a/core/server/master/src/main/java/alluxio/master/metastore/rocks/RocksStore.java b/core/server/master/src/main/java/alluxio/master/metastore/rocks/RocksStore.java deleted file mode 100644 index 42b125d0ef6d..000000000000 --- a/core/server/master/src/main/java/alluxio/master/metastore/rocks/RocksStore.java +++ /dev/null @@ -1,347 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.metastore.rocks; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.journal.checkpoint.CheckpointInputStream; -import alluxio.master.journal.checkpoint.CheckpointOutputStream; -import alluxio.master.journal.checkpoint.CheckpointType; -import alluxio.retry.TimeoutRetry; -import alluxio.util.ParallelZipUtils; -import alluxio.util.TarUtils; -import alluxio.util.io.FileUtils; - -import com.google.common.base.Preconditions; -import org.apache.commons.io.IOUtils; -import org.rocksdb.BlockBasedTableConfig; -import org.rocksdb.BloomFilter; -import org.rocksdb.Cache; -import org.rocksdb.Checkpoint; -import org.rocksdb.ColumnFamilyDescriptor; -import org.rocksdb.ColumnFamilyHandle; -import org.rocksdb.DBOptions; -import org.rocksdb.DataBlockIndexType; -import org.rocksdb.Filter; -import org.rocksdb.IndexType; -import org.rocksdb.LRUCache; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Class for managing a rocksdb database. This class handles common functionality such as - * initializing the database and performing database backup/restore. - * - * Thread safety is achieved by synchronizing all public methods. - */ -@ThreadSafe -public final class RocksStore implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(RocksStore.class); - public static final int ROCKS_OPEN_RETRY_TIMEOUT = 20 * Constants.SECOND_MS; - private final String mName; - private final String mDbPath; - private final String mDbCheckpointPath; - private final Integer mParallelBackupPoolSize; - private final Collection mColumnFamilyDescriptors; - private final DBOptions mDbOpts; - - private RocksDB mDb; - private Checkpoint mCheckpoint; - // When we create the database, we must set these handles. - private final List> mColumnHandles; - - /** - * @param name a name to distinguish what store this is - * @param dbPath a path for the rocks database - * @param checkpointPath a path for taking database checkpoints - * @param dbOpts the configured RocksDB options - * @param columnFamilyDescriptors columns to create within the rocks database - * @param columnHandles column handle references to populate - */ - public RocksStore(String name, String dbPath, String checkpointPath, DBOptions dbOpts, - Collection columnFamilyDescriptors, - List> columnHandles) { - Preconditions.checkState(columnFamilyDescriptors.size() == columnHandles.size()); - mName = name; - mDbPath = dbPath; - mDbCheckpointPath = checkpointPath; - mParallelBackupPoolSize = Configuration.getInt( - PropertyKey.MASTER_METASTORE_ROCKS_PARALLEL_BACKUP_THREADS); - mColumnFamilyDescriptors = columnFamilyDescriptors; - mDbOpts = dbOpts; - mColumnHandles = columnHandles; - try { - resetDb(); - } catch (RocksDBException e) { - throw new RuntimeException(e); - } - } - - /** - * @return the underlying rocksdb instance. The instance changes when clear() is called, so if the - * caller caches the returned db, they must reset it after calling clear() - */ - public synchronized RocksDB getDb() { - return mDb; - } - - /** - * Clears and re-initializes the database. - */ - public synchronized void clear() { - try { - resetDb(); - } catch (RocksDBException e) { - throw new RuntimeException(e); - } - LOG.info("Cleared store at {}", mDbPath); - } - - private void resetDb() throws RocksDBException { - stopDb(); - formatDbDirs(); - createDb(); - } - - private void stopDb() { - LOG.info("Closing {} rocks database", mName); - if (mDb != null) { - try { - // Column handles must be closed before closing the db, or an exception gets thrown. - mColumnHandles.forEach(handle -> { - if (handle != null) { - handle.get().close(); - handle.set(null); - } - }); - mDb.close(); - mCheckpoint.close(); - } catch (Throwable t) { - LOG.error("Failed to close rocks database", t); - } - mDb = null; - mCheckpoint = null; - } - } - - private void formatDbDirs() { - try { - FileUtils.deletePathRecursively(mDbPath); - FileUtils.deletePathRecursively(mDbCheckpointPath); - } catch (IOException e) { - throw new RuntimeException(e); - } - new File(mDbPath).mkdirs(); - } - - private void createDb() throws RocksDBException { - List cfDescriptors = new ArrayList<>(); - cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); - cfDescriptors.addAll(mColumnFamilyDescriptors); - // a list which will hold the handles for the column families once the db is opened - List columns = new ArrayList<>(); - final TimeoutRetry retryPolicy = new TimeoutRetry(ROCKS_OPEN_RETRY_TIMEOUT, 100); - RocksDBException lastException = null; - while (retryPolicy.attempt()) { - try { - mDb = RocksDB.open(mDbOpts, mDbPath, cfDescriptors, columns); - break; - } catch (RocksDBException e) { - // sometimes the previous terminated process's lock may not have been fully cleared yet - // retry until timeout to make sure that isn't the case - lastException = e; - } - } - if (mDb == null && lastException != null) { - throw lastException; - } - mCheckpoint = Checkpoint.create(mDb); - for (int i = 0; i < columns.size() - 1; i++) { - // Skip the default column. - mColumnHandles.get(i).set(columns.get(i + 1)); - } - LOG.info("Opened rocks database under path {}", mDbPath); - } - - /** - * Writes a checkpoint of the database's content to the given output stream. - * - * @param output the stream to write to - */ - public synchronized void writeToCheckpoint(OutputStream output) - throws IOException, InterruptedException { - LOG.info("Creating rocksdb checkpoint at {}", mDbCheckpointPath); - long startNano = System.nanoTime(); - - try { - // createCheckpoint requires that the directory not already exist. - FileUtils.deletePathRecursively(mDbCheckpointPath); - mCheckpoint.createCheckpoint(mDbCheckpointPath); - } catch (RocksDBException e) { - throw new IOException(e); - } - - if (Configuration.getBoolean(PropertyKey.MASTER_METASTORE_ROCKS_PARALLEL_BACKUP)) { - CheckpointOutputStream out = new CheckpointOutputStream(output, - CheckpointType.ROCKS_PARALLEL); - LOG.info("Checkpoint complete, compressing with {} threads", mParallelBackupPoolSize); - int compressLevel = Configuration.getInt( - PropertyKey.MASTER_METASTORE_ROCKS_PARALLEL_BACKUP_COMPRESSION_LEVEL); - ParallelZipUtils.compress(Paths.get(mDbCheckpointPath), out, - mParallelBackupPoolSize, compressLevel); - } else { - CheckpointOutputStream out = new CheckpointOutputStream(output, CheckpointType.ROCKS_SINGLE); - LOG.info("Checkpoint complete, compressing with one thread"); - TarUtils.writeTarGz(Paths.get(mDbCheckpointPath), out); - } - - LOG.info("Completed rocksdb checkpoint in {}ms", (System.nanoTime() - startNano) / 1_000_000); - // Checkpoint is no longer needed, delete to save space. - FileUtils.deletePathRecursively(mDbCheckpointPath); - } - - /** - * Restores the database from a checkpoint. - * - * @param input the checkpoint stream to restore from - */ - public synchronized void restoreFromCheckpoint(CheckpointInputStream input) throws IOException { - LOG.info("Restoring rocksdb from checkpoint"); - long startNano = System.nanoTime(); - Preconditions.checkState(input.getType() == CheckpointType.ROCKS_SINGLE - || input.getType() == CheckpointType.ROCKS_PARALLEL, - "Unexpected checkpoint type in RocksStore: " + input.getType()); - stopDb(); - FileUtils.deletePathRecursively(mDbPath); - - if (input.getType() == CheckpointType.ROCKS_PARALLEL) { - List tmpDirs = Configuration.getList(PropertyKey.TMP_DIRS); - String tmpZipFilePath = new File(tmpDirs.get(0), "alluxioRockStore-" + UUID.randomUUID()) - .getPath(); - - try { - try (FileOutputStream fos = new FileOutputStream(tmpZipFilePath)) { - IOUtils.copy(input, fos); - } - - ParallelZipUtils.decompress(Paths.get(mDbPath), tmpZipFilePath, - mParallelBackupPoolSize); - - FileUtils.deletePathRecursively(tmpZipFilePath); - } catch (Exception e) { - LOG.warn("Failed to decompress checkpoint from {} to {}", tmpZipFilePath, mDbPath); - throw e; - } - } else { - TarUtils.readTarGz(Paths.get(mDbPath), input); - } - - try { - createDb(); - } catch (RocksDBException e) { - throw new IOException(e); - } - LOG.info("Restored rocksdb checkpoint in {}ms", - (System.nanoTime() - startNano) / Constants.MS_NANO); - } - - @Override - public synchronized void close() { - stopDb(); - LOG.info("Closed store at {}", mDbPath); - } - - // helper function to load RockDB configuration options based on property key configurations. - static Optional checkSetTableConfig( - PropertyKey cacheSize, PropertyKey bloomFilter, PropertyKey indexType, - PropertyKey blockIndexType, List toClose) { - // The following options are set by property keys as they are not able to be - // set using configuration files. - BlockBasedTableConfig blockConfig = new BlockBasedTableConfig(); - boolean shoudSetConfig = false; - if (Configuration.isSet(cacheSize)) { - shoudSetConfig = true; - // Set the inodes column options - Cache inodeCache = new LRUCache(Configuration.getInt(cacheSize)); - toClose.add(inodeCache); - blockConfig.setBlockCache(inodeCache); - } - if (Configuration.getBoolean(bloomFilter)) { - shoudSetConfig = true; - Filter filter = new BloomFilter(); - toClose.add(filter); - blockConfig.setFilterPolicy(filter); - } - if (Configuration.isSet(indexType)) { - shoudSetConfig = true; - blockConfig.setIndexType(toRocksIndexType(Configuration.getEnum( - indexType, alluxio.master.metastore.rocks.IndexType.class))); - } - if (Configuration.isSet(blockIndexType)) { - shoudSetConfig = true; - blockConfig.setDataBlockIndexType(toRocksDataBlockIndexType(Configuration.getEnum( - blockIndexType, alluxio.master.metastore.rocks.DataBlockIndexType.class))); - } - if (shoudSetConfig) { - return Optional.of(blockConfig); - } - return Optional.empty(); - } - - // helper function to convert alluxio enum to rocksDb enum - private static DataBlockIndexType toRocksDataBlockIndexType( - alluxio.master.metastore.rocks.DataBlockIndexType index) { - switch (index) { - case kDataBlockBinarySearch: - return DataBlockIndexType.kDataBlockBinarySearch; - case kDataBlockBinaryAndHash: - return DataBlockIndexType.kDataBlockBinaryAndHash; - default: - throw new IllegalArgumentException(String.format("Unknown DataBlockIndexType %s", index)); - } - } - - // helper function to convert alluxio enum to rocksDb enum - private static IndexType toRocksIndexType( - alluxio.master.metastore.rocks.IndexType index) { - switch (index) { - case kBinarySearch: - return IndexType.kBinarySearch; - case kHashSearch: - return IndexType.kHashSearch; - case kBinarySearchWithFirstKey: - return IndexType.kBinarySearchWithFirstKey; - case kTwoLevelIndexSearch: - return IndexType.kTwoLevelIndexSearch; - default: - throw new IllegalArgumentException(String.format("Unknown IndexType %s", index)); - } - } -} diff --git a/core/server/master/src/main/java/alluxio/master/throttle/DefaultThrottleMaster.java b/core/server/master/src/main/java/alluxio/master/throttle/DefaultThrottleMaster.java deleted file mode 100644 index 0c7385373f4a..000000000000 --- a/core/server/master/src/main/java/alluxio/master/throttle/DefaultThrottleMaster.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.throttle; - -import alluxio.Constants; -import alluxio.Server; -import alluxio.annotation.SuppressFBWarnings; -import alluxio.clock.SystemClock; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.GrpcService; -import alluxio.grpc.ServiceType; -import alluxio.heartbeat.HeartbeatContext; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.heartbeat.HeartbeatThread; -import alluxio.master.AbstractMaster; -import alluxio.master.MasterContext; -import alluxio.master.MasterProcess; -import alluxio.master.MasterRegistry; -import alluxio.master.block.BlockMaster; -import alluxio.master.file.FileSystemMaster; -import alluxio.master.journal.NoopJournaled; -import alluxio.master.metrics.MetricsMaster; -import alluxio.util.executor.ExecutorServiceFactories; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Future; -import javax.annotation.concurrent.ThreadSafe; - -/** - * This service periodically monitors the system and throttles in case of busy. - */ -@ThreadSafe -public final class DefaultThrottleMaster extends AbstractMaster implements NoopJournaled { - private static final Logger LOG = LoggerFactory.getLogger(DefaultThrottleMaster.class); - private static final Set> DEPS = - ImmutableSet.of(BlockMaster.class, FileSystemMaster.class, - MetricsMaster.class); - - /** The Alluxio master process. */ - private MasterProcess mMasterProcess; - - private ThrottleExecutor mThrottleExecutor; - - /** - * The service that performs license check. - */ - @SuppressFBWarnings("URF_UNREAD_FIELD") - private Future mThrottleService; - - /** - * Creates a new instance of {@link DefaultThrottleMaster}. - * - * @param registry the master registry - * @param masterContext the context for Alluxio master - */ - public DefaultThrottleMaster(MasterRegistry registry, MasterContext masterContext) { - super(masterContext, new SystemClock(), ExecutorServiceFactories - .cachedThreadPool(Constants.THROTTLE_MASTER_NAME)); - registry.add(DefaultThrottleMaster.class, this); - } - - /** - * Sets the master to be used in {@link ThrottleExecutor} for throttling. - * This should be called before calling {@link #start(Boolean)}. - * - * @param masterProcess the Alluxio master process - */ - public void setMaster(MasterProcess masterProcess) { - mMasterProcess = masterProcess; - mThrottleExecutor = new ThrottleExecutor(mMasterProcess); - } - - @Override - public Set> getDependencies() { - return DEPS; - } - - @Override - public String getName() { - return Constants.THROTTLE_MASTER_NAME; - } - - @Override - public void start(Boolean isLeader) throws IOException { - super.start(isLeader); - Preconditions.checkNotNull(mMasterProcess, "Alluxio master process is not specified"); - Preconditions.checkNotNull(mThrottleExecutor, "ThrottleExecutor is not specified"); - if (!isLeader) { - return; - } - LOG.info("Starting {}", getName()); - mThrottleService = getExecutorService().submit( - new HeartbeatThread(HeartbeatContext.MASTER_THROTTLE, mThrottleExecutor, - Configuration.getMs(PropertyKey.MASTER_THROTTLE_HEARTBEAT_INTERVAL), - Configuration.global(), - mMasterContext.getUserState())); - LOG.info("{} is started", getName()); - } - - @Override - public Map getServices() { - return new HashMap<>(); - } - - /** - * Collects and saves call home information during the heartbeat. - */ - @ThreadSafe - public static final class ThrottleExecutor implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(ThrottleExecutor.class); - - private MasterProcess mMasterProcess; - private SystemMonitor mSystemMonitor; - - /** - * Creates a new instance of {@link ThrottleExecutor}. - * - * @param masterProcess the Alluxio master process - */ - public ThrottleExecutor(MasterProcess masterProcess) { - mMasterProcess = masterProcess; - mSystemMonitor = new SystemMonitor(mMasterProcess); - } - - @Override - public void heartbeat() throws InterruptedException { - mSystemMonitor.run(); - } - - @Override - public void close() { - // Nothing to close. - } - } -} - diff --git a/core/server/master/src/main/java/alluxio/master/throttle/FileSystemIndicator.java b/core/server/master/src/main/java/alluxio/master/throttle/FileSystemIndicator.java deleted file mode 100644 index adeb825a8c66..000000000000 --- a/core/server/master/src/main/java/alluxio/master/throttle/FileSystemIndicator.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.throttle; - -import alluxio.metrics.MetricsSystem; - -import com.google.common.base.MoreObjects; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * The filesystem indicator object. - */ -public class FileSystemIndicator { - // The observed counter names - private static final List OBSERVED_MASTER_COUNTER = new LinkedList<>(); - - static { - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_COMPLETE_FILE_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_COMPLETED_OPERATION_RETRY_COUNT); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_CREATE_FILE_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_CREATE_DIRECTORIES_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_DELETE_PATH_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_DIRECTORIES_CREATED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_FILE_BLOCK_INFOS_GOT); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_FILE_INFOS_GOT); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_FILES_COMPLETED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_FILES_CREATED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_FILES_FREED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_FILES_PERSISTED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_FREE_FILE_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_GET_FILE_BLOCK_INFO_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_GET_FILE_INFO_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_GET_NEW_BLOCK_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_LISTING_CACHE_EVICTIONS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_LISTING_CACHE_HITS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_LISTING_CACHE_LOAD_TIMES); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_LISTING_CACHE_MISSES); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_ACTIVE_PATHS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_FAIL); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_NO_CHANGE); - - // This is the concerns - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_OPS_COUNT); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PATHS_CANCEL); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PATHS_FAIL); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PATHS_SUCCESS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PENDING_PATHS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PREFETCH_CANCEL); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PREFETCH_FAIL); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PREFETCH_OPS_COUNT); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PREFETCH_PATHS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PREFETCH_RETRIES); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_PREFETCH_SUCCESS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_SKIPPED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_SUCCESS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_METADATA_SYNC_TIME_MS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_MOUNT_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_NEW_BLOCKS_GOT); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_PATHS_DELETED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_PATHS_MOUNTED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_PATHS_RENAMED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_PATHS_UNMOUNTED); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_RENAME_PATH_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_SET_ACL_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_SET_ATTRIBUTE_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_UFS_STATUS_CACHE_CHILDREN_SIZE); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_UFS_STATUS_CACHE_SIZE); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_UNMOUNT_OPS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_GET_CONFIG_HASH_IN_PROGRESS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_GET_CONFIGURATION_IN_PROGRESS); - OBSERVED_MASTER_COUNTER - .add(MetricsMonitorUtils.FileSystemCounterName.MASTER_REGISTER_WORKER_START_IN_PROGRESS); - } - - private Map mMasterIndicators; - private long mPitTimeMS; - - /** - * Filesystem indicator Constructor. - */ - public FileSystemIndicator() { - mMasterIndicators = new HashMap<>(); - for (String countName : OBSERVED_MASTER_COUNTER) { - mMasterIndicators.put(countName, MetricsSystem.METRIC_REGISTRY - .counter(countName).getCount()); - } - mPitTimeMS = System.currentTimeMillis(); - } - - /** - * Filesystem indicator constructor. - * - * @param fileSystemIndicator the filesystem indicator - */ - public FileSystemIndicator(FileSystemIndicator fileSystemIndicator) { - mMasterIndicators = new HashMap<>(); - mMasterIndicators.putAll(fileSystemIndicator.mMasterIndicators); - mPitTimeMS = fileSystemIndicator.mPitTimeMS; - } - - /** - * @param name the counter name - * @return the counter - */ - public Optional getCounter(String name) { - if (mMasterIndicators.containsKey(name)) { - return Optional.of(mMasterIndicators.get(name)); - } - return Optional.empty(); - } - - /** - * @return the point in time - */ - public long getPitTimeMS() { - return mPitTimeMS; - } - - /** - * Sets the counter. - * - * @param name the counter name - * @param value the counter value - */ - public void setCounter(String name, long value) { - mMasterIndicators.put(name, value); - } - - /** - * Sets the pit time. - * - * @param pitTimeMS the pit - */ - public void setPitTimeMS(long pitTimeMS) { - mPitTimeMS = pitTimeMS; - } - - /** - * Calculates the delta. - * - * @param baselineIndicators the baseline indicator object - */ - public void deltaTo(FileSystemIndicator baselineIndicators) { - for (String name : OBSERVED_MASTER_COUNTER) { - mMasterIndicators.computeIfPresent(name, - (key, value) - -> { return value - baselineIndicators.mMasterIndicators.getOrDefault(key, 0L); }); - } - } - - /** - * @return the string of the indicators - */ - public String toString() { - MoreObjects.ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); - for (Map.Entry entry : mMasterIndicators.entrySet()) { - toStringHelper.add((String) (entry.getKey()), entry.getValue()); - } - return toStringHelper.toString(); - } -} - diff --git a/core/server/master/src/main/java/alluxio/master/throttle/MetricsMonitorUtils.java b/core/server/master/src/main/java/alluxio/master/throttle/MetricsMonitorUtils.java deleted file mode 100644 index ecb1cf77cf66..000000000000 --- a/core/server/master/src/main/java/alluxio/master/throttle/MetricsMonitorUtils.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.throttle; - -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; - -/** - * The Metrics monitor utils. - */ -public class MetricsMonitorUtils { - /** - * System level gauges. - */ - public static class MemoryGaugeName { - public static final String TOTAL_COMMITTED = "total.committed"; - public static final String TOTAL_INIT = "total.init"; - public static final String TOTAL_MAX = "total.max"; - public static final String TOTAL_USED = "total.used"; - - public static final String NON_HEAP_COMMITTED = "non-heap.committed"; - public static final String NON_HEAP_INIT = "non-heap.init"; - public static final String NON_HEAP_MAX = "non-heap.max"; - public static final String NON_HEAP_USAGE = "non-heap.usage"; - public static final String NON_HEAP_USED = "non-heap.used"; - - public static final String HEAP_COMMITTED = "heap.committed"; - public static final String HEAP_INIT = "heap.init"; - public static final String HEAP_MAX = "heap.max"; - public static final String HEAP_USAGE = "heap.usage"; - public static final String HEAP_USED = "heap.used"; - - public static final String DIRECT_MEM_USED = MetricKey.PROCESS_POOL_DIRECT_MEM_USED.getName(); - } - - /** - * OS gauge names. - */ - public static class OSGaugeName { - public static final String OS_CPU_LOAD = "os.cpuLoad"; - public static final String OS_FREE_PHYSICAL_MEMORY = "os.freePhysicalMemory"; - public static final String OS_MAX_FILE_COUNT = "os.maxFileCount"; - public static final String OS_OPEN_FILE_COUNT = "os.openFileCount"; - public static final String OS_TOTAL_PHYSICAL_MEMORY = "os.totalPhysicalMemory"; - } - - /** - * Server prefix. - */ - public static class ServerPrefix { - public static final String MASTER = "Master."; - public static final String WORKER = "Worker."; - } - - /** - * Server gauge names. - */ - public static class ServerGaugeName { - public static final String RPC_QUEUE_LENGTH = MetricsSystem.getMetricName("RpcQueueLength"); - - // JVM total pause time - public static final String TOTAL_EXTRA_TIME - = MetricsSystem.getMetricName(MetricKey.TOTAL_EXTRA_TIME.getName()); - } - - /** - * Filesystem gauge name. - */ - public static class FileSystemGaugeName { - public static final String MASTER_EDGE_CACHE_EVICTIONS - = MetricKey.MASTER_EDGE_CACHE_EVICTIONS.getName(); - public static final String MASTER_EDGE_CACHE_HITS - = MetricKey.MASTER_EDGE_CACHE_HITS.getName(); - public static final String MASTER_EDGE_CACHE_LOAD_TIMES - = MetricKey.MASTER_EDGE_CACHE_LOAD_TIMES.getName(); - public static final String MASTER_EDGE_CACHE_MISSES - = MetricKey.MASTER_EDGE_CACHE_MISSES.getName(); - public static final String MASTER_INODE_CACHE_EVICTIONS - = MetricKey.MASTER_INODE_CACHE_EVICTIONS.getName(); - public static final String MASTER_INODE_CACHE_HIT_RATIO - = MetricKey.MASTER_INODE_CACHE_HIT_RATIO.getName(); - public static final String MASTER_INODE_CACHE_LOAD_TIMES - = MetricKey.MASTER_INODE_CACHE_LOAD_TIMES.getName(); - public static final String MASTER_INODE_CACHE_MISSES - = MetricKey.MASTER_INODE_CACHE_MISSES.getName(); - } - - /** - * Filesystem counter names. - */ - public static class FileSystemCounterName { - public static final String MASTER_COMPLETE_FILE_OPS - = MetricKey.MASTER_COMPLETE_FILE_OPS.getName(); - public static final String MASTER_COMPLETED_OPERATION_RETRY_COUNT - = MetricKey.MASTER_COMPLETED_OPERATION_RETRY_COUNT.getName(); - public static final String MASTER_CREATE_DIRECTORIES_OPS - = MetricKey.MASTER_CREATE_DIRECTORIES_OPS.getName(); - public static final String MASTER_CREATE_FILE_OPS - = MetricKey.MASTER_CREATE_FILES_OPS.getName(); - public static final String MASTER_DELETE_PATH_OPS - = MetricKey.MASTER_DELETE_PATHS_OPS.getName(); - public static final String MASTER_DIRECTORIES_CREATED - = MetricKey.MASTER_DIRECTORIES_CREATED.getName(); - public static final String MASTER_FILE_BLOCK_INFOS_GOT - = MetricKey.MASTER_FILE_BLOCK_INFOS_GOT.getName(); - public static final String MASTER_FILE_INFOS_GOT - = MetricKey.MASTER_FILE_INFOS_GOT.getName(); - public static final String MASTER_FILES_COMPLETED - = MetricKey.MASTER_FILES_COMPLETED.getName(); - public static final String MASTER_FILES_CREATED - = MetricKey.MASTER_FILES_CREATED.getName(); - public static final String MASTER_FILES_FREED - = MetricKey.MASTER_FILES_FREED.getName(); - public static final String MASTER_FILES_PERSISTED - = MetricKey.MASTER_FILES_PERSISTED.getName(); - public static final String MASTER_FREE_FILE_OPS - = MetricKey.MASTER_FREE_FILE_OPS.getName(); - public static final String MASTER_GET_FILE_BLOCK_INFO_OPS - = MetricKey.MASTER_GET_FILE_BLOCK_INFO_OPS.getName(); - public static final String MASTER_GET_FILE_INFO_OPS - = MetricKey.MASTER_GET_FILE_INFO_OPS.getName(); - public static final String MASTER_GET_NEW_BLOCK_OPS - = MetricKey.MASTER_GET_NEW_BLOCK_OPS.getName(); - public static final String MASTER_LISTING_CACHE_EVICTIONS - = MetricKey.MASTER_LISTING_CACHE_EVICTIONS.getName(); - public static final String MASTER_LISTING_CACHE_HITS - = MetricKey.MASTER_LISTING_CACHE_HITS.getName(); - public static final String MASTER_LISTING_CACHE_LOAD_TIMES - = MetricKey.MASTER_LISTING_CACHE_LOAD_TIMES.getName(); - public static final String MASTER_LISTING_CACHE_MISSES - = MetricKey.MASTER_LISTING_CACHE_MISSES.getName(); - public static final String MASTER_METADATA_SYNC_ACTIVE_PATHS - = MetricKey.MASTER_METADATA_SYNC_ACTIVE_PATHS.getName(); - public static final String MASTER_METADATA_SYNC_FAIL - = MetricKey.MASTER_METADATA_SYNC_FAIL.getName(); - public static final String MASTER_METADATA_SYNC_NO_CHANGE - = MetricKey.MASTER_METADATA_SYNC_NO_CHANGE.getName(); - - // This is the concerns - public static final String MASTER_METADATA_SYNC_OPS_COUNT - = MetricKey.MASTER_METADATA_SYNC_OPS_COUNT.getName(); - public static final String MASTER_METADATA_SYNC_PATHS_CANCEL - = MetricKey.MASTER_METADATA_SYNC_PATHS_CANCEL.getName(); - public static final String MASTER_METADATA_SYNC_PATHS_FAIL - = MetricKey.MASTER_METADATA_SYNC_PATHS_FAIL.getName(); - public static final String MASTER_METADATA_SYNC_PATHS_SUCCESS - = MetricKey.MASTER_METADATA_SYNC_PATHS_SUCCESS.getName(); - public static final String MASTER_METADATA_SYNC_PENDING_PATHS - = MetricKey.MASTER_METADATA_SYNC_PENDING_PATHS.getName(); - public static final String MASTER_METADATA_SYNC_PREFETCH_CANCEL - = MetricKey.MASTER_METADATA_SYNC_PREFETCH_CANCEL.getName(); - public static final String MASTER_METADATA_SYNC_PREFETCH_FAIL - = MetricKey.MASTER_METADATA_SYNC_PREFETCH_FAIL.getName(); - public static final String MASTER_METADATA_SYNC_PREFETCH_OPS_COUNT - = MetricKey.MASTER_METADATA_SYNC_PREFETCH_OPS_COUNT.getName(); - public static final String MASTER_METADATA_SYNC_PREFETCH_PATHS - = MetricKey.MASTER_METADATA_SYNC_PREFETCH_PATHS.getName(); - public static final String MASTER_METADATA_SYNC_PREFETCH_RETRIES - = MetricKey.MASTER_METADATA_SYNC_PREFETCH_RETRIES.getName(); - public static final String MASTER_METADATA_SYNC_PREFETCH_SUCCESS - = MetricKey.MASTER_METADATA_SYNC_PREFETCH_SUCCESS.getName(); - public static final String MASTER_METADATA_SYNC_SKIPPED - = MetricKey.MASTER_METADATA_SYNC_SKIPPED.getName(); - public static final String MASTER_METADATA_SYNC_SUCCESS - = MetricKey.MASTER_METADATA_SYNC_SUCCESS.getName(); - public static final String MASTER_METADATA_SYNC_TIME_MS - = MetricKey.MASTER_METADATA_SYNC_TIME_MS.getName(); - public static final String MASTER_MOUNT_OPS - = MetricKey.MASTER_MOUNT_OPS.getName(); - public static final String MASTER_NEW_BLOCKS_GOT - = MetricKey.MASTER_NEW_BLOCKS_GOT.getName(); - public static final String MASTER_PATHS_DELETED - = MetricKey.MASTER_PATHS_DELETED.getName(); - public static final String MASTER_PATHS_MOUNTED - = MetricKey.MASTER_PATHS_MOUNTED.getName(); - public static final String MASTER_PATHS_RENAMED - = MetricKey.MASTER_PATHS_RENAMED.getName(); - public static final String MASTER_PATHS_UNMOUNTED - = MetricKey.MASTER_PATHS_UNMOUNTED.getName(); - public static final String MASTER_RENAME_PATH_OPS - = MetricKey.MASTER_RENAME_PATH_OPS.getName(); - public static final String MASTER_SET_ACL_OPS - = MetricKey.MASTER_SET_ACL_OPS.getName(); - public static final String MASTER_SET_ATTRIBUTE_OPS - = MetricKey.MASTER_SET_ATTRIBUTE_OPS.getName(); - public static final String MASTER_UFS_STATUS_CACHE_CHILDREN_SIZE - = MetricKey.MASTER_UFS_STATUS_CACHE_CHILDREN_SIZE.getName(); - public static final String MASTER_UFS_STATUS_CACHE_SIZE - = MetricKey.MASTER_UFS_STATUS_CACHE_SIZE.getName(); - public static final String MASTER_UNMOUNT_OPS - = MetricKey.MASTER_UNMOUNT_OPS.getName(); - public static final String MASTER_GET_CONFIG_HASH_IN_PROGRESS - = "Master.getConfigHashInProgress"; - public static final String MASTER_GET_CONFIGURATION_IN_PROGRESS - = "Master.getConfigurationInProgress"; - public static final String MASTER_REGISTER_WORKER_START_IN_PROGRESS - = "Master.registerWorkerStartInProgress"; - } -} - diff --git a/core/server/master/src/main/java/alluxio/master/throttle/ServerIndicator.java b/core/server/master/src/main/java/alluxio/master/throttle/ServerIndicator.java deleted file mode 100644 index 576d35a6aaa8..000000000000 --- a/core/server/master/src/main/java/alluxio/master/throttle/ServerIndicator.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.throttle; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.metrics.MetricsSystem; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The server indicators. - * It can be used as a point-in-time indicator, an aggregated indicator, a single threshold - * indicator or aggregated threshold. - * For several PIT indicators can be aggregated, the addition function can be used - * for that. - * For one indicator can also be reduced, the reduction function can be used. - * One example is the sliding window: - * aggregated indicator1 = ( pit1 + pit2 + pit3) - * aggregated indicator2 = (aggregated indicator1 + pit4 - pit1) - */ -public class ServerIndicator { - private static final Logger LOG = LoggerFactory.getLogger(ServerIndicator.class); - private long mHeapMax; - private long mHeapUsed; - private long mDirectMemUsed; - private double mCpuLoad; - - private long mTotalJVMPauseTimeMS; - private long mPITTotalJVMPauseTimeMS; - private long mRpcQueueSize; - - // The time the point-in-time indicator generated - // It is not useful if it is an aggregated one - private long mPitTimeMS; - - /** - * @param directMemUsed the allocated direct memory - * @param heapMax the max heap - * @param heapUsed the used heap - * @param cpuLoad the cpu load - * @param totalJVMPauseTimeMS the total JVM pause time since previous PIT - * @param pitTotalJVMPauseTimeMS the pit total JVM pause time - * @param rpcQueueSize the pit rpc queue size - * @param snapshotTimeMS the pit time - */ - public ServerIndicator(long directMemUsed, long heapMax, long heapUsed, double cpuLoad, - long totalJVMPauseTimeMS, long pitTotalJVMPauseTimeMS, long rpcQueueSize, - long snapshotTimeMS) { - mDirectMemUsed = directMemUsed; - mHeapMax = heapMax; - mHeapUsed = heapUsed; - mCpuLoad = cpuLoad; - mTotalJVMPauseTimeMS = totalJVMPauseTimeMS; - mPITTotalJVMPauseTimeMS = pitTotalJVMPauseTimeMS; - mRpcQueueSize = rpcQueueSize; - mPitTimeMS = snapshotTimeMS; - } - - /** - * @param serverIndicator the server indicator - */ - public ServerIndicator(ServerIndicator serverIndicator) { - this(serverIndicator, 1); - } - - /** - * The scaled server indicator according to the multiple. - * The threshold is set for single point in time value, and sometimes the sliding window is used - * to check if the threshold is crossed, in that case multiple times of single pit threshold is - * required. Eg, if the sliding window side is 3, (pit1, pit2, pit3), (pit2, pit3, pit4) ... are - * calculated. The threshold value * 3 would be used to check the value of sliding window. This - * constructor is helping generate the threshold of sliding window. - * - * @param serverIndicator the base server indicator - * @param multiple the times - */ - public ServerIndicator(ServerIndicator serverIndicator, int multiple) { - Preconditions.checkNotNull(serverIndicator, "serverIndicator"); - mDirectMemUsed = serverIndicator.mDirectMemUsed * multiple; - mHeapMax = serverIndicator.mHeapMax; - mHeapUsed = serverIndicator.mHeapUsed * multiple; - mCpuLoad = serverIndicator.mCpuLoad * multiple; - mTotalJVMPauseTimeMS = serverIndicator.mTotalJVMPauseTimeMS * multiple; - mPITTotalJVMPauseTimeMS = serverIndicator.mPITTotalJVMPauseTimeMS; - mRpcQueueSize = serverIndicator.mRpcQueueSize * multiple; - mPitTimeMS = serverIndicator.mPitTimeMS; - } - - /** - * @return current total JVM pause time - */ - public static long getSystemTotalJVMPauseTime() { - if (Configuration.getBoolean(PropertyKey.MASTER_JVM_MONITOR_ENABLED)) { - return (long) getMetrics(MetricsMonitorUtils.ServerGaugeName.TOTAL_EXTRA_TIME, 0L); - } - return 0; - } - - /** - * Creates a server indicator based on current status. - * - * @param prevTotalJVMPauseTime the previous total JVM pause time - * @return the server indicator - */ - public static ServerIndicator createFromMetrics(long prevTotalJVMPauseTime) { - long pitTotalJVMPauseTime = 0; - long totalJVMPauseTime = 0; - if (Configuration.getBoolean(PropertyKey.MASTER_JVM_MONITOR_ENABLED)) { - pitTotalJVMPauseTime = (long) getMetrics(MetricsMonitorUtils.ServerGaugeName.TOTAL_EXTRA_TIME, - 0L); - totalJVMPauseTime = pitTotalJVMPauseTime - prevTotalJVMPauseTime; - } - - long rpcQueueSize = getMetrics(MetricsMonitorUtils.ServerGaugeName.RPC_QUEUE_LENGTH, 0L); - - return new ServerIndicator((long) getMetrics( - MetricsSystem.getMetricName(MetricsMonitorUtils.MemoryGaugeName.DIRECT_MEM_USED), 0L), - (long) getMetrics(MetricsMonitorUtils.MemoryGaugeName.HEAP_MAX, 0L), - (long) getMetrics(MetricsMonitorUtils.MemoryGaugeName.HEAP_USED, 0L), - (double) getMetrics(MetricsMonitorUtils.OSGaugeName.OS_CPU_LOAD, 0d), - totalJVMPauseTime, pitTotalJVMPauseTime, rpcQueueSize, System.currentTimeMillis()); - } - - private static T getMetrics(String name, T value) { - try { - return (T) (MetricsSystem.METRIC_REGISTRY - .gauge(name, null).getValue()); - } catch (Exception e) { - return value; - } - } - - /** - * Creates a threshold indicator object, the parameters are set according the input values. - * - * @param directMemUsed the used direct mem - * @param ratioOfUsedHeap the ratio of used heap - * @param cpuLoad the cpu load - * @param totalJVMPauseTimeMS the total JVM pause time - * @param rpcQueueSize the rpc queue size - * @return the threshold indicator object - */ - public static ServerIndicator createThresholdIndicator(long directMemUsed, - double ratioOfUsedHeap, double cpuLoad, long totalJVMPauseTimeMS, long rpcQueueSize) { - long pitTotalJVMPauseTimeMS = 0; - long heapMax = (long) getMetrics(MetricsMonitorUtils.MemoryGaugeName.HEAP_MAX, 0L); - long heapUsed = (long) (ratioOfUsedHeap * heapMax); - - if (Configuration.getBoolean(PropertyKey.MASTER_JVM_MONITOR_ENABLED)) { - pitTotalJVMPauseTimeMS - = (long) getMetrics(MetricsMonitorUtils.ServerGaugeName.TOTAL_EXTRA_TIME, 0L); - } - return new ServerIndicator(directMemUsed, heapMax, - heapUsed, cpuLoad, totalJVMPauseTimeMS, pitTotalJVMPauseTimeMS, - rpcQueueSize, System.currentTimeMillis()); - } - - /** - * @return the direct mem used - */ - public long getDirectMemUsed() { - return mDirectMemUsed; - } - - /** - * @return the max heap - */ - public long getHeapMax() { - return mHeapMax; - } - - /** - * @return the used heap - */ - public long getHeapUsed() { - return mHeapUsed; - } - - /** - * @return the cpu load - */ - public double getCpuLoad() { - return mCpuLoad; - } - - /** - * @return the total JVM pause time - */ - public long getTotalJVMPauseTimeMS() { - return mTotalJVMPauseTimeMS; - } - - /** - * @return the PIT total JVM pause time - */ - public long getPITTotalJVMPauseTimeMS() { - return mPITTotalJVMPauseTimeMS; - } - - /** - * @return the rpc queue size - */ - public long getRpcQueueSize() { - return mRpcQueueSize; - } - - /** - * @return the time to measure the indicators - */ - public long getPitTimeMS() { - return mPitTimeMS; - } - - /** - * Sets the pit time. - * - * @param pitTimeMS the pit time - */ - public void setPitTimeMS(long pitTimeMS) { - mPitTimeMS = pitTimeMS; - } - - /** - * Addition the server indicator object. - * - * @param serverIndicator the server indicator - */ - public void addition(ServerIndicator serverIndicator) { - // This is to record the aggregated time - mDirectMemUsed += serverIndicator.getDirectMemUsed(); - mRpcQueueSize += serverIndicator.getRpcQueueSize(); - mCpuLoad += serverIndicator.getCpuLoad(); - mHeapUsed += serverIndicator.getHeapUsed(); - mTotalJVMPauseTimeMS += serverIndicator.getTotalJVMPauseTimeMS(); - } - - /** - * Gets the server indicator after reducing the serverIndicator. - * It can be used in sliding window calculation, eg: - * Aggregated1 = indicator1 + indicator2 + indicator3 - * Aggregated2 = Aggregated1 - indicator1 + indicator4 - * - * @param serverIndicator the base server indicator - */ - public void reduction(ServerIndicator serverIndicator) { - mDirectMemUsed = mDirectMemUsed > serverIndicator.mDirectMemUsed - ? mDirectMemUsed - serverIndicator.mDirectMemUsed : 0; - mRpcQueueSize = mRpcQueueSize > serverIndicator.mRpcQueueSize - ? mRpcQueueSize - serverIndicator.mRpcQueueSize : 0; - mCpuLoad = Double.compare(mCpuLoad, serverIndicator.mCpuLoad) > 0 - ? mCpuLoad - serverIndicator.mCpuLoad : 0; - mHeapUsed = mHeapUsed > serverIndicator.mHeapUsed - ? mHeapUsed - serverIndicator.mHeapUsed : 0; - mTotalJVMPauseTimeMS = mTotalJVMPauseTimeMS > serverIndicator.mTotalJVMPauseTimeMS - ? mTotalJVMPauseTimeMS - serverIndicator.mTotalJVMPauseTimeMS : 0; - } - - /** - * @return the string object - */ - public String toString() { - return MoreObjects.toStringHelper(this) - .add("directMemUsed", mDirectMemUsed) - .add("heapMax", mHeapMax) - .add("heapUsed", mHeapUsed) - .add("cpuLoad", mCpuLoad) - .add("pitTotalJVMPauseTimeMS", mPITTotalJVMPauseTimeMS) - .add("totalJVMPauseTimeMS", mTotalJVMPauseTimeMS) - .add("rpcQueueSize", mRpcQueueSize) - .add("pitTimeMS", mPitTimeMS) - .toString(); - } -} diff --git a/core/server/master/src/main/java/alluxio/master/throttle/SystemMonitor.java b/core/server/master/src/main/java/alluxio/master/throttle/SystemMonitor.java deleted file mode 100644 index 071d4a83c20a..000000000000 --- a/core/server/master/src/main/java/alluxio/master/throttle/SystemMonitor.java +++ /dev/null @@ -1,423 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.throttle; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.MasterProcess; -import alluxio.metrics.MetricsSystem; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.LinkedList; -import java.util.List; -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * The system monitor. - */ -@NotThreadSafe -public class SystemMonitor { - private static final Logger LOG = LoggerFactory.getLogger(SystemMonitor.class); - - private List mServerIndicatorsList; - private ServerIndicator mAggregatedServerIndicators; - - private FileSystemIndicator mCurrentFileSystemIndicators; - private FileSystemIndicator mPrevFileSystemIndicators; - private FileSystemIndicator mDeltaFilesystemIndicators; - private final int mMaxNumberOfSnapshot; - - private long mCurrentThresholdTimeIn68Sec; - private ServerIndicator mPitThresholdActive; - private ServerIndicator mAggregateThresholdActive; - private ServerIndicator mPitThresholdStressed; - private ServerIndicator mAggregateThresholdStressed; - private ServerIndicator mPitThresholdOverloaded; - private ServerIndicator mAggregateThresholdOverloaded; - - private volatile SystemStatus mCurrentSystemStatus; - - private PitInfo mPrevPitInfo; - - private long mLastHeartBeatTimeMS; - private long mHeartBeatIntervalMS; - - /** - * Pit information. - */ - private static class PitInfo { - private SystemStatus mSystemStatus; - private long mPitTime; - private long mJVMPauseTime; - - /** - * Constructor. - * - * @param systemStatus the system status - * @param pitTime the pit time - * @param jvmPauseTime the JVM pause time - */ - public PitInfo(SystemStatus systemStatus, long pitTime, long jvmPauseTime) { - mSystemStatus = systemStatus; - mPitTime = pitTime; - mJVMPauseTime = jvmPauseTime; - } - - /** - * @return the pit time - */ - public long getPitTime() { - return mPitTime; - } - - /** - * @return the JVP pause time - */ - public long getJVMPauseTime() { - return mJVMPauseTime; - } - - /** - * @return the system status - */ - public SystemStatus getSystemStatus() { - return mSystemStatus; - } - - /** - * Sts the system status. - * - * @param systemStatus the system status - */ - public void setSystemStatus(SystemStatus systemStatus) { - mSystemStatus = systemStatus; - } - - /** - * Sets the JVM pause time. - * - * @param jvmPauseTime the JVM pause time - */ - public void setJVMPauseTime(long jvmPauseTime) { - mJVMPauseTime = jvmPauseTime; - } - - /** - * Sets the pit time. - * - * @param pitTime the pit time - */ - public void setPitTime(long pitTime) { - mPitTime = pitTime; - } - } - - /** - * The system status. - */ - public enum SystemStatus { - IDLE, - ACTIVE, - STRESSED, - OVERLOADED, - } - - /** - * The status transition. - */ - public enum StatusTransition { - UNCHANGED, - ESCALATE, - DEESCALATE, - } - - /** - * The system monitor constructor. - * - * @param masterProcess the master process - */ - public SystemMonitor(MasterProcess masterProcess) { - mServerIndicatorsList = new LinkedList<>(); - mCurrentSystemStatus = SystemStatus.IDLE; - - // Configurable - mMaxNumberOfSnapshot = Configuration.getInt(PropertyKey.MASTER_THROTTLE_OBSERVED_PIT_NUMBER); - - mCurrentThresholdTimeIn68Sec = 0; - reInitTheThresholds(); - mPrevPitInfo = new PitInfo(mCurrentSystemStatus, - ServerIndicator.getSystemTotalJVMPauseTime(), System.currentTimeMillis()); - - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName("system.status"), - () -> mCurrentSystemStatus); - - mLastHeartBeatTimeMS = 0; - mHeartBeatIntervalMS = Configuration.getMs(PropertyKey.MASTER_THROTTLE_HEARTBEAT_INTERVAL); - } - - private void reInitTheThresholds() { - // Only update thresholds every 68 seconds - long current = (System.nanoTime() >> 36); - if (current == mCurrentThresholdTimeIn68Sec) { - return; - } - mCurrentThresholdTimeIn68Sec = current; - - // covert the percentage of memory usage to usedHeap - // mPitThresholdActive; - mPitThresholdActive = ServerIndicator.createThresholdIndicator(0, - Configuration.getDouble(PropertyKey.MASTER_THROTTLE_ACTIVE_HEAP_USED_RATIO), - Configuration.getDouble(PropertyKey.MASTER_THROTTLE_ACTIVE_CPU_LOAD_RATIO), - Configuration.getMs(PropertyKey.MASTER_THROTTLE_ACTIVE_HEAP_GC_TIME), - Configuration.getInt(PropertyKey.MASTER_THROTTLE_ACTIVE_RPC_QUEUE_SIZE)); - // mPitThresholdStressed; - mPitThresholdStressed = ServerIndicator.createThresholdIndicator(0, - Configuration.getDouble(PropertyKey.MASTER_THROTTLE_STRESSED_HEAP_USED_RATIO), - Configuration.getDouble(PropertyKey.MASTER_THROTTLE_STRESSED_CPU_LOAD_RATIO), - Configuration.getMs(PropertyKey.MASTER_THROTTLE_STRESSED_HEAP_GC_TIME), - Configuration.getInt(PropertyKey.MASTER_THROTTLE_STRESSED_RPC_QUEUE_SIZE)); - // mPitThresholdOverloaded; - mPitThresholdOverloaded = ServerIndicator.createThresholdIndicator(0, - Configuration.getDouble(PropertyKey.MASTER_THROTTLE_OVERLOADED_HEAP_USED_RATIO), - Configuration.getDouble(PropertyKey.MASTER_THROTTLE_OVERLOADED_CPU_LOAD_RATIO), - Configuration.getMs(PropertyKey.MASTER_THROTTLE_OVERLOADED_HEAP_GC_TIME), - Configuration.getInt(PropertyKey.MASTER_THROTTLE_OVERLOADED_RPC_QUEUE_SIZE)); - mAggregateThresholdActive = new ServerIndicator(mPitThresholdActive, mMaxNumberOfSnapshot); - mAggregateThresholdStressed - = new ServerIndicator(mPitThresholdStressed, mMaxNumberOfSnapshot); - mAggregateThresholdOverloaded - = new ServerIndicator(mPitThresholdOverloaded, mMaxNumberOfSnapshot); - } - - /** - * Main run. - */ - public void run() { - // the indicator pit time is not necessary accurate, - long pitTimeMS = System.currentTimeMillis(); - collectIndicators(pitTimeMS); - checkAndBackPressure(); - mPrevPitInfo.setPitTime(pitTimeMS); - mPrevPitInfo.setSystemStatus(mCurrentSystemStatus); - mPrevPitInfo.setJVMPauseTime(mServerIndicatorsList - .get(mServerIndicatorsList.size() - 1).getPITTotalJVMPauseTimeMS()); - } - - private void collectIndicators(long pitTimeMS) { - collectServerIndicators(pitTimeMS); - collectFileSystemIndicators(pitTimeMS); - } - - private void produceDeltaFilesystemIndicators() { - // Only be able to calculate the delta if the two filesystem indicators are fetched - // continuously, this means both previous and current status should be either STRESSED - // or OVERLOADED - if ((mPrevPitInfo.getSystemStatus() == SystemStatus.STRESSED - || mPrevPitInfo.getSystemStatus() == SystemStatus.OVERLOADED) - && (mCurrentSystemStatus == SystemStatus.STRESSED - || mCurrentSystemStatus == SystemStatus.OVERLOADED) - && mPrevFileSystemIndicators != null - && mCurrentFileSystemIndicators != null) { - mDeltaFilesystemIndicators = new FileSystemIndicator(mCurrentFileSystemIndicators); - mDeltaFilesystemIndicators.deltaTo(mPrevFileSystemIndicators); - } else { - mDeltaFilesystemIndicators = null; - } - } - - private void collectFileSystemIndicators(long snapshotTimeMS) { - mPrevFileSystemIndicators = mCurrentFileSystemIndicators; - if (mCurrentSystemStatus == SystemStatus.IDLE - || mCurrentSystemStatus == SystemStatus.ACTIVE) { - // don't collect information in case of IDLE and ACTIVE - mCurrentFileSystemIndicators = null; - mDeltaFilesystemIndicators = null; - return; - } - - FileSystemIndicator fileSystemIndicator = new FileSystemIndicator(); - fileSystemIndicator.setPitTimeMS(snapshotTimeMS); - - mCurrentFileSystemIndicators = fileSystemIndicator; - produceDeltaFilesystemIndicators(); - } - - private void collectServerIndicators(long snapshotTimeMS) { - ServerIndicator serverIndicator - = ServerIndicator.createFromMetrics( - mPrevPitInfo != null ? mPrevPitInfo.getJVMPauseTime() : 0); - serverIndicator.setPitTimeMS(snapshotTimeMS); - - mServerIndicatorsList.add(serverIndicator); - if (mServerIndicatorsList.size() > mMaxNumberOfSnapshot && !mServerIndicatorsList.isEmpty()) { - ServerIndicator toBeRemoved = mServerIndicatorsList.get(0); - if (mAggregatedServerIndicators != null) { - mAggregatedServerIndicators.reduction(toBeRemoved); - } - mServerIndicatorsList.remove(0); - } - - if (mAggregatedServerIndicators == null) { - mAggregatedServerIndicators = new ServerIndicator(serverIndicator); - } else { - mAggregatedServerIndicators.addition(serverIndicator); - } - } - - /** - * Checks whether to escalate, deescalate or unchanged, if the pit low boundary is null, - * not deescalated, and if the high aggregated boundary is null, not escalated. - * - * @param aggregatedServerIndicator the aggregated server indicator - * @param pitServerIndicator the pit server indicator - * @param lowBoundaryPitIndicator the low pit boundary indicator - * @param highBoundaryPitIndicator the high pit boundary indicator - * @param lowBoundaryAggregateIndicator the low aggregated boundary indicator - * @param highBoundaryAggregateIndicator the high aggregated boundary indicator - * @return Whether to escalate, deescalate or no change - */ - private StatusTransition checkBoundary(ServerIndicator aggregatedServerIndicator, - ServerIndicator pitServerIndicator, - @Nullable ServerIndicator lowBoundaryPitIndicator, - @Nullable ServerIndicator highBoundaryPitIndicator, - @Nullable ServerIndicator lowBoundaryAggregateIndicator, - @Nullable ServerIndicator highBoundaryAggregateIndicator) { - // Just start with simple. - // When it is clear, more rules can be added - - // The pitServerIndicator shows the current status, it reflects the current status. - // so it is checked first. If it is in reasonable range, the system should be - if (lowBoundaryPitIndicator != null - && pitServerIndicator.getHeapUsed() < lowBoundaryPitIndicator.getHeapUsed()) { - return StatusTransition.DEESCALATE; - } - - if (highBoundaryPitIndicator != null - && pitServerIndicator.getHeapUsed() < highBoundaryPitIndicator.getHeapUsed()) { - // if the heap usage is not crossing the higher boundary, no change - return StatusTransition.UNCHANGED; - } - - if (highBoundaryAggregateIndicator != null - && aggregatedServerIndicator.getHeapUsed() - > highBoundaryAggregateIndicator.getHeapUsed()) { - return StatusTransition.ESCALATE; - } - - return StatusTransition.UNCHANGED; - } - - private void checkAndBackPressure() { - if (mServerIndicatorsList.isEmpty()) { - return; - } - - reInitTheThresholds(); - - ServerIndicator pitIndicator = mServerIndicatorsList.get(mServerIndicatorsList.size() - 1); - StatusTransition statusTransition = StatusTransition.UNCHANGED; - if (mLastHeartBeatTimeMS != 0 - && (System.currentTimeMillis() - mLastHeartBeatTimeMS - - mHeartBeatIntervalMS) >= mPitThresholdOverloaded.getTotalJVMPauseTimeMS()) { - // This heartbeat time is far from the heartbeat time last time - // Directly set the status to OVERLOADED - // Since it is set to overloaded, skip the rest check. - statusTransition = (mCurrentSystemStatus == SystemStatus.OVERLOADED) - ? StatusTransition.UNCHANGED : StatusTransition.ESCALATE; - mCurrentSystemStatus = SystemStatus.OVERLOADED; - } else { - // If it is not decided yet, check according to the low and up boundary. - - // Status transition would look like: IDLE <--> ACTIVE <--> STRESSED <--> OVERLOADED - // There are three set of thresholds, - // pit ACTIVE threshold, aggregated ACTIVE threshold - // pit STRESSED threshold, aggregated STRESSED threshold - // pit OVERLOADED threshold, aggregated OVERLOADED threshold - // Every status would have two set of boundaries: low boundary and up boundary - // IDLE: - // low boundary: null, null - // up boundary: pit STRESSED threshold, aggregated STRESSED threshold - // ACTIVE: - // low boundary: pit ACTIVE threshold, aggregated ACTIVE threshold - // up boundary: pit STRESSED threshold, aggregated STRESSED threshold - // STRESSED: - // low boundary: pit STRESSED threshold, aggregated STRESSED threshold - // up boundary: pit OVERLOADED threshold, aggregated OVERLOADED threshold - // OVERLOADED: - // low boundary: pit OVERLOADED threshold, aggregated OVERLOADED threshold - // up boundary: null, null - switch (mCurrentSystemStatus) { - case IDLE: - statusTransition = checkBoundary(mAggregatedServerIndicators, pitIndicator, null, - mPitThresholdActive, null, mAggregateThresholdActive); - if (statusTransition == StatusTransition.ESCALATE) { - mCurrentSystemStatus = SystemStatus.ACTIVE; - } - break; - case ACTIVE: - statusTransition = checkBoundary(mAggregatedServerIndicators, pitIndicator, - mPitThresholdActive, mPitThresholdStressed, - mAggregateThresholdActive, mAggregateThresholdStressed); - if (statusTransition == StatusTransition.DEESCALATE) { - mCurrentSystemStatus = SystemStatus.IDLE; - } else if (statusTransition == StatusTransition.ESCALATE) { - mCurrentSystemStatus = SystemStatus.STRESSED; - } - break; - case STRESSED: - statusTransition = checkBoundary(mAggregatedServerIndicators, pitIndicator, - mPitThresholdStressed, mPitThresholdOverloaded, - mAggregateThresholdStressed, mAggregateThresholdOverloaded); - if (statusTransition == StatusTransition.DEESCALATE) { - mCurrentSystemStatus = SystemStatus.ACTIVE; - } else if (statusTransition == StatusTransition.ESCALATE) { - mCurrentSystemStatus = SystemStatus.OVERLOADED; - } - break; - case OVERLOADED: - statusTransition = checkBoundary(mAggregatedServerIndicators, pitIndicator, - mPitThresholdOverloaded, null, - mAggregateThresholdOverloaded, null); - if (statusTransition == StatusTransition.DEESCALATE) { - mCurrentSystemStatus = SystemStatus.STRESSED; - } - break; - default: - } - } - - if (mCurrentSystemStatus == SystemStatus.STRESSED - || mCurrentSystemStatus == SystemStatus.OVERLOADED) { - LOG.warn("System transition status is {}, status is {}, related Server aggregate" - + " indicators:{}, pit indicators:{}", - statusTransition, mCurrentSystemStatus, mAggregatedServerIndicators, pitIndicator); - if (mDeltaFilesystemIndicators != null) { - LOG.warn("The delta filesystem indicators {}", mDeltaFilesystemIndicators); - } - } else { - LOG.debug("System transition status is {}, status is {}, related Server aggregate" - + " indicators:{}, pit indicators:{}", - statusTransition, mCurrentSystemStatus, mAggregatedServerIndicators, pitIndicator); - - if (mDeltaFilesystemIndicators != null) { - LOG.debug("The delta filesystem indicators {}", mDeltaFilesystemIndicators); - } - } - - mLastHeartBeatTimeMS = System.currentTimeMillis(); - } -} - diff --git a/core/server/master/src/main/java/alluxio/master/throttle/ThrottleMasterFactory.java b/core/server/master/src/main/java/alluxio/master/throttle/ThrottleMasterFactory.java deleted file mode 100644 index bfb224cd4590..000000000000 --- a/core/server/master/src/main/java/alluxio/master/throttle/ThrottleMasterFactory.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.throttle; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.MasterContext; -import alluxio.master.MasterFactory; -import alluxio.master.MasterRegistry; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Factory to create a {@link DefaultThrottleMaster} instance. - */ -@ThreadSafe -public final class ThrottleMasterFactory implements MasterFactory { - private static final Logger LOG = LoggerFactory.getLogger(ThrottleMasterFactory.class); - - /** - * Constructs a new {@link ThrottleMasterFactory}. - */ - public ThrottleMasterFactory() {} - - @Override - public boolean isEnabled() { - return Configuration.getBoolean(PropertyKey.MASTER_THROTTLE_ENABLED); - } - - @Override - public String getName() { - return Constants.THROTTLE_MASTER_NAME; - } - - @Override - public DefaultThrottleMaster create(MasterRegistry registry, MasterContext context) { - if (!isEnabled()) { - return null; - } - LOG.info("Creating {} ", DefaultThrottleMaster.class.getName()); - return new DefaultThrottleMaster(registry, context); - } -} - diff --git a/core/server/master/src/main/resources/META-INF/services/alluxio.master.MasterFactory b/core/server/master/src/main/resources/META-INF/services/alluxio.master.MasterFactory deleted file mode 100644 index 7f0753743dd4..000000000000 --- a/core/server/master/src/main/resources/META-INF/services/alluxio.master.MasterFactory +++ /dev/null @@ -1,17 +0,0 @@ -# -# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 -# (the "License"). You may not use this work except in compliance with the License, which is -# available at www.apache.org/licenses/LICENSE-2.0 -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied, as more fully set forth in the License. -# -# See the NOTICE file distributed with this work for information regarding copyright ownership. -# - -alluxio.master.block.BlockMasterFactory -alluxio.master.file.FileSystemMasterFactory -alluxio.master.journal.JournalMasterFactory -alluxio.master.meta.MetaMasterFactory -alluxio.master.metrics.MetricsMasterFactory -alluxio.master.throttle.ThrottleMasterFactory diff --git a/core/server/master/src/test/java/alluxio/master/MasterTestUtils.java b/core/server/master/src/test/java/alluxio/master/MasterTestUtils.java deleted file mode 100644 index 552fc4f05292..000000000000 --- a/core/server/master/src/test/java/alluxio/master/MasterTestUtils.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master; - -import static org.mockito.Mockito.mock; - -import alluxio.master.journal.JournalSystem; -import alluxio.master.journal.noop.NoopJournalSystem; -import alluxio.master.metastore.BlockMetaStore; -import alluxio.master.metastore.InodeStore; -import alluxio.master.metastore.heap.HeapBlockMetaStore; -import alluxio.master.metastore.heap.HeapInodeStore; -import alluxio.security.user.UserState; -import alluxio.underfs.MasterUfsManager; - -/** - * Util methods to help with master testing. - */ -public final class MasterTestUtils { - - /** - * @return a basic master context for the purpose of testing - */ - public static CoreMasterContext testMasterContext() { - return testMasterContext(new NoopJournalSystem()); - } - - /** - * @return a basic master context for the purpose of testing - * @param journalSystem a journal system to use in the context - */ - public static CoreMasterContext testMasterContext(JournalSystem journalSystem) { - return testMasterContext(journalSystem, null); - } - - /** - * @return a basic master context for the purpose of testing - * @param journalSystem a journal system to use in the context - * @param userState the user state to use in the context - */ - public static CoreMasterContext testMasterContext(JournalSystem journalSystem, - UserState userState) { - return testMasterContext(journalSystem, userState, - HeapBlockMetaStore::new, x -> new HeapInodeStore()); - } - - /** - * @return a basic master context for the purpose of testing - * @param journalSystem a journal system to use in the context - * @param userState the user state to use in the context - * @param blockStoreFactory a factory to create {@link BlockMetaStore} - * @param inodeStoreFactory a factory to create {@link InodeStore} - */ - public static CoreMasterContext testMasterContext( - JournalSystem journalSystem, UserState userState, - BlockMetaStore.Factory blockStoreFactory, - InodeStore.Factory inodeStoreFactory) { - return CoreMasterContext.newBuilder() - .setJournalSystem(journalSystem) - .setPrimarySelector(new AlwaysStandbyPrimarySelector()) - .setUserState(userState) - .setSafeModeManager(new TestSafeModeManager()) - .setBackupManager(mock(BackupManager.class)) - .setBlockStoreFactory(blockStoreFactory) - .setInodeStoreFactory(inodeStoreFactory) - .setStartTimeMs(-1) - .setPort(-1) - .setUfsManager(new MasterUfsManager()) - .build(); - } - - private MasterTestUtils() {} // Not intended for instantiation. -} diff --git a/core/server/master/src/test/java/alluxio/master/block/BlockMasterTest.java b/core/server/master/src/test/java/alluxio/master/block/BlockMasterTest.java deleted file mode 100644 index 900c179a48c4..000000000000 --- a/core/server/master/src/test/java/alluxio/master/block/BlockMasterTest.java +++ /dev/null @@ -1,518 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.block; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; -import alluxio.clock.ManualClock; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.NotFoundException; -import alluxio.grpc.BuildVersion; -import alluxio.grpc.Command; -import alluxio.grpc.CommandType; -import alluxio.grpc.RegisterWorkerPOptions; -import alluxio.grpc.StorageList; -import alluxio.grpc.WorkerLostStorageInfo; -import alluxio.heartbeat.HeartbeatContext; -import alluxio.heartbeat.HeartbeatScheduler; -import alluxio.heartbeat.ManuallyScheduleHeartbeat; -import alluxio.master.CoreMasterContext; -import alluxio.master.MasterRegistry; -import alluxio.master.MasterTestUtils; -import alluxio.master.journal.JournalSystem; -import alluxio.master.journal.noop.NoopJournalSystem; -import alluxio.master.metrics.MetricsMaster; -import alluxio.master.metrics.MetricsMasterFactory; -import alluxio.metrics.Metric; -import alluxio.proto.meta.Block; -import alluxio.util.ThreadFactoryUtils; -import alluxio.util.executor.ExecutorServiceFactories; -import alluxio.wire.BlockInfo; -import alluxio.wire.BlockLocation; -import alluxio.wire.WorkerInfo; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -/** - * Unit tests for {@link BlockMaster}. - */ -public class BlockMasterTest { - private static final WorkerNetAddress NET_ADDRESS_1 = new WorkerNetAddress().setHost("localhost") - .setRpcPort(80).setDataPort(81).setWebPort(82); - private static final WorkerNetAddress NET_ADDRESS_2 = new WorkerNetAddress().setHost("localhost") - .setRpcPort(83).setDataPort(84).setWebPort(85); - - private static final List NO_BLOCKS = ImmutableList.of(); - private static final Map> NO_BLOCKS_ON_LOCATION - = ImmutableMap.of(); - private static final Map NO_LOST_STORAGE = ImmutableMap.of(); - - private BlockMaster mBlockMaster; - private MasterRegistry mRegistry; - private ManualClock mClock; - private ExecutorService mExecutorService; - private ExecutorService mClientExecutorService; - private MetricsMaster mMetricsMaster; - private List mMetrics; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - /** The exception expected to be thrown. */ - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - @ClassRule - public static ManuallyScheduleHeartbeat sManuallySchedule = new ManuallyScheduleHeartbeat( - HeartbeatContext.MASTER_LOST_WORKER_DETECTION); - - /** - * Sets up the dependencies before a test runs. - */ - @Before - public void before() throws Exception { - // set a large value of PropertyKey.MASTER_LOST_WORKER_DELETION_TIMEOUT_MS - // to prevent worker to be deleted after it is lost - Configuration.set(PropertyKey.MASTER_LOST_WORKER_DELETION_TIMEOUT_MS, Integer.MAX_VALUE); - mRegistry = new MasterRegistry(); - mMetrics = Lists.newArrayList(); - JournalSystem journalSystem = new NoopJournalSystem(); - CoreMasterContext masterContext = MasterTestUtils.testMasterContext(); - mMetricsMaster = new MetricsMasterFactory().create(mRegistry, masterContext); - mClock = new ManualClock(); - mExecutorService = - Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("TestBlockMaster-%d", true)); - mBlockMaster = new DefaultBlockMaster(mMetricsMaster, masterContext, mClock, - ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService)); - mRegistry.add(BlockMaster.class, mBlockMaster); - mRegistry.start(true); - } - - /** - * Stops the master after a test ran. - */ - @After - public void after() throws Exception { - mRegistry.stop(); - } - - @Test - public void buildVersion() throws Exception { - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - - // Sequence to simulate worker upgrade and downgrade, - // with or without buildVersion in registerWorkerPOptions - BuildVersion[] buildVersions = new BuildVersion[]{ - null, - BuildVersion.newBuilder().setVersion("1.0.0") - .setRevision("foobar").build(), - BuildVersion.newBuilder().setVersion("1.1.0") - .setRevision("fizzbuzz").build(), - null, - }; - - for (BuildVersion bv : buildVersions) { - RegisterWorkerPOptions options = (bv == null) - ? RegisterWorkerPOptions.getDefaultInstance() - : RegisterWorkerPOptions.newBuilder().setBuildVersion(bv).build(); - - mBlockMaster.workerRegister(worker1, - ImmutableList.of(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 10L), - NO_BLOCKS_ON_LOCATION, - NO_LOST_STORAGE, - options); - - BuildVersion actual = mBlockMaster.getWorker(worker1).getBuildVersion(); - assertEquals(bv == null ? "" : bv.getVersion(), actual.getVersion()); - assertEquals(bv == null ? "" : bv.getRevision(), actual.getRevision()); - } - } - - @Test - public void countBytes() throws Exception { - // Register two workers - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2); - List tiers = Arrays.asList(Constants.MEDIUM_MEM, Constants.MEDIUM_SSD); - Map worker1TotalBytesOnTiers = - ImmutableMap.of(Constants.MEDIUM_MEM, 10L, Constants.MEDIUM_SSD, 20L); - Map worker2TotalBytesOnTiers = - ImmutableMap.of(Constants.MEDIUM_MEM, 1000L, Constants.MEDIUM_SSD, 2000L); - Map worker1UsedBytesOnTiers = - ImmutableMap.of(Constants.MEDIUM_MEM, 1L, Constants.MEDIUM_SSD, 2L); - Map worker2UsedBytesOnTiers = - ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_SSD, 200L); - mBlockMaster.workerRegister(worker1, tiers, worker1TotalBytesOnTiers, worker1UsedBytesOnTiers, - NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance()); - mBlockMaster.workerRegister(worker2, tiers, worker2TotalBytesOnTiers, worker2UsedBytesOnTiers, - NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance()); - - // Check that byte counts are summed correctly. - assertEquals(3030, mBlockMaster.getCapacityBytes()); - assertEquals(303L, mBlockMaster.getUsedBytes()); - assertEquals(ImmutableMap.of(Constants.MEDIUM_MEM, 1010L, Constants.MEDIUM_SSD, 2020L), - mBlockMaster.getTotalBytesOnTiers()); - assertEquals(ImmutableMap.of(Constants.MEDIUM_MEM, 101L, Constants.MEDIUM_SSD, 202L), - mBlockMaster.getUsedBytesOnTiers()); - } - - @Test - public void detectLostWorkers() throws Exception { - // Register a worker. - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - mBlockMaster.workerRegister(worker1, - ImmutableList.of(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 10L), - NO_BLOCKS_ON_LOCATION, - NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - - // Advance the block master's clock by an hour so that worker appears lost. - mClock.setTimeMs(System.currentTimeMillis() + Constants.HOUR_MS); - - // Run the lost worker detector. - HeartbeatScheduler.execute(HeartbeatContext.MASTER_LOST_WORKER_DETECTION); - - // Make sure the worker is detected as lost. - List info = mBlockMaster.getLostWorkersInfoList(); - assertEquals(worker1, Iterables.getOnlyElement(info).getId()); - } - - @Test - public void autoDeleteTimeoutWorker() throws Exception { - - // In default configuration the lost worker will never be deleted. So set a short timeout - Configuration.set(PropertyKey.MASTER_LOST_WORKER_DELETION_TIMEOUT_MS, 1000); - // Register a worker. - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - mBlockMaster.workerRegister(worker1, - ImmutableList.of(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 10L), - NO_BLOCKS_ON_LOCATION, - NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - - // Advance the block master's clock by an hour so that worker can be deleted. - mClock.setTimeMs(System.currentTimeMillis() + Constants.HOUR_MS); - - // Run the lost worker detector. - HeartbeatScheduler.execute(HeartbeatContext.MASTER_LOST_WORKER_DETECTION); - - // Make sure the worker has been deleted. - List info = mBlockMaster.getLostWorkersInfoList(); - assertEquals(0, mBlockMaster.getLostWorkersInfoList().size()); - assertThrows(NotFoundException.class, () -> mBlockMaster.getWorker(worker1)); - assertEquals(0, mBlockMaster.getWorkerCount()); - } - - @Test - public void workerReregisterRemembersLostWorker() throws Exception { - // Register a worker. - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - mBlockMaster.workerRegister(worker1, - ImmutableList.of(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 10L), - NO_BLOCKS_ON_LOCATION, - NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - - // Advance the block master's clock by an hour so that the worker appears lost. - mClock.setTimeMs(System.currentTimeMillis() + Constants.HOUR_MS); - - // Run the lost worker detector. - HeartbeatScheduler.execute(HeartbeatContext.MASTER_LOST_WORKER_DETECTION); - - // Reregister the worker using its original worker id. - mBlockMaster.getWorkerId(NET_ADDRESS_1); - mBlockMaster.workerRegister(worker1, - ImmutableList.of(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 10L), - NO_BLOCKS_ON_LOCATION, - NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - - // Check that there are no longer any lost workers and there is a live worker. - assertEquals(1, mBlockMaster.getWorkerCount()); - assertEquals(0, mBlockMaster.getLostWorkersInfoList().size()); - } - - @Test - public void removeBlockTellsWorkersToRemoveTheBlock() throws Exception { - // Create a worker with a block. - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - long blockId = 1L; - mBlockMaster.workerRegister(worker1, Arrays.asList(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - mBlockMaster.commitBlock(worker1, 50L, - Constants.MEDIUM_MEM, Constants.MEDIUM_MEM, blockId, 20L); - - // Remove the block - mBlockMaster.removeBlocks(Arrays.asList(1L), /*delete=*/false); - - // Check that the worker heartbeat tells the worker to remove the block. - Map memUsage = ImmutableMap.of(Constants.MEDIUM_MEM, 0L); - alluxio.grpc.Command heartBeat = mBlockMaster.workerHeartbeat(worker1, null, memUsage, - NO_BLOCKS, NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, mMetrics); - assertEquals(ImmutableList.of(1L), heartBeat.getDataList()); - } - - @Test - public void registerCleansUpOrphanedBlocks() throws Exception { - // Create a worker with unknown blocks. - long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1); - List orphanedBlocks = Arrays.asList(1L, 2L); - Map memUsage = ImmutableMap.of(Constants.MEDIUM_MEM, 10L); - - Block.BlockLocation blockLoc = Block.BlockLocation.newBuilder() - .setWorkerId(workerId).setTier(Constants.MEDIUM_MEM) - .setMediumType(Constants.MEDIUM_MEM).build(); - mBlockMaster.workerRegister(workerId, Arrays.asList(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - memUsage, ImmutableMap.of(blockLoc, orphanedBlocks), NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - - // Check that the worker heartbeat tells the worker to remove the blocks. - alluxio.grpc.Command heartBeat = mBlockMaster.workerHeartbeat(workerId, null, - memUsage, NO_BLOCKS, NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, mMetrics); - assertEquals(orphanedBlocks, - heartBeat.getDataList().stream().sorted().collect(Collectors.toList())); - } - - @Test - public void workerHeartbeatUpdatesMemoryCount() throws Exception { - // Create a worker. - long worker = mBlockMaster.getWorkerId(NET_ADDRESS_1); - Map initialUsedBytesOnTiers = ImmutableMap.of(Constants.MEDIUM_MEM, 50L); - mBlockMaster.workerRegister(worker, Arrays.asList(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - initialUsedBytesOnTiers, NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - - // Update used bytes with a worker heartbeat. - Map newUsedBytesOnTiers = ImmutableMap.of(Constants.MEDIUM_MEM, 50L); - mBlockMaster.workerHeartbeat(worker, null, newUsedBytesOnTiers, - NO_BLOCKS, NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, mMetrics); - - WorkerInfo workerInfo = Iterables.getOnlyElement(mBlockMaster.getWorkerInfoList()); - assertEquals(50, workerInfo.getUsedBytes()); - } - - @Test - public void workerHeartbeatUpdatesRemovedBlocks() throws Exception { - // Create a worker. - long worker = mBlockMaster.getWorkerId(NET_ADDRESS_1); - mBlockMaster.workerRegister(worker, Arrays.asList(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - long blockId = 1L; - mBlockMaster.commitBlock(worker, 50L, Constants.MEDIUM_MEM, - Constants.MEDIUM_MEM, blockId, 20L); - - // Indicate that blockId is removed on the worker. - mBlockMaster.workerHeartbeat(worker, null, - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), - ImmutableList.of(blockId), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, mMetrics); - assertTrue(mBlockMaster.getBlockInfo(blockId).getLocations().isEmpty()); - } - - @Test - public void workerHeartbeatUpdatesAddedBlocks() throws Exception { - // Create two workers. - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - mBlockMaster.workerRegister(worker1, Arrays.asList(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2); - mBlockMaster.workerRegister(worker2, Arrays.asList(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - - // Commit blockId to worker1. - long blockId = 1L; - mBlockMaster.commitBlock(worker1, 50L, Constants.MEDIUM_MEM, - Constants.MEDIUM_MEM, blockId, 20L); - - // Send a heartbeat from worker2 saying that it's added blockId. - List addedBlocks = ImmutableList.of(blockId); - Block.BlockLocation blockOnWorker2 = Block.BlockLocation.newBuilder() - .setWorkerId(worker2).setTier(Constants.MEDIUM_MEM) - .setMediumType(Constants.MEDIUM_MEM).build(); - mBlockMaster.workerHeartbeat(worker2, null, - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS, - ImmutableMap.of(blockOnWorker2, addedBlocks), - NO_LOST_STORAGE, mMetrics); - - // The block now has two locations. - assertEquals(2, mBlockMaster.getBlockInfo(blockId).getLocations().size()); - } - - @Test - public void workerHeartbeatUpdatesLostStorage() throws Exception { - // Create two workers. - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - mBlockMaster.workerRegister(worker1, Arrays.asList(Constants.MEDIUM_MEM, Constants.MEDIUM_SSD), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_SSD, 200L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_SSD, 0L), - NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2); - mBlockMaster.workerRegister(worker2, Arrays.asList(Constants.MEDIUM_MEM, Constants.MEDIUM_HDD), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_HDD, 300L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_HDD, 0L), - NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - - Map lostStorageOnWorker1 = new HashMap<>(); - lostStorageOnWorker1.put(Constants.MEDIUM_SSD, StorageList.newBuilder() - .addAllStorage(Arrays.asList("/ssd/one", "/ssd/two")).build()); - Map lostStorageOnWorker2 = new HashMap<>(); - lostStorageOnWorker2.put(Constants.MEDIUM_HDD, - StorageList.newBuilder().addStorage("/hdd/one").build()); - - mBlockMaster.workerHeartbeat(worker1, - ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_SSD, 0L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_SSD, 0L), NO_BLOCKS, - NO_BLOCKS_ON_LOCATION, lostStorageOnWorker1, mMetrics); - mBlockMaster.workerHeartbeat(worker2, - ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_HDD, 200L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_HDD, 0L), NO_BLOCKS, - NO_BLOCKS_ON_LOCATION, lostStorageOnWorker2, mMetrics); - - // Two workers have lost storage paths - assertEquals(2, mBlockMaster.getWorkerLostStorage().size()); - int lostStorageNum = 0; - for (WorkerLostStorageInfo info : mBlockMaster.getWorkerLostStorage()) { - for (StorageList list : info.getLostStorageMap().values()) { - lostStorageNum += list.getStorageList().size(); - } - } - assertEquals(3, lostStorageNum); - } - - @Test - public void unknownWorkerHeartbeatTriggersRegisterRequest() { - Command heartBeat = mBlockMaster.workerHeartbeat(0, null, null, null, null, null, mMetrics); - assertEquals(Command.newBuilder().setCommandType(CommandType.Register).build(), heartBeat); - } - - @Test - public void stopTerminatesExecutorService() throws Exception { - mBlockMaster.stop(); - assertTrue(mExecutorService.isTerminated()); - } - - @Test - public void getBlockInfo() throws Exception { - // Create a worker with a block. - long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); - long blockId = 1L; - long blockLength = 20L; - mBlockMaster.workerRegister(worker1, Arrays.asList(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - mBlockMaster.commitBlock(worker1, 50L, Constants.MEDIUM_MEM, - Constants.MEDIUM_MEM, blockId, blockLength); - - BlockLocation blockLocation = new BlockLocation() - .setTierAlias(Constants.MEDIUM_MEM) - .setWorkerAddress(NET_ADDRESS_1) - .setWorkerId(worker1) - .setMediumType(Constants.MEDIUM_MEM); - BlockInfo expectedBlockInfo = new BlockInfo() - .setBlockId(1L) - .setLength(20L) - .setLocations(ImmutableList.of(blockLocation)); - assertEquals(expectedBlockInfo, mBlockMaster.getBlockInfo(blockId)); - } - - @Test - public void getNewContainerId() throws Exception { - final int total = 10_000; - long containerIdReservationSize = Configuration.getInt( - PropertyKey.MASTER_CONTAINER_ID_RESERVATION_SIZE); - - final int parallelNum = 5; - ExecutorService containerIdFetcher = Executors.newFixedThreadPool(parallelNum); - AtomicInteger times = new AtomicInteger(0); - assertEquals(0L, mBlockMaster.getNewContainerId()); - - CyclicBarrier barrier = new CyclicBarrier(parallelNum); - - for (int count = 0; count < parallelNum; count++) { - containerIdFetcher.submit(() -> { - try { - barrier.await(); - while (times.incrementAndGet() < total) { - mBlockMaster.getNewContainerId(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } - - containerIdFetcher.shutdown(); - containerIdFetcher.awaitTermination(6, TimeUnit.SECONDS); - mBlockMaster.close(); - - long journaledNextContainerId = mBlockMaster.getJournaledNextContainerId(); - - assertTrue(journaledNextContainerId >= total); - assertTrue(journaledNextContainerId <= total + containerIdReservationSize); - } - - @Test - public void stop() throws Exception { - mRegistry.stop(); - assertTrue(mExecutorService.isShutdown()); - assertTrue(mExecutorService.isTerminated()); - } -} diff --git a/core/server/master/src/test/java/alluxio/master/file/AccessTimeUpdaterTest.java b/core/server/master/src/test/java/alluxio/master/file/AccessTimeUpdaterTest.java deleted file mode 100644 index 496a56a6ce6d..000000000000 --- a/core/server/master/src/test/java/alluxio/master/file/AccessTimeUpdaterTest.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.powermock.api.mockito.PowerMockito.when; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.CoreMasterContext; -import alluxio.master.MasterRegistry; -import alluxio.master.MasterTestUtils; -import alluxio.master.block.BlockMaster; -import alluxio.master.block.BlockMasterFactory; -import alluxio.master.file.contexts.CreateFileContext; -import alluxio.master.file.meta.Inode; -import alluxio.master.file.meta.InodeDirectoryIdGenerator; -import alluxio.master.file.meta.InodeLockManager; -import alluxio.master.file.meta.InodeTree; -import alluxio.master.file.meta.LockedInodePath; -import alluxio.master.file.meta.MountTable; -import alluxio.master.file.meta.MutableInode; -import alluxio.master.file.meta.options.MountInfo; -import alluxio.master.journal.JournalContext; -import alluxio.master.journal.JournalSystem; -import alluxio.master.journal.JournalTestUtils; -import alluxio.master.journal.JournalType; -import alluxio.master.journal.NoopJournalContext; -import alluxio.master.metastore.InodeStore; -import alluxio.master.metrics.MetricsMasterFactory; -import alluxio.proto.journal.Journal; -import alluxio.security.authorization.Mode; -import alluxio.underfs.UfsManager; -import alluxio.util.CommonUtils; -import alluxio.util.executor.ControllableScheduler; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; - -import java.time.Clock; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * Unit tests for {@link PermissionChecker}. - */ -public final class AccessTimeUpdaterTest { - private static final String TEST_OWNER = "user1"; - private static final String TEST_GROUP = ""; - private static final Mode TEST_MODE = new Mode((short) 0755); - - private ControllableScheduler mScheduler; - private FileSystemMaster mFileSystemMaster; - private AccessTimeUpdater mAccessTimeUpdater; - private BlockMaster mBlockMaster; - private InodeStore mInodeStore; - private CoreMasterContext mContext; - - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - private InodeTree mInodeTree; - - @Before - public final void before() throws Exception { - mFileSystemMaster = Mockito.mock(FileSystemMaster.class); - when(mFileSystemMaster.getName()).thenReturn(Constants.FILE_SYSTEM_MASTER_NAME); - Configuration.set(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS); - MasterRegistry registry = new MasterRegistry(); - JournalSystem journalSystem = JournalTestUtils.createJournalSystem(mTestFolder); - mContext = MasterTestUtils.testMasterContext(journalSystem); - new MetricsMasterFactory().create(registry, mContext); - mBlockMaster = new BlockMasterFactory().create(registry, mContext); - InodeDirectoryIdGenerator directoryIdGenerator = new InodeDirectoryIdGenerator(mBlockMaster); - UfsManager manager = mock(UfsManager.class); - MountTable mountTable = new MountTable(manager, mock(MountInfo.class), Clock.systemUTC()); - InodeLockManager lockManager = new InodeLockManager(); - mInodeStore = mContext.getInodeStoreFactory().apply(lockManager); - mInodeTree = - new InodeTree(mInodeStore, mBlockMaster, directoryIdGenerator, mountTable, lockManager); - - journalSystem.start(); - journalSystem.gainPrimacy(); - mBlockMaster.start(true); - - Configuration.set(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED, true); - Configuration - .set(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_SUPERGROUP, "test-supergroup"); - mInodeTree.initializeRoot(TEST_OWNER, TEST_GROUP, TEST_MODE, NoopJournalContext.INSTANCE); - mScheduler = new ControllableScheduler(); - } - - private void createInode(String path, CreateFileContext context) - throws Exception { - try (LockedInodePath inodePath = - mInodeTree.lockInodePath( - new AlluxioURI(path), - InodeTree.LockPattern.WRITE_EDGE, NoopJournalContext.INSTANCE - ) - ) { - List result = mInodeTree.createPath(RpcContext.NOOP, inodePath, context); - MutableInode inode = mInodeStore.getMutable(result.get(result.size() - 1).getId()).get(); - mInodeStore.writeInode(inode); - } - } - - @Test - public void updateAccessTimeImmediately() throws Exception { - mAccessTimeUpdater = new AccessTimeUpdater(mFileSystemMaster, mInodeTree, - mContext.getJournalSystem(), 0, 0, 0); - mAccessTimeUpdater.start(); - String path = "/foo"; - JournalContext journalContext = mock(JournalContext.class); - when(journalContext.get()).thenReturn(journalContext); - createInode(path, CreateFileContext.defaults()); - long accessTime = CommonUtils.getCurrentMs() + 100L; - long inodeId; - try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), - InodeTree.LockPattern.READ, journalContext)) { - mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), accessTime); - inodeId = lockedInodes.getInode().getId(); - } - - // verify journal entry is logged - ArgumentCaptor captor = - ArgumentCaptor.forClass(Journal.JournalEntry.class); - verify(journalContext).append(captor.capture()); - assertTrue(captor.getValue().hasUpdateInode()); - assertEquals(inodeId, captor.getValue().getUpdateInode().getId()); - assertEquals(accessTime, captor.getValue().getUpdateInode().getLastAccessTimeMs()); - - // verify inode attribute is updated - assertEquals(accessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs()); - } - - @Test - public void updateAccessTimeAsync() throws Exception { - mAccessTimeUpdater = new AccessTimeUpdater(mFileSystemMaster, mInodeTree, - mContext.getJournalSystem(), 10 * Constants.SECOND_MS, 0, 0); - mAccessTimeUpdater.start(mScheduler); - String path = "/foo"; - createInode(path, CreateFileContext.defaults()); - JournalContext journalContext = mock(JournalContext.class); - when(journalContext.get()).thenReturn(journalContext); - when(mFileSystemMaster.createJournalContext()).thenReturn(journalContext); - long accessTime = CommonUtils.getCurrentMs() + 100L; - long inodeId; - try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), - InodeTree.LockPattern.READ, journalContext)) { - mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), accessTime); - inodeId = lockedInodes.getInode().getId(); - } - - // verify inode attribute is updated - assertEquals(accessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs()); - - mScheduler.jumpAndExecute(1, TimeUnit.SECONDS); - - // verify journal entry is NOT logged yet - verify(journalContext, never()).append(any(Journal.JournalEntry.class)); - - // wait for the flush to complete - mScheduler.jumpAndExecute(11, TimeUnit.SECONDS); - - /// verify journal entry is logged after the flush interval - ArgumentCaptor captor = - ArgumentCaptor.forClass(Journal.JournalEntry.class); - verify(journalContext).append(captor.capture()); - assertTrue(captor.getValue().hasUpdateInode()); - assertEquals(inodeId, captor.getValue().getUpdateInode().getId()); - assertEquals(accessTime, captor.getValue().getUpdateInode().getLastAccessTimeMs()); - } - - @Test - public void updateAccessTimePrecision() throws Exception { - mAccessTimeUpdater = new AccessTimeUpdater(mFileSystemMaster, mInodeTree, - mContext.getJournalSystem(), 0, Constants.HOUR_MS, 0); - mAccessTimeUpdater.start(); - String path = "/foo"; - createInode(path, CreateFileContext.defaults()); - JournalContext journalContext = mock(JournalContext.class); - when(journalContext.get()).thenReturn(journalContext); - when(mFileSystemMaster.createJournalContext()).thenReturn(journalContext); - long accessTime = CommonUtils.getCurrentMs() + 100L; - long inodeId; - try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), - InodeTree.LockPattern.READ, journalContext)) { - mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), accessTime); - inodeId = lockedInodes.getInode().getId(); - } - - // verify inode attribute is not updated - assertNotEquals(accessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs()); - - // verify journal entry is not logged yet - verify(journalContext, never()).append(any(Journal.JournalEntry.class)); - - long newAccessTime = CommonUtils.getCurrentMs() + 2 * Constants.HOUR_MS; - - // update access time with a much later timestamp - try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), - InodeTree.LockPattern.READ, journalContext)) { - mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), newAccessTime); - inodeId = lockedInodes.getInode().getId(); - } - - // verify inode attribute is updated - assertEquals(newAccessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs()); - /// verify journal entry is logged - ArgumentCaptor captor = - ArgumentCaptor.forClass(Journal.JournalEntry.class); - verify(journalContext).append(captor.capture()); - assertTrue(captor.getValue().hasUpdateInode()); - assertEquals(inodeId, captor.getValue().getUpdateInode().getId()); - assertEquals(newAccessTime, captor.getValue().getUpdateInode().getLastAccessTimeMs()); - } - - @Test - public void updateAccessTimePrecisionAsync() throws Exception { - mAccessTimeUpdater = new AccessTimeUpdater(mFileSystemMaster, mInodeTree, - mContext.getJournalSystem(), Constants.MINUTE_MS, Constants.HOUR_MS, 0); - mAccessTimeUpdater.start(mScheduler); - String path = "/foo"; - createInode(path, CreateFileContext.defaults()); - JournalContext journalContext = mock(JournalContext.class); - when(journalContext.get()).thenReturn(journalContext); - when(mFileSystemMaster.createJournalContext()).thenReturn(journalContext); - long accessTime = CommonUtils.getCurrentMs() + 100L; - long inodeId; - try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), - InodeTree.LockPattern.READ, journalContext)) { - mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), accessTime); - inodeId = lockedInodes.getInode().getId(); - } - - mScheduler.jumpAndExecute(2, TimeUnit.MINUTES); - - // verify inode attribute is not updated - assertNotEquals(accessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs()); - - // verify journal entry is not logged - verify(journalContext, never()).append(any(Journal.JournalEntry.class)); - - long newAccessTime = CommonUtils.getCurrentMs() + 2 * Constants.HOUR_MS; - - // update access time with a much later timestamp - try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), - InodeTree.LockPattern.READ, journalContext)) { - mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), newAccessTime); - inodeId = lockedInodes.getInode().getId(); - } - - // verify inode attribute is updated - assertEquals(newAccessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs()); - - mScheduler.jumpAndExecute(2, TimeUnit.SECONDS); - - // verify journal entry is not logged - verify(journalContext, never()).append(any(Journal.JournalEntry.class)); - - mScheduler.jumpAndExecute(2, TimeUnit.MINUTES); - - /// verify journal entry is logged after the flush interval - ArgumentCaptor captor = - ArgumentCaptor.forClass(Journal.JournalEntry.class); - verify(journalContext).append(captor.capture()); - assertTrue(captor.getValue().hasUpdateInode()); - assertEquals(inodeId, captor.getValue().getUpdateInode().getId()); - assertEquals(newAccessTime, captor.getValue().getUpdateInode().getLastAccessTimeMs()); - } - - @Test - public void updateAccessTimeAsyncOnShutdown() throws Exception { - mAccessTimeUpdater = new AccessTimeUpdater(mFileSystemMaster, mInodeTree, - mContext.getJournalSystem(), 10 * Constants.SECOND_MS, 0, 0); - mAccessTimeUpdater.start(mScheduler); - String path = "/foo"; - createInode(path, CreateFileContext.defaults()); - JournalContext journalContext = mock(JournalContext.class); - when(journalContext.get()).thenReturn(journalContext); - when(mFileSystemMaster.createJournalContext()).thenReturn(journalContext); - long accessTime = CommonUtils.getCurrentMs() + 100L; - long inodeId; - try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), - InodeTree.LockPattern.READ, journalContext)) { - mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), accessTime); - inodeId = lockedInodes.getInode().getId(); - } - - // verify inode attribute is updated - assertEquals(accessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs()); - - mScheduler.jumpAndExecute(1, TimeUnit.SECONDS); - - // verify journal entry is NOT logged yet - verify(journalContext, never()).append(any(Journal.JournalEntry.class)); - - // wait for the flush to complete - mContext.getJournalSystem().stop(); - - /// verify journal entry is logged after the flush interval - ArgumentCaptor captor = - ArgumentCaptor.forClass(Journal.JournalEntry.class); - verify(journalContext).append(captor.capture()); - assertTrue(captor.getValue().hasUpdateInode()); - assertEquals(inodeId, captor.getValue().getUpdateInode().getId()); - assertEquals(accessTime, captor.getValue().getUpdateInode().getLastAccessTimeMs()); - } -} diff --git a/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterSyncMetadataTest.java b/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterSyncMetadataTest.java deleted file mode 100644 index fe4cf44cc9fc..000000000000 --- a/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterSyncMetadataTest.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; - -import alluxio.AlluxioURI; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.AccessControlException; -import alluxio.exception.FileAlreadyExistsException; -import alluxio.exception.FileDoesNotExistException; -import alluxio.exception.InvalidPathException; -import alluxio.file.options.DescendantType; -import alluxio.grpc.DeletePOptions; -import alluxio.grpc.FileSystemMasterCommonPOptions; -import alluxio.grpc.GetStatusPOptions; -import alluxio.grpc.ListStatusPOptions; -import alluxio.heartbeat.HeartbeatContext; -import alluxio.heartbeat.ManuallyScheduleHeartbeat; -import alluxio.master.CoreMasterContext; -import alluxio.master.MasterFactory; -import alluxio.master.MasterRegistry; -import alluxio.master.MasterTestUtils; -import alluxio.master.block.BlockMaster; -import alluxio.master.block.BlockMasterFactory; -import alluxio.master.file.contexts.CreateDirectoryContext; -import alluxio.master.file.contexts.DeleteContext; -import alluxio.master.file.contexts.GetStatusContext; -import alluxio.master.file.contexts.ListStatusContext; -import alluxio.master.file.contexts.MountContext; -import alluxio.master.file.meta.Inode; -import alluxio.master.file.meta.LockedInodePath; -import alluxio.master.journal.JournalSystem; -import alluxio.master.journal.JournalTestUtils; -import alluxio.master.journal.JournalType; -import alluxio.master.metrics.MetricsMasterFactory; -import alluxio.security.authentication.AuthenticatedClientUser; -import alluxio.security.user.UserState; -import alluxio.underfs.Fingerprint; -import alluxio.underfs.UfsDirectoryStatus; -import alluxio.underfs.UfsFileStatus; -import alluxio.underfs.UfsStatus; -import alluxio.underfs.UnderFileSystem; -import alluxio.util.IdUtils; -import alluxio.util.ModeUtils; -import alluxio.util.ThreadFactoryUtils; -import alluxio.util.executor.ExecutorServiceFactories; -import alluxio.util.executor.ExecutorServiceFactory; -import alluxio.util.io.PathUtils; -import alluxio.wire.FileInfo; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.File; -import java.io.IOException; -import java.time.Clock; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; -import javax.annotation.Nullable; - -/** - * Unit tests for {@link FileSystemMaster}. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({UnderFileSystem.Factory.class}) -public final class FileSystemMasterSyncMetadataTest { - private File mJournalFolder; - private MasterRegistry mRegistry; - private FileSystemMaster mFileSystemMaster; - private UnderFileSystem mUfs; - private ExecutorService mExecutorService; - - @Rule - public ManuallyScheduleHeartbeat mManualScheduler = - new ManuallyScheduleHeartbeat(HeartbeatContext.MASTER_PERSISTENCE_CHECKER, - HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER); - - @Before - public void before() throws Exception { - UserState s = UserState.Factory.create(Configuration.global()); - AuthenticatedClientUser.set(s.getUser().getName()); - TemporaryFolder tmpFolder = new TemporaryFolder(); - tmpFolder.create(); - File ufsRoot = tmpFolder.newFolder(); - Configuration.set(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS); - Configuration.set(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, ufsRoot.getAbsolutePath()); - Configuration.set(PropertyKey.MASTER_PERSISTENCE_INITIAL_INTERVAL_MS, 0); - Configuration.set(PropertyKey.MASTER_PERSISTENCE_MAX_INTERVAL_MS, 1000); - Configuration.set(PropertyKey.MASTER_PERSISTENCE_MAX_TOTAL_WAIT_TIME_MS, 1000); - mJournalFolder = tmpFolder.newFolder(); - startServices(); - } - - /** - * Resets global state after each test run. - */ - @After - public void after() throws Exception { - stopServices(); - } - - @Test - public void setAttributeOwnerGroupOnMetadataUpdate() throws Exception { - AlluxioURI ufsMount = setupMockUfsS3Mount(); - String fname = "file"; - AlluxioURI uri = new AlluxioURI("/mnt/local/" + fname); - short mode = ModeUtils.getUMask("0700").toShort(); - - // Mock dir1 ufs path - AlluxioURI filePath = ufsMount.join("file"); - UfsFileStatus fileStatus = new UfsFileStatus( - "file", "", 0L, System.currentTimeMillis(), - "owner1", "owner1", (short) 777, null, 100L); - Mockito.when(mUfs.getParsedFingerprint(filePath.toString())) - .thenReturn(Fingerprint.create("s3", fileStatus)); - Mockito.when(mUfs.exists(filePath.toString())).thenReturn(true); - Mockito.when(mUfs.isDirectory(filePath.toString())).thenReturn(false); - Mockito.when(mUfs.isFile(filePath.toString())).thenReturn(true); - Mockito.when(mUfs.getStatus(filePath.toString())).thenReturn(fileStatus); - - List f1 = mFileSystemMaster.listStatus(uri, ListStatusContext.mergeFrom( - ListStatusPOptions.newBuilder().setCommonOptions( - FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build()))); - UfsFileStatus updatedStatus = new UfsFileStatus( - "file", "", 0, System.currentTimeMillis(), - "owner2", "owner2", (short) 777, null, 100); - Mockito.when(mUfs.getStatus(filePath.toString())).thenReturn(updatedStatus); - Mockito.when(mUfs.getParsedFingerprint(filePath.toString())) - .thenReturn(Fingerprint.create("s3", updatedStatus)); - - FileInfo res = mFileSystemMaster.getFileInfo(uri, - GetStatusContext.mergeFrom(GetStatusPOptions.newBuilder().setCommonOptions( - FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build()))); - assertEquals("owner2", res.getOwner()); - assertEquals("owner2", res.getGroup()); - } - - @Test - public void listStatusWithSyncMetadataAndEmptyS3Owner() throws Exception { - - AlluxioURI ufsMount = setupMockUfsS3Mount(); - short mode = ModeUtils.getUMask("0700").toShort(); - - // Mock dir1 ufs path - AlluxioURI dir1Path = ufsMount.join("dir1"); - UfsDirectoryStatus dir1Status = new UfsDirectoryStatus(dir1Path.getPath(), "", "", mode); - Mockito.when(mUfs.getParsedFingerprint(dir1Path.toString())) - .thenReturn(Fingerprint.create("s3", dir1Status)); - Mockito.when(mUfs.exists(dir1Path.toString())).thenReturn(true); - Mockito.when(mUfs.isDirectory(dir1Path.toString())).thenReturn(true); - Mockito.when(mUfs.isFile(dir1Path.toString())).thenReturn(false); - Mockito.when(mUfs.getStatus(dir1Path.toString())).thenReturn(dir1Status); - Mockito.when(mUfs.getDirectoryStatus(dir1Path.toString())).thenReturn(dir1Status); - - // Mock nested ufs path - AlluxioURI nestedFilePath = ufsMount.join("dir1").join("file1"); - UfsFileStatus nestedFileStatus = new UfsFileStatus(nestedFilePath.getPath(), "dummy", 0, - null, "", "", mode, 1024); - Mockito.when(mUfs.getParsedFingerprint(nestedFilePath.toString())) - .thenReturn(Fingerprint.create("s3", nestedFileStatus)); - Mockito.when(mUfs.getStatus(nestedFilePath.toString())).thenReturn(nestedFileStatus); - Mockito.when(mUfs.isDirectory(nestedFilePath.toString())).thenReturn(false); - Mockito.when(mUfs.isFile(nestedFilePath.toString())).thenReturn(true); - Mockito.when(mUfs.getFileStatus(nestedFilePath.toString())).thenReturn(nestedFileStatus); - Mockito.when(mUfs.exists(nestedFilePath.toString())).thenReturn(true); - - // Create directory in Alluxio only - AlluxioURI dir1 = new AlluxioURI("/mnt/local/dir1"); - mFileSystemMaster.createDirectory(dir1, CreateDirectoryContext.defaults()); - - // Mock creating the same directory and nested file in UFS out of band - Mockito.when(mUfs.listStatus(eq(dir1Path.toString()))) - .thenReturn(new UfsStatus[]{new UfsFileStatus("file1", "dummy", 0, - null, "", "", mode, 1024)}); - - // List with sync.interval=0 - List fileInfoList = - mFileSystemMaster.listStatus(dir1, ListStatusContext.mergeFrom( - ListStatusPOptions.newBuilder().setCommonOptions( - FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build()))); - assertEquals(1, fileInfoList.size()); - - // Verify owner/group is not empty - FileInfo mountLocalInfo = - mFileSystemMaster.getFileInfo(new AlluxioURI("/mnt/local"), GetStatusContext.defaults()); - assertEquals(mountLocalInfo.getOwner(), - mFileSystemMaster.getFileInfo(dir1, GetStatusContext.defaults()).getOwner()); - assertEquals(mountLocalInfo.getGroup(), - mFileSystemMaster.getFileInfo(dir1, GetStatusContext.defaults()).getGroup()); - AlluxioURI file1 = new AlluxioURI("/mnt/local/dir1/file1"); - assertEquals(mountLocalInfo.getOwner(), - mFileSystemMaster.getFileInfo(file1, GetStatusContext.defaults()).getOwner()); - assertEquals(mountLocalInfo.getGroup(), - mFileSystemMaster.getFileInfo(file1, GetStatusContext.defaults()).getGroup()); - } - - private AlluxioURI setupMockUfsS3Mount() - throws IOException, FileDoesNotExistException, FileAlreadyExistsException, - AccessControlException, InvalidPathException { - mFileSystemMaster.createDirectory(new AlluxioURI("/mnt/"), CreateDirectoryContext.defaults()); - // Mock ufs mount - AlluxioURI ufsMount = new AlluxioURI("s3a://bucket/"); - Mockito.when(mUfs.getUnderFSType()).thenReturn("s3"); - Mockito.when(mUfs.isObjectStorage()).thenReturn(true); - Mockito.when(mUfs.isDirectory(ufsMount.toString())).thenReturn(true); - short mode = ModeUtils.getUMask("0700").toShort(); - Mockito.when(mUfs.getExistingDirectoryStatus(ufsMount.toString())) - .thenReturn(new UfsDirectoryStatus(ufsMount.toString(), "", "", mode)); - Mockito.when(mUfs.resolveUri(Mockito.eq(ufsMount), anyString())) - .thenAnswer(invocation -> new AlluxioURI(ufsMount, - PathUtils.concatPath(ufsMount.getPath(), - invocation.getArgument(1, String.class)), false)); - - // Mount - AlluxioURI mountLocal = new AlluxioURI("/mnt/local"); - mFileSystemMaster.mount(mountLocal, ufsMount, MountContext.defaults()); - - return ufsMount; - } - - @Test - public void deleteAlluxioOnlyNoSync() throws Exception { - // Prepare files - mFileSystemMaster.createDirectory(new AlluxioURI("/a/"), CreateDirectoryContext.defaults()); - mFileSystemMaster.createDirectory(new AlluxioURI("/a/b/"), CreateDirectoryContext.defaults()); - mFileSystemMaster.createDirectory(new AlluxioURI("/b/"), CreateDirectoryContext.defaults()); - // If the sync operation happens, the flag will be marked - SyncAwareFileSystemMaster delegateMaster = (SyncAwareFileSystemMaster) mFileSystemMaster; - delegateMaster.setSynced(false); - - delegateMaster.delete(new AlluxioURI("/a/"), - DeleteContext.mergeFrom(DeletePOptions.newBuilder() - .setRecursive(true).setAlluxioOnly(true))); - // The files have been deleted - assertEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(new AlluxioURI("/a/"))); - assertEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(new AlluxioURI("/a/b/"))); - // Irrelevant files are not affected - assertNotEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(new AlluxioURI("/b/"))); - // Sync has not happened - assertFalse(delegateMaster.mSynced.get()); - } - - private static class SyncAwareFileSystemMaster extends DefaultFileSystemMaster { - AtomicBoolean mSynced = new AtomicBoolean(false); - - public SyncAwareFileSystemMaster(BlockMaster blockMaster, CoreMasterContext masterContext, - ExecutorServiceFactory executorServiceFactory) { - super(blockMaster, masterContext, executorServiceFactory, Clock.systemUTC()); - } - - @Override - InodeSyncStream.SyncStatus syncMetadata(RpcContext rpcContext, AlluxioURI path, - FileSystemMasterCommonPOptions options, DescendantType syncDescendantType, - @Nullable FileSystemMasterAuditContext auditContext, - @Nullable Function auditContextSrcInodeFunc) - throws AccessControlException, InvalidPathException { - mSynced.set(true); - return super.syncMetadata(rpcContext, path, options, syncDescendantType, auditContext, - auditContextSrcInodeFunc); - } - - void setSynced(boolean synced) { - mSynced.set(synced); - } - } - - private class SyncAwareFileSystemMasterFactory implements MasterFactory { - @Override - public boolean isEnabled() { - return true; - } - - @Override - public String getName() { - return "SyncAwareFileSystemMasterFactory"; - } - - @Override - public FileSystemMaster create(MasterRegistry registry, CoreMasterContext context) { - BlockMaster blockMaster = registry.get(BlockMaster.class); - FileSystemMaster fileSystemMaster = new SyncAwareFileSystemMaster(blockMaster, context, - ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService)); - registry.add(FileSystemMaster.class, fileSystemMaster); - return fileSystemMaster; - } - } - - private void startServices() throws Exception { - mExecutorService = Executors - .newFixedThreadPool(4, ThreadFactoryUtils.build("DefaultFileSystemMasterTest-%d", true)); - mRegistry = new MasterRegistry(); - JournalSystem journalSystem = - JournalTestUtils.createJournalSystem(mJournalFolder.getAbsolutePath()); - CoreMasterContext context = MasterTestUtils.testMasterContext(journalSystem); - new MetricsMasterFactory().create(mRegistry, context); - new BlockMasterFactory().create(mRegistry, context); - mFileSystemMaster = new SyncAwareFileSystemMasterFactory().create(mRegistry, context); - journalSystem.start(); - journalSystem.gainPrimacy(); - mRegistry.start(true); - - mUfs = Mockito.mock(UnderFileSystem.class); - PowerMockito.mockStatic(UnderFileSystem.Factory.class); - Mockito.when(UnderFileSystem.Factory.createWithRecorder(anyString(), any(), any())) - .thenReturn(mUfs); - } - - private void stopServices() throws Exception { - mRegistry.stop(); - } -} diff --git a/core/server/master/src/test/java/alluxio/master/file/meta/TtlBucketListTest.java b/core/server/master/src/test/java/alluxio/master/file/meta/TtlBucketListTest.java deleted file mode 100644 index 2570719a13a3..000000000000 --- a/core/server/master/src/test/java/alluxio/master/file/meta/TtlBucketListTest.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file.meta; - -import static org.mockito.Mockito.mock; - -import alluxio.master.metastore.InodeStore; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import org.junit.Assert; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; - -import java.util.Collections; -import java.util.List; - -/** - * Unit tests for {@link TtlBucketList}. - */ -public final class TtlBucketListTest { - private static final long BUCKET_INTERVAL = 10; - private static final long BUCKET1_START = 0; - private static final long BUCKET1_END = BUCKET1_START + BUCKET_INTERVAL; - private static final long BUCKET2_START = BUCKET1_END; - private static final long BUCKET2_END = BUCKET2_START + BUCKET_INTERVAL; - private static final Inode BUCKET1_FILE1 = - TtlTestUtils.createFileWithIdAndTtl(0, BUCKET1_START); - private static final Inode BUCKET1_FILE2 = - TtlTestUtils.createFileWithIdAndTtl(1, BUCKET1_END - 1); - private static final Inode BUCKET2_FILE = - TtlTestUtils.createFileWithIdAndTtl(2, BUCKET2_START); - - private TtlBucketList mBucketList; - - @ClassRule - public static TtlIntervalRule sTtlIntervalRule = new TtlIntervalRule(BUCKET_INTERVAL); - - /** - * Sets up a new {@link TtlBucketList} before a test runs. - */ - @Before - public void before() { - mBucketList = new TtlBucketList(mock(InodeStore.class)); - } - - private List getSortedExpiredBuckets(long expireTime) { - List buckets = Lists.newArrayList(mBucketList.getExpiredBuckets(expireTime)); - Collections.sort(buckets); - return buckets; - } - - private void assertExpired(List expiredBuckets, int bucketIndex, - Inode... inodes) { - TtlBucket bucket = expiredBuckets.get(bucketIndex); - Assert.assertEquals(inodes.length, bucket.getInodes().size()); - Assert.assertTrue(bucket.getInodes().containsAll(Lists.newArrayList(inodes))); - } - - /** - * Tests the {@link TtlBucketList#insert(Inode)} method. - */ - @Test - public void insert() { - // No bucket should expire. - List expired = getSortedExpiredBuckets(BUCKET1_START); - Assert.assertTrue(expired.isEmpty()); - - mBucketList.insert(BUCKET1_FILE1); - // The first bucket should expire. - expired = getSortedExpiredBuckets(BUCKET1_END); - assertExpired(expired, 0, BUCKET1_FILE1); - - mBucketList.insert(BUCKET1_FILE2); - // Only the first bucket should expire. - for (long end = BUCKET2_START; end < BUCKET2_END; end++) { - expired = getSortedExpiredBuckets(end); - assertExpired(expired, 0, BUCKET1_FILE1, BUCKET1_FILE2); - } - - mBucketList.insert(BUCKET2_FILE); - // All buckets should expire. - expired = getSortedExpiredBuckets(BUCKET2_END); - assertExpired(expired, 0, BUCKET1_FILE1, BUCKET1_FILE2); - assertExpired(expired, 1, BUCKET2_FILE); - } - - /** - * Tests the {@link TtlBucketList#remove(InodeView)} method. - */ - @Test - public void remove() { - mBucketList.insert(BUCKET1_FILE1); - mBucketList.insert(BUCKET1_FILE2); - mBucketList.insert(BUCKET2_FILE); - - List expired = getSortedExpiredBuckets(BUCKET1_END); - assertExpired(expired, 0, BUCKET1_FILE1, BUCKET1_FILE2); - - mBucketList.remove(BUCKET1_FILE1); - expired = getSortedExpiredBuckets(BUCKET1_END); - // Only the first bucket should expire, and there should be only one BUCKET1_FILE2 in it. - assertExpired(expired, 0, BUCKET1_FILE2); - - mBucketList.remove(BUCKET1_FILE2); - expired = getSortedExpiredBuckets(BUCKET1_END); - // Only the first bucket should expire, and there should be no files in it. - assertExpired(expired, 0); // nothing in bucket 0. - - expired = getSortedExpiredBuckets(BUCKET2_END); - // All buckets should expire. - assertExpired(expired, 0); // nothing in bucket 0. - assertExpired(expired, 1, BUCKET2_FILE); - - // Remove bucket 0. - expired = getSortedExpiredBuckets(BUCKET1_END); - mBucketList.removeBuckets(Sets.newHashSet(expired)); - - expired = getSortedExpiredBuckets(BUCKET2_END); - // The only remaining bucket is bucket 1, it should expire. - assertExpired(expired, 0, BUCKET2_FILE); - - mBucketList.remove(BUCKET2_FILE); - expired = getSortedExpiredBuckets(BUCKET2_END); - assertExpired(expired, 0); // nothing in bucket. - - mBucketList.removeBuckets(Sets.newHashSet(expired)); - // No bucket should exist now. - expired = getSortedExpiredBuckets(BUCKET2_END); - Assert.assertEquals(0, expired.size()); - } -} diff --git a/core/server/master/src/test/java/alluxio/master/file/replication/ReplicationCheckerTest.java b/core/server/master/src/test/java/alluxio/master/file/replication/ReplicationCheckerTest.java deleted file mode 100644 index 45c4db8333d1..000000000000 --- a/core/server/master/src/test/java/alluxio/master/file/replication/ReplicationCheckerTest.java +++ /dev/null @@ -1,479 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.file.replication; - -import static org.mockito.Mockito.mock; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.CreateFilePOptions; -import alluxio.grpc.RegisterWorkerPOptions; -import alluxio.grpc.StorageList; -import alluxio.job.plan.replicate.ReplicationHandler; -import alluxio.job.wire.Status; -import alluxio.master.CoreMasterContext; -import alluxio.master.MasterRegistry; -import alluxio.master.MasterTestUtils; -import alluxio.master.block.BlockMaster; -import alluxio.master.block.BlockMasterFactory; -import alluxio.master.file.RpcContext; -import alluxio.master.file.contexts.CreateFileContext; -import alluxio.master.file.contexts.CreatePathContext; -import alluxio.master.file.meta.Inode; -import alluxio.master.file.meta.InodeDirectoryIdGenerator; -import alluxio.master.file.meta.InodeLockManager; -import alluxio.master.file.meta.InodeTree; -import alluxio.master.file.meta.InodeTree.LockPattern; -import alluxio.master.file.meta.LockedInodePath; -import alluxio.master.file.meta.MountTable; -import alluxio.master.file.meta.MutableInodeFile; -import alluxio.master.file.meta.options.MountInfo; -import alluxio.master.journal.JournalSystem; -import alluxio.master.journal.JournalTestUtils; -import alluxio.master.journal.JournalType; -import alluxio.master.journal.NoopJournalContext; -import alluxio.master.metastore.InodeStore; -import alluxio.master.metrics.MetricsMasterFactory; -import alluxio.metrics.Metric; -import alluxio.proto.meta.Block; -import alluxio.security.authorization.Mode; -import alluxio.underfs.UfsManager; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.time.Clock; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Unit tests for {@link ReplicationChecker}. - */ -public final class ReplicationCheckerTest { - private static final String TEST_OWNER = "user1"; - private static final String TEST_GROUP = ""; - private static final Mode TEST_MODE = new Mode((short) 0755); - private static final AlluxioURI TEST_FILE_1 = new AlluxioURI("/test1"); - private static final AlluxioURI TEST_FILE_2 = new AlluxioURI("/test2"); - private static final AlluxioURI TEST_FILE_3 = new AlluxioURI("/test3"); - private static final List NO_BLOCKS = ImmutableList.of(); - private static final List NO_METRICS = ImmutableList.of(); - private static final Map> NO_BLOCKS_ON_LOCATION - = ImmutableMap.of(); - private static final Map NO_LOST_STORAGE = ImmutableMap.of(); - private static final Map EMPTY = ImmutableMap.of(); - - /** - * A mock class of AdjustReplicationHandler, used to test the output of ReplicationChecker. - */ - @ThreadSafe - private static class MockHandler implements ReplicationHandler { - private final Map mSetReplicaRequests = Maps.newHashMap(); - private final Map> - mMigrateRequests = Maps.newHashMap(); - private final List mJobStatusRequests = Lists.newArrayList(); - private final Map mJobStatus = Maps.newHashMap(); - - private long mNextJobId = 0; - - public void setJobStatus(long jobId, Status status) { - mJobStatus.put(jobId, status); - } - - @Override - public Status getJobStatus(long jobId) throws IOException { - mJobStatusRequests.add(jobId); - if (mJobStatus.containsKey(jobId)) { - return mJobStatus.get(jobId); - } - return Status.RUNNING; - } - - @Override - public List findJobs(String jobName, Set statusList) throws IOException { - return mJobStatus.entrySet().stream() - .filter(x -> statusList.isEmpty() || statusList.contains(x.getValue())) - .map(x -> x.getKey()).collect(Collectors.toList()); - } - - @Override - public long setReplica(AlluxioURI uri, long blockId, int numReplicas) { - mSetReplicaRequests.put(blockId, numReplicas); - return ++mNextJobId; - } - - @Override - public long migrate(AlluxioURI uri, long blockId, String workerHost, String mediumType) { - mMigrateRequests.put(blockId, new Pair<>(workerHost, mediumType)); - return ++mNextJobId; - } - - public Map getSetReplicaRequests() { - return mSetReplicaRequests; - } - - public Map> getMigrateRequests() { - return mMigrateRequests; - } - } - - private CoreMasterContext mContext; - private InodeStore mInodeStore; - private InodeTree mInodeTree; - private BlockMaster mBlockMaster; - private ReplicationChecker mReplicationChecker; - private MockHandler mMockReplicationHandler; - private CreateFileContext mFileContext = - CreateFileContext.mergeFrom(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) - .setMode(TEST_MODE.toProto())).setOwner(TEST_OWNER).setGroup(TEST_GROUP); - private Set mKnownWorkers = Sets.newHashSet(); - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - @Before - public void before() throws Exception { - Configuration.set(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS); - MasterRegistry registry = new MasterRegistry(); - JournalSystem journalSystem = JournalTestUtils.createJournalSystem(mTestFolder); - mContext = MasterTestUtils.testMasterContext(journalSystem); - new MetricsMasterFactory().create(registry, mContext); - mBlockMaster = new BlockMasterFactory().create(registry, mContext); - InodeDirectoryIdGenerator directoryIdGenerator = new InodeDirectoryIdGenerator(mBlockMaster); - UfsManager manager = mock(UfsManager.class); - MountTable mountTable = new MountTable(manager, mock(MountInfo.class), Clock.systemUTC()); - InodeLockManager lockManager = new InodeLockManager(); - mInodeStore = mContext.getInodeStoreFactory().apply(lockManager); - mInodeTree = - new InodeTree(mInodeStore, mBlockMaster, directoryIdGenerator, mountTable, lockManager); - - journalSystem.start(); - journalSystem.gainPrimacy(); - mBlockMaster.start(true); - - Configuration.set(PropertyKey.TEST_MODE, true); - Configuration.set(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED, true); - Configuration - .set(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_SUPERGROUP, "test-supergroup"); - mInodeTree.initializeRoot(TEST_OWNER, TEST_GROUP, TEST_MODE, NoopJournalContext.INSTANCE); - - mMockReplicationHandler = new MockHandler(); - mReplicationChecker = new ReplicationChecker(mInodeTree, mBlockMaster, - mContext.getSafeModeManager(), mMockReplicationHandler); - } - - @After - public void after() { - Configuration.reloadProperties(); - } - - /** - * Helper to create a file with a single block. - * - * @param path Alluxio path of the file - * @param context context to create the file - * @param pinLocation - * @return the block ID - */ - private long createBlockHelper(AlluxioURI path, CreatePathContext context, - String pinLocation) throws Exception { - try (LockedInodePath inodePath = mInodeTree.lockInodePath( - path, LockPattern.WRITE_EDGE, NoopJournalContext.INSTANCE) - ) { - List created = mInodeTree.createPath(RpcContext.NOOP, inodePath, context); - if (!pinLocation.equals("")) { - mInodeTree.setPinned(RpcContext.NOOP, inodePath, true, ImmutableList.of(pinLocation), 0); - } - MutableInodeFile inodeFile = mInodeStore.getMutable(created.get(0).getId()).get().asFile(); - inodeFile.setBlockSizeBytes(1); - inodeFile.setBlockIds(Arrays.asList(inodeFile.getNewBlockId())); - inodeFile.setCompleted(true); - mInodeStore.writeInode(inodeFile); - return inodeFile.getBlockIdByIndex(0); - } - } - - /** - * Helper to create and add a given number of locations for block ID. - * - * @param blockId ID of the block to add location - * @param numLocations number of locations to add - */ - private void addBlockLocationHelper(long blockId, int numLocations) throws Exception { - // Commit blockId to the first worker. - mBlockMaster.commitBlock(createWorkerHelper(0), 50L, - Constants.MEDIUM_MEM, Constants.MEDIUM_MEM, blockId, 20L); - - // Send a heartbeat from other workers saying that it's added blockId. - for (int i = 1; i < numLocations; i++) { - heartbeatToAddLocationHelper(blockId, createWorkerHelper(i)); - } - } - - /** - * Helper to register a new worker. - * - * @param workerIndex the index of the worker in all workers - * @return the created worker ID - */ - private long createWorkerHelper(int workerIndex) throws Exception { - WorkerNetAddress address = new WorkerNetAddress().setHost("host" + workerIndex).setRpcPort(1000) - .setDataPort(2000).setWebPort(3000); - long workerId = mBlockMaster.getWorkerId(address); - if (!mKnownWorkers.contains(workerId)) { - // Do not re-register works, otherwise added block will be removed - mBlockMaster.workerRegister(workerId, ImmutableList.of(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, - RegisterWorkerPOptions.getDefaultInstance()); - mKnownWorkers.add(workerId); - } - return workerId; - } - - /** - * Helper to heartbeat to a worker and report a newly added block. - * - * @param blockId ID of the block to add location - * @param workerId ID of the worker to heartbeat - */ - private void heartbeatToAddLocationHelper(long blockId, long workerId) throws Exception { - List addedBlocks = ImmutableList.of(blockId); - Block.BlockLocation blockLocation = - Block.BlockLocation.newBuilder().setWorkerId(workerId) - .setTier(Constants.MEDIUM_MEM).setMediumType(Constants.MEDIUM_MEM).build(); - - mBlockMaster.workerHeartbeat(workerId, null, - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS, - ImmutableMap.of(blockLocation, addedBlocks), NO_LOST_STORAGE, NO_METRICS); - } - - @Test - public void heartbeatWhenTreeIsEmpty() throws Exception { - mReplicationChecker.heartbeat(); - Assert.assertEquals(EMPTY, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatFileWithinRange() throws Exception { - mFileContext.getOptions().setReplicationMin(1).setReplicationMax(3); - long blockId = - createBlockHelper(TEST_FILE_1, mFileContext, ""); - // One replica, meeting replication min - addBlockLocationHelper(blockId, 1); - mReplicationChecker.heartbeat(); - Assert.assertEquals(EMPTY, mMockReplicationHandler.getSetReplicaRequests()); - - // Two replicas, good - heartbeatToAddLocationHelper(blockId, createWorkerHelper(1)); - mReplicationChecker.heartbeat(); - Assert.assertEquals(EMPTY, mMockReplicationHandler.getSetReplicaRequests()); - - // Three replicas, meeting replication max, still good - heartbeatToAddLocationHelper(blockId, createWorkerHelper(2)); - mReplicationChecker.heartbeat(); - Assert.assertEquals(EMPTY, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatFileUnderReplicatedBy1() throws Exception { - mFileContext.getOptions().setReplicationMin(1); - long blockId = createBlockHelper(TEST_FILE_1, mFileContext, ""); - - mReplicationChecker.heartbeat(); - Map expected = ImmutableMap.of(blockId, 1); - Assert.assertEquals(expected, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatFileNeedsMove() throws Exception { - mFileContext.getOptions().setReplicationMin(1); - long blockId = createBlockHelper(TEST_FILE_1, mFileContext, Constants.MEDIUM_SSD); - addBlockLocationHelper(blockId, 1); - - mReplicationChecker.heartbeat(); - Map> expected = - ImmutableMap.of(blockId, new Pair<>("host0", Constants.MEDIUM_SSD)); - Assert.assertEquals(EMPTY, mMockReplicationHandler.getSetReplicaRequests()); - Assert.assertEquals(expected, mMockReplicationHandler.getMigrateRequests()); - } - - @Test - public void heartbeatFileDoesnotNeedMove() throws Exception { - mFileContext.getOptions().setReplicationMin(1); - long blockId = createBlockHelper(TEST_FILE_1, mFileContext, Constants.MEDIUM_MEM); - addBlockLocationHelper(blockId, 1); - - mReplicationChecker.heartbeat(); - Assert.assertEquals(EMPTY, mMockReplicationHandler.getSetReplicaRequests()); - Assert.assertEquals(EMPTY, mMockReplicationHandler.getMigrateRequests()); - } - - @Test - public void heartbeatFileUnderReplicatedBy10() throws Exception { - mFileContext.getOptions().setReplicationMin(10); - long blockId = createBlockHelper(TEST_FILE_1, mFileContext, ""); - - mReplicationChecker.heartbeat(); - Map expected = ImmutableMap.of(blockId, 10); - Assert.assertEquals(expected, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatMultipleFilesUnderReplicated() throws Exception { - mFileContext.getOptions().setReplicationMin(1); - long blockId1 = createBlockHelper(TEST_FILE_1, mFileContext, ""); - mFileContext.getOptions().setReplicationMin(2); - long blockId2 = createBlockHelper(TEST_FILE_2, mFileContext, ""); - - mReplicationChecker.heartbeat(); - Map expected = ImmutableMap.of(blockId1, 1, blockId2, 2); - Assert.assertEquals(expected, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatFileUnderReplicatedAndLost() throws Exception { - mFileContext.getOptions().setReplicationMin(2); - long blockId = createBlockHelper(TEST_FILE_1, mFileContext, ""); - - // Create a worker. - long workerId = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("localhost") - .setRpcPort(80).setDataPort(81).setWebPort(82)); - mBlockMaster.workerRegister(workerId, Collections.singletonList(Constants.MEDIUM_MEM), - ImmutableMap.of(Constants.MEDIUM_MEM, 100L), - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, - NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance()); - mBlockMaster.commitBlock(workerId, 50L, - Constants.MEDIUM_MEM, Constants.MEDIUM_MEM, blockId, 20L); - - // Indicate that blockId is removed on the worker. - mBlockMaster.workerHeartbeat(workerId, null, - ImmutableMap.of(Constants.MEDIUM_MEM, 0L), - ImmutableList.of(blockId), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, NO_METRICS); - - mReplicationChecker.heartbeat(); - Assert.assertEquals(EMPTY, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatFileOverReplicatedBy1() throws Exception { - mFileContext.getOptions().setReplicationMax(1); - long blockId = createBlockHelper(TEST_FILE_1, mFileContext, ""); - addBlockLocationHelper(blockId, 2); - - mReplicationChecker.heartbeat(); - Map expected = ImmutableMap.of(blockId, 1); - Assert.assertEquals(expected, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatFileOverReplicatedBy10() throws Exception { - mFileContext.getOptions().setReplicationMax(1); - long blockId = createBlockHelper(TEST_FILE_1, mFileContext, ""); - addBlockLocationHelper(blockId, 11); - - mReplicationChecker.heartbeat(); - Map expected = ImmutableMap.of(blockId, 1); - Assert.assertEquals(expected, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatMultipleFilesOverReplicated() throws Exception { - mFileContext.getOptions().setReplicationMax(1); - long blockId1 = createBlockHelper(TEST_FILE_1, mFileContext, ""); - mFileContext.getOptions().setReplicationMax(2); - long blockId2 = createBlockHelper(TEST_FILE_2, mFileContext, ""); - addBlockLocationHelper(blockId1, 2); - addBlockLocationHelper(blockId2, 4); - - mReplicationChecker.heartbeat(); - Map expected = ImmutableMap.of(blockId1, 1, blockId2, 2); - Assert.assertEquals(expected, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatFilesUnderAndOverReplicated() throws Exception { - mFileContext.getOptions().setReplicationMin(2).setReplicationMax(-1); - long blockId1 = createBlockHelper(TEST_FILE_1, mFileContext, ""); - mFileContext.getOptions().setReplicationMin(0).setReplicationMax(3); - long blockId2 = createBlockHelper(TEST_FILE_2, mFileContext, ""); - addBlockLocationHelper(blockId1, 1); - addBlockLocationHelper(blockId2, 5); - - mReplicationChecker.heartbeat(); - Map expected1 = ImmutableMap.of(blockId1, 2, blockId2, 3); - Assert.assertEquals(expected1, mMockReplicationHandler.getSetReplicaRequests()); - } - - @Test - public void heartbeatPartial() throws Exception { - Configuration.set(PropertyKey.JOB_MASTER_JOB_CAPACITY, 20); - mReplicationChecker = new ReplicationChecker(mInodeTree, mBlockMaster, - mContext.getSafeModeManager(), mMockReplicationHandler); - mFileContext.getOptions().setReplicationMin(3).setReplicationMax(-1); - long blockId1 = createBlockHelper(TEST_FILE_1, mFileContext, ""); - long blockId2 = createBlockHelper(TEST_FILE_2, mFileContext, ""); - long blockId3 = createBlockHelper(TEST_FILE_3, mFileContext, ""); - addBlockLocationHelper(blockId1, 1); - addBlockLocationHelper(blockId2, 1); - addBlockLocationHelper(blockId3, 1); - - mReplicationChecker.heartbeat(); - final Map replicateRequests = mMockReplicationHandler.getSetReplicaRequests(); - System.out.println(replicateRequests); - Assert.assertEquals(2, replicateRequests.size()); - Assert.assertEquals(3, replicateRequests.values().toArray()[0]); - Assert.assertEquals(3, replicateRequests.values().toArray()[1]); - replicateRequests.clear(); - - mMockReplicationHandler.setJobStatus(1, Status.RUNNING); - mMockReplicationHandler.setJobStatus(2, Status.RUNNING); - mReplicationChecker.heartbeat(); - Assert.assertEquals(0, replicateRequests.size()); - - mMockReplicationHandler.setJobStatus(1, Status.FAILED); - mReplicationChecker.heartbeat(); - Assert.assertEquals(1, replicateRequests.size()); - Assert.assertEquals(3, replicateRequests.values().toArray()[0]); - - replicateRequests.clear(); - addBlockLocationHelper(blockId1, 3); - addBlockLocationHelper(blockId2, 3); - mMockReplicationHandler.setJobStatus(2, Status.COMPLETED); - mMockReplicationHandler.setJobStatus(3, Status.COMPLETED); - - mReplicationChecker.heartbeat(); - Assert.assertEquals(1, replicateRequests.size()); - Assert.assertTrue(replicateRequests.containsKey(blockId3)); - Assert.assertEquals(3, replicateRequests.values().toArray()[0]); - } -} diff --git a/core/server/master/src/test/java/alluxio/master/journal/JournalTestUtils.java b/core/server/master/src/test/java/alluxio/master/journal/JournalTestUtils.java deleted file mode 100644 index 8cadd7e84a7d..000000000000 --- a/core/server/master/src/test/java/alluxio/master/journal/JournalTestUtils.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal; - -import alluxio.util.CommonUtils.ProcessType; - -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -/** - * Utility methods for testing against a journal system. - */ -public class JournalTestUtils { - - public static JournalSystem createJournalSystem(TemporaryFolder folder) { - try { - return createJournalSystem(folder.newFolder("journal").getAbsolutePath()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public static JournalSystem createJournalSystem(String folder) { - try { - return new JournalSystem.Builder() - .setLocation(new URI(folder)) - .setQuietTimeMs(0) - .build(ProcessType.MASTER); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } -} diff --git a/core/server/master/src/test/java/alluxio/master/journal/raft/SnapshotReplicationManagerTest.java b/core/server/master/src/test/java/alluxio/master/journal/raft/SnapshotReplicationManagerTest.java deleted file mode 100644 index 02a90fec31cc..000000000000 --- a/core/server/master/src/test/java/alluxio/master/journal/raft/SnapshotReplicationManagerTest.java +++ /dev/null @@ -1,484 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.journal.raft; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.argThat; - -import alluxio.ConfigurationRule; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.JournalQueryRequest; -import alluxio.grpc.NetAddress; -import alluxio.grpc.QuorumServerInfo; -import alluxio.grpc.RaftJournalServiceGrpc; -import alluxio.grpc.SnapshotData; -import alluxio.grpc.UploadSnapshotPRequest; -import alluxio.grpc.UploadSnapshotPResponse; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; -import alluxio.util.io.BufferUtils; - -import io.grpc.ManagedChannel; -import io.grpc.Server; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.stub.StreamObserver; -import net.bytebuddy.utility.RandomString; -import org.apache.commons.io.FileUtils; -import org.apache.ratis.protocol.Message; -import org.apache.ratis.protocol.RaftClientReply; -import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.server.RaftServerConfigKeys; -import org.apache.ratis.server.protocol.TermIndex; -import org.apache.ratis.server.storage.RaftStorage; -import org.apache.ratis.server.storage.StorageImplUtils; -import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage; -import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo; -import org.junit.After; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; -import org.mockito.stubbing.Answer; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -public class SnapshotReplicationManagerTest { - private static final int SNAPSHOT_SIZE = 100_000; - private static final int DEFAULT_SNAPSHOT_TERM = 0; - private static final int DEFAULT_SNAPSHOT_INDEX = 1; - - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - @Rule - public ConfigurationRule mConfigurationRule = - new ConfigurationRule(PropertyKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_REPLICATION_CHUNK_SIZE, - "32KB", Configuration.modifiableGlobal()); - - private final WaitForOptions mWaitOptions = WaitForOptions.defaults().setTimeoutMs(30_000); - private SnapshotReplicationManager mLeaderSnapshotManager; - private RaftJournalSystem mLeader; - private SimpleStateMachineStorage mLeaderStore; - private final Map mFollowers = new HashMap<>(); - - private RaftJournalServiceClient mClient; - private Server mServer; - - private void before(int numFollowers) throws Exception { - Configuration.set(PropertyKey.MASTER_JOURNAL_REQUEST_INFO_TIMEOUT, 550); - Configuration.set(PropertyKey.MASTER_JOURNAL_REQUEST_DATA_TIMEOUT, 550); - mLeader = Mockito.mock(RaftJournalSystem.class); - Mockito.when(mLeader.isLeader()).thenReturn(true); - Mockito.when(mLeader.getLocalPeerId()).thenReturn(RaftPeerId.getRaftPeerId("leader")); - mLeaderStore = getSimpleStateMachineStorage(); - mLeaderSnapshotManager = Mockito.spy(new SnapshotReplicationManager(mLeader, mLeaderStore)); - - String serverName = InProcessServerBuilder.generateName(); - mServer = InProcessServerBuilder.forName(serverName) - .directExecutor() - .addService(new RaftJournalServiceHandler(mLeaderSnapshotManager)).build(); - mServer.start(); - ManagedChannel channel = InProcessChannelBuilder.forName(serverName).directExecutor().build(); - RaftJournalServiceGrpc.RaftJournalServiceStub stub = RaftJournalServiceGrpc.newStub(channel); - // mock RaftJournalServiceClient - mClient = Mockito.mock(RaftJournalServiceClient.class); - Mockito.doNothing().when(mClient).close(); - // download rpc mock - Mockito.when(mClient.downloadSnapshot(any())).thenAnswer((args) -> { - StreamObserver responseObserver = args.getArgument(0, StreamObserver.class); - return stub.downloadSnapshot(responseObserver); - }); - // upload rpc mock - Mockito.when(mClient.uploadSnapshot(any())).thenAnswer((args) -> { - StreamObserver responseObserver = args.getArgument(0, StreamObserver.class); - return stub.uploadSnapshot(responseObserver); - }); - Mockito.doReturn(mClient).when(mLeaderSnapshotManager).createJournalServiceClient(); - - for (int i = 0; i < numFollowers; i++) { - Follower follower = new Follower(mClient); - mFollowers.put(follower.getRaftPeerId(), follower); - } - - List quorumServerInfos = mFollowers.values().stream().map(follower -> { - return QuorumServerInfo.newBuilder().setServerAddress( - NetAddress.newBuilder().setHost(follower.mHost).setRpcPort(follower.mRpcPort)).build(); - }).collect(Collectors.toList()); - - Mockito.when(mLeader.getQuorumServerInfoList()).thenReturn(quorumServerInfos); - Answer fn = (args) -> { - RaftPeerId peerId = args.getArgument(0, RaftPeerId.class); - Message message = args.getArgument(1, Message.class); - JournalQueryRequest queryRequest = JournalQueryRequest.parseFrom( - message.getContent().asReadOnlyByteBuffer()); - return CompletableFuture.supplyAsync(() -> { - CompletableFuture fut = CompletableFuture.supplyAsync(() -> { - Message response; - try { - response = mFollowers.get(peerId).mSnapshotManager.handleRequest(queryRequest); - } catch (IOException e) { - throw new CompletionException(e); - } - RaftClientReply reply = Mockito.mock(RaftClientReply.class); - Mockito.when(reply.getMessage()).thenReturn(response); - return reply; - }); - RaftClientReply result; - try { - if (args.getArguments().length == 3) { - result = fut.get(args.getArgument(2), TimeUnit.MILLISECONDS); - } else { - result = fut.get(); - } - return result; - } catch (Exception e) { - throw new CompletionException(e); - } - }); - }; - Mockito.when(mLeader.sendMessageAsync(any(), any())).thenAnswer(fn); - Mockito.when(mLeader.sendMessageAsync(any(), any(), anyLong())).thenAnswer(fn); - } - - private SimpleStateMachineStorage getSimpleStateMachineStorage() throws IOException { - RaftStorage rs = StorageImplUtils.newRaftStorage( - mFolder.newFolder(CommonUtils.randomAlphaNumString(6)), - RaftServerConfigKeys.Log.CorruptionPolicy.getDefault(), - RaftStorage.StartupOption.FORMAT, - RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT.getSize()); - rs.initialize(); - SimpleStateMachineStorage snapshotStore = new SimpleStateMachineStorage(); - snapshotStore.init(rs); - return snapshotStore; - } - - private void createSnapshotFile(SimpleStateMachineStorage storage) throws IOException { - createSnapshotFile(storage, DEFAULT_SNAPSHOT_TERM, DEFAULT_SNAPSHOT_INDEX); - } - - private void createSnapshotFile(SimpleStateMachineStorage storage, long term, long index) - throws IOException { - java.io.File file = storage.getSnapshotFile(term, index); - FileUtils.writeByteArrayToFile(file, BufferUtils.getIncreasingByteArray(SNAPSHOT_SIZE)); - storage.loadLatestSnapshot(); - } - - private void validateSnapshotFile(SimpleStateMachineStorage storage) throws IOException { - validateSnapshotFile(storage, DEFAULT_SNAPSHOT_TERM, DEFAULT_SNAPSHOT_INDEX); - } - - private void validateSnapshotFile(SimpleStateMachineStorage storage, long term, long index) - throws IOException { - SingleFileSnapshotInfo snapshot = storage.getLatestSnapshot(); - Assert.assertNotNull(snapshot); - Assert.assertEquals(TermIndex.valueOf(term, index), snapshot.getTermIndex()); - byte[] received = FileUtils.readFileToByteArray(snapshot.getFiles().get(0).getPath().toFile()); - Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SNAPSHOT_SIZE, received)); - } - - @After - public void After() throws Exception { - mServer.shutdown(); - mServer.awaitTermination(); - } - - @Test - public void copySnapshotToLeader() throws Exception { - before(1); - Follower follower = mFollowers.values().stream().findFirst().get(); - createSnapshotFile(follower.mStore); - - Assert.assertNull(mLeaderStore.getLatestSnapshot()); - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - - CommonUtils.waitFor("leader snapshot to complete", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower() != -1, mWaitOptions); - validateSnapshotFile(mLeaderStore); - } - - @Test - public void copySnapshotToFollower() throws Exception { - before(1); - createSnapshotFile(mLeaderStore); - - Follower follower = mFollowers.values().stream().findFirst().get(); - Assert.assertNull(follower.mStore.getLatestSnapshot()); - - follower.mSnapshotManager.installSnapshotFromLeader(); - - CommonUtils.waitFor("follower snapshot to complete", - () -> follower.mStore.getLatestSnapshot() != null, mWaitOptions); - validateSnapshotFile(follower.mStore); - } - - @Test - public void requestSnapshotEqualTermHigherIndex() throws Exception { - before(2); - List followers = new ArrayList<>(mFollowers.values()); - Follower firstFollower = followers.get(0); - Follower secondFollower = followers.get(1); - - createSnapshotFile(firstFollower.mStore); // create default 0, 1 snapshot - createSnapshotFile(secondFollower.mStore, 0, 2); // preferable to the default 0, 1 snapshot - - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - - CommonUtils.waitFor("leader snapshot to complete", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower() != -1, mWaitOptions); - // verify that the leader still requests and gets the best snapshot - validateSnapshotFile(mLeaderStore, 0, 2); - } - - @Test - public void failGetInfoEqualTermHigherIndex() throws Exception { - before(2); - List followers = new ArrayList<>(mFollowers.values()); - Follower firstFollower = followers.get(0); - Follower secondFollower = followers.get(1); - - createSnapshotFile(firstFollower.mStore); // create default 0, 1 snapshot - createSnapshotFile(secondFollower.mStore, 0, 2); // preferable to the default 0, 1 snapshot - // the second follower will not reply to the getInfo request, so the leader will request from - // the first after a timeout - secondFollower.disableGetInfo(); - - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - - CommonUtils.waitFor("leader snapshot to complete", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower() != -1, mWaitOptions); - // verify that the leader still requests and get the snapshot from the first follower - validateSnapshotFile(mLeaderStore, 0, 1); - } - - @Test - public void failSnapshotRequestEqualTermHigherIndex() throws Exception { - before(2); - List followers = new ArrayList<>(mFollowers.values()); - Follower firstFollower = followers.get(0); - Follower secondFollower = followers.get(1); - - createSnapshotFile(firstFollower.mStore); // create default 0, 1 snapshot - createSnapshotFile(secondFollower.mStore, 0, 2); // preferable to the default 0, 1 snapshot - // the second follower will not start the snapshot upload, so the leader will request from the - // first after a timeout - secondFollower.disableFollowerUpload(); - - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - - CommonUtils.waitFor("leader snapshot to complete", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower() != -1, mWaitOptions); - // verify that the leader still requests and get the snapshot from the first follower - validateSnapshotFile(mLeaderStore, 0, 1); - } - - @Test - public void failFailThenSuccess() throws Exception { - before(3); - List followers = new ArrayList<>(mFollowers.values()); - Follower firstFollower = followers.get(0); - Follower secondFollower = followers.get(1); - - createSnapshotFile(firstFollower.mStore, 0, 1); - createSnapshotFile(secondFollower.mStore, 0, 1); - - firstFollower.disableFollowerUpload(); - secondFollower.disableGetInfo(); - - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - - try { - CommonUtils.waitForResult("upload failure", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower(), - (num) -> num == 1, - WaitForOptions.defaults().setInterval(10).setTimeoutMs(100)); - } catch (Exception e) { - // expected to fail: no snapshot could be uploaded - } - - Follower thirdFollower = followers.get(2); - createSnapshotFile(thirdFollower.mStore, 0, 2); - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - CommonUtils.waitForResult("upload failure", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower(), - (num) -> num == 2, mWaitOptions); - validateSnapshotFile(mLeaderStore, 0, 2); - } - - @Test - public void requestSnapshotHigherTermLowerIndex() throws Exception { - before(2); - List followers = new ArrayList<>(mFollowers.values()); - Follower firstFollower = followers.get(0); - Follower secondFollower = followers.get(1); - - createSnapshotFile(firstFollower.mStore, 1, 10); - createSnapshotFile(secondFollower.mStore, 2, 1); - - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - - CommonUtils.waitFor("leader snapshot to complete", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower() != -1, mWaitOptions); - // verify that the leader still requests and gets the best snapshot - validateSnapshotFile(mLeaderStore, 2, 1); - } - - @Test - public void installSnapshotsInSuccession() throws Exception { - before(2); - List followers = new ArrayList<>(mFollowers.values()); - Follower firstFollower = followers.get(0); - Follower secondFollower = followers.get(1); - - createSnapshotFile(firstFollower.mStore); // create default 0, 1 snapshot - - for (int i = 2; i < 12; i++) { - if (i % 2 == 0) { - createSnapshotFile(secondFollower.mStore, 0, i); - secondFollower.notifySnapshotInstalled(); - } else { - createSnapshotFile(firstFollower.mStore, 0, i); - firstFollower.notifySnapshotInstalled(); - } - CommonUtils.waitFor("leader snapshot to complete", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower() != -1, mWaitOptions); - validateSnapshotFile(mLeaderStore, 0, i); - } - } - - /** - * Simulates a {@link SnapshotDownloader} error. - */ - @Test - public void downloadFailure() throws Exception { - before(2); - List followers = new ArrayList<>(mFollowers.values()); - Follower firstFollower = followers.get(0); - Follower secondFollower = followers.get(1); - - createSnapshotFile(firstFollower.mStore); // create default 0, 1 snapshot - createSnapshotFile(secondFollower.mStore, 0, 2); // preferable to the default 0, 1 snapshot - - // make sure to error out when requesting the better snapshot from secondFollower - Mockito.doAnswer(mock -> { - SingleFileSnapshotInfo snapshot = secondFollower.mStore.getLatestSnapshot(); - StreamObserver responseObserver = - SnapshotUploader.forFollower(secondFollower.mStore, snapshot); - StreamObserver requestObserver = mClient - .uploadSnapshot(responseObserver); - requestObserver.onError(new IOException("failed snapshot upload")); - return null; - }).when(secondFollower.mSnapshotManager).sendSnapshotToLeader(); - - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - - CommonUtils.waitFor("leader snapshot to complete", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower() != -1, mWaitOptions); - // verify that the leader still requests and gets second best snapshot - validateSnapshotFile(mLeaderStore); - } - - /** - * Simulates a {@link SnapshotUploader} error. - */ - @Test - public void uploadFailure() throws Exception { - before(2); - List followers = new ArrayList<>(mFollowers.values()); - Follower firstFollower = followers.get(0); - Follower secondFollower = followers.get(1); - - createSnapshotFile(firstFollower.mStore); // create default 0, 1 snapshot - createSnapshotFile(secondFollower.mStore, 0, 2); // preferable to the default 0, 1 snapshot - - // make sure to error out when requesting the better snapshot from secondFollower - Mockito.doAnswer(mock -> { - SingleFileSnapshotInfo snapshot = secondFollower.mStore.getLatestSnapshot(); - StreamObserver responseObserver = - SnapshotUploader.forFollower(secondFollower.mStore, snapshot); - StreamObserver requestObserver = mClient - .uploadSnapshot(responseObserver); - responseObserver.onError(new StatusRuntimeException(Status.UNAVAILABLE)); - requestObserver.onNext(UploadSnapshotPRequest.newBuilder() - .setData(SnapshotData.newBuilder() - .setSnapshotTerm(snapshot.getTerm()) - .setSnapshotIndex(snapshot.getIndex()) - .setOffset(0)) - .build()); - return null; - }).when(secondFollower.mSnapshotManager).sendSnapshotToLeader(); - - mLeaderSnapshotManager.maybeCopySnapshotFromFollower(); - - CommonUtils.waitFor("leader snapshot to complete", - () -> mLeaderSnapshotManager.maybeCopySnapshotFromFollower() != -1, mWaitOptions); - // verify that the leader still requests and gets second best snapshot - validateSnapshotFile(mLeaderStore); - } - - private class Follower { - final String mHost; - final int mRpcPort; - final SnapshotReplicationManager mSnapshotManager; - RaftJournalSystem mJournalSystem; - SimpleStateMachineStorage mStore; - - Follower(RaftJournalServiceClient client) throws IOException { - mHost = String.format("follower-%s", RandomString.make()); - mRpcPort = ThreadLocalRandom.current().nextInt(10_000, 99_999); - mStore = getSimpleStateMachineStorage(); - mJournalSystem = Mockito.mock(RaftJournalSystem.class); - mSnapshotManager = Mockito.spy(new SnapshotReplicationManager(mJournalSystem, mStore)); - Mockito.doReturn(client).when(mSnapshotManager).createJournalServiceClient(); - } - - void notifySnapshotInstalled() { - synchronized (mSnapshotManager) { - mSnapshotManager.notifyAll(); - } - } - - void disableFollowerUpload() throws IOException { - Mockito.doNothing().when(mSnapshotManager).sendSnapshotToLeader(); - } - - void disableGetInfo() throws IOException { - Mockito.doAnswer((args) -> { - synchronized (mSnapshotManager) { - // we sleep so nothing is returned - mSnapshotManager.wait(Configuration.global().getMs( - PropertyKey.MASTER_JOURNAL_REQUEST_INFO_TIMEOUT)); - } - throw new IOException("get info disabled"); - }).when(mSnapshotManager) - .handleRequest(argThat(JournalQueryRequest::hasSnapshotInfoRequest)); - } - - RaftPeerId getRaftPeerId() { - return RaftPeerId.valueOf(String.format("%s_%d", mHost, mRpcPort)); - } - } -} diff --git a/core/server/master/src/test/java/alluxio/master/metastore/rocks/RocksInodeStoreTest.java b/core/server/master/src/test/java/alluxio/master/metastore/rocks/RocksInodeStoreTest.java deleted file mode 100644 index 0e4a4561bb07..000000000000 --- a/core/server/master/src/test/java/alluxio/master/metastore/rocks/RocksInodeStoreTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.metastore.rocks; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -import alluxio.master.file.contexts.CreateDirectoryContext; -import alluxio.master.file.meta.MutableInodeDirectory; -import alluxio.master.metastore.InodeStore.WriteBatch; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; - -public class RocksInodeStoreTest { - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - @Test - public void batchWrite() throws IOException { - RocksInodeStore store = new RocksInodeStore(mFolder.newFolder().getAbsolutePath()); - WriteBatch batch = store.createWriteBatch(); - for (int i = 1; i < 20; i++) { - batch.writeInode( - MutableInodeDirectory.create(i, 0, "dir" + i, CreateDirectoryContext.defaults())); - } - batch.commit(); - for (int i = 1; i < 20; i++) { - assertEquals("dir" + i, store.get(i).get().getName()); - } - } - - @Test - public void toStringEntries() throws IOException { - RocksInodeStore store = new RocksInodeStore(mFolder.newFolder().getAbsolutePath()); - assertEquals("", store.toStringEntries()); - - store.writeInode(MutableInodeDirectory.create(1, 0, "dir", CreateDirectoryContext.defaults())); - assertEquals("dir", store.get(1).get().getName()); - assertThat(store.toStringEntries(), containsString("name=dir")); - } -} diff --git a/core/server/master/src/test/java/alluxio/master/metastore/rocks/RocksStoreTest.java b/core/server/master/src/test/java/alluxio/master/metastore/rocks/RocksStoreTest.java deleted file mode 100644 index e89cf874e7a2..000000000000 --- a/core/server/master/src/test/java/alluxio/master/metastore/rocks/RocksStoreTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.metastore.rocks; - -import static org.junit.Assert.assertArrayEquals; - -import alluxio.master.journal.checkpoint.CheckpointInputStream; - -import com.google.common.primitives.Longs; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.rocksdb.ColumnFamilyDescriptor; -import org.rocksdb.ColumnFamilyHandle; -import org.rocksdb.ColumnFamilyOptions; -import org.rocksdb.CompressionType; -import org.rocksdb.DBOptions; -import org.rocksdb.HashLinkedListMemTableConfig; -import org.rocksdb.RocksDB; -import org.rocksdb.WriteOptions; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; - -public class RocksStoreTest { - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - @Test - public void backupRestore() throws Exception { - ColumnFamilyOptions cfOpts = new ColumnFamilyOptions() - .setMemTableConfig(new HashLinkedListMemTableConfig()) - .setCompressionType(CompressionType.NO_COMPRESSION) - .useFixedLengthPrefixExtractor(Longs.BYTES); // We always search using the initial long key - - List columnDescriptors = - Arrays.asList(new ColumnFamilyDescriptor("test".getBytes(), cfOpts)); - String dbDir = mFolder.newFolder("rocks").getAbsolutePath(); - String backupsDir = mFolder.newFolder("rocks-backups").getAbsolutePath(); - AtomicReference testColumn = new AtomicReference<>(); - DBOptions dbOpts = new DBOptions().setCreateIfMissing(true) - .setCreateMissingColumnFamilies(true) - .setAllowConcurrentMemtableWrite(false); - RocksStore store = - new RocksStore("test", dbDir, backupsDir, dbOpts, columnDescriptors, - Arrays.asList(testColumn)); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - RocksDB db = store.getDb(); - int count = 10; - for (int i = 0; i < count; i++) { - db.put(testColumn.get(), new WriteOptions().setDisableWAL(true), ("a" + i).getBytes(), - "b".getBytes()); - } - store.writeToCheckpoint(baos); - store.close(); - - String newBbDir = mFolder.newFolder("rocks-new").getAbsolutePath(); - dbOpts = new DBOptions().setCreateIfMissing(true) - .setCreateMissingColumnFamilies(true) - .setAllowConcurrentMemtableWrite(false); - store = - new RocksStore("test-new", newBbDir, backupsDir, dbOpts, columnDescriptors, - Arrays.asList(testColumn)); - store.restoreFromCheckpoint( - new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray()))); - db = store.getDb(); - for (int i = 0; i < count; i++) { - assertArrayEquals("b".getBytes(), db.get(testColumn.get(), ("a" + i).getBytes())); - } - store.close(); - cfOpts.close(); - } -} diff --git a/core/server/master/src/test/java/alluxio/master/service/rpc/RpcServerServiceTest.java b/core/server/master/src/test/java/alluxio/master/service/rpc/RpcServerServiceTest.java deleted file mode 100644 index babc788cea42..000000000000 --- a/core/server/master/src/test/java/alluxio/master/service/rpc/RpcServerServiceTest.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.service.rpc; - -import alluxio.conf.Configuration; -import alluxio.grpc.GrpcServerAddress; -import alluxio.grpc.GrpcServerBuilder; -import alluxio.master.AlluxioMasterProcess; -import alluxio.master.MasterRegistry; -import alluxio.master.PortReservationRule; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.net.ConnectException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.util.Optional; - -/** - * Test for RpcSimpleService. - */ -public class RpcServerServiceTest { - @Rule - public PortReservationRule mPort = new PortReservationRule(); - - private final MasterRegistry mRegistry = new MasterRegistry(); - private InetSocketAddress mRpcAddress; - private AlluxioMasterProcess mMasterProcess; - - @Before - public void setUp() { - mRpcAddress = new InetSocketAddress(mPort.getPort()); - mMasterProcess = Mockito.mock(AlluxioMasterProcess.class); - Mockito.when(mMasterProcess.createBaseRpcServer()).thenAnswer(mock -> - GrpcServerBuilder.forAddress(GrpcServerAddress.create(mRpcAddress.getHostName(), - mRpcAddress), Configuration.global())); - Mockito.when(mMasterProcess.createRpcExecutorService()).thenReturn(Optional.empty()); - Mockito.when(mMasterProcess.getSafeModeManager()).thenReturn(Optional.empty()); - } - - @Test - public void primaryOnlyTest() { - RpcServerService service = - RpcServerService.Factory.create(mRpcAddress, mMasterProcess, mRegistry); - Assert.assertTrue(waitForFree()); - - Assert.assertFalse(service.isServing()); - service.start(); - // after start and before stop the rpc port is always bound as either the rpc server or the - // rejecting server is bound to is (depending on whether it is in PRIMARY or STANDBY state) - Assert.assertTrue(isBound()); - Assert.assertFalse(service.isServing()); - for (int i = 0; i < 5; i++) { - service.promote(); - Assert.assertTrue(service.isServing()); - Assert.assertTrue(isBound()); - service.demote(); - Assert.assertTrue(isBound()); - Assert.assertFalse(service.isServing()); - } - service.stop(); - Assert.assertFalse(service.isServing()); - Assert.assertFalse(isBound()); - } - - @Test - public void doubleStartRejectingServer() { - RpcServerService service = - RpcServerService.Factory.create(mRpcAddress, mMasterProcess, mRegistry); - - service.start(); - Assert.assertThrows("rejecting server must not be running", - IllegalStateException.class, service::start); - } - - @Test - public void doubleStartRpcServer() { - RpcServerService service = - RpcServerService.Factory.create(mRpcAddress, mMasterProcess, mRegistry); - - service.start(); - service.promote(); - Assert.assertThrows("rpc server must not be running", - IllegalStateException.class, service::promote); - } - - private boolean isBound() { - try (Socket socket = new Socket(mRpcAddress.getAddress(), mRpcAddress.getPort())) { - return true; - } catch (ConnectException e) { - return false; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private boolean waitForFree() { - try { - CommonUtils.waitFor("wait for socket to be free", () -> !isBound(), - WaitForOptions.defaults().setTimeoutMs(1_000).setInterval(10)); - return true; - } catch (Exception e) { - return false; - } - } -} diff --git a/core/server/master/src/test/java/alluxio/master/throttle/IndicatorsTests.java b/core/server/master/src/test/java/alluxio/master/throttle/IndicatorsTests.java deleted file mode 100644 index 1f521d41ac95..000000000000 --- a/core/server/master/src/test/java/alluxio/master/throttle/IndicatorsTests.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.master.throttle; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.util.JvmPauseMonitor; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.nio.ByteBuffer; - -/** - * The indicators test. - */ -public class IndicatorsTests { - private static final JvmPauseMonitor JVM_PAUSE_MONITOR = new JvmPauseMonitor( - Configuration.getMs(PropertyKey.JVM_MONITOR_SLEEP_INTERVAL_MS), - Configuration.getMs(PropertyKey.JVM_MONITOR_WARN_THRESHOLD_MS), - Configuration.getMs(PropertyKey.JVM_MONITOR_INFO_THRESHOLD_MS)); - - @Before - public void before() { - Configuration.set(PropertyKey.MASTER_JVM_MONITOR_ENABLED, true); - JVM_PAUSE_MONITOR.start(); - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.TOTAL_EXTRA_TIME.getName()), - JVM_PAUSE_MONITOR::getTotalExtraTime); - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.INFO_TIME_EXCEEDED.getName()), - JVM_PAUSE_MONITOR::getInfoTimeExceeded); - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.WARN_TIME_EXCEEDED.getName()), - JVM_PAUSE_MONITOR::getWarnTimeExceeded); - } - - @After - public void after() { - JVM_PAUSE_MONITOR.stop(); - } - - @Test - public void basicIndicatorCreationTest() throws InterruptedException { - int directMemSize = 101; - ByteBuffer.allocateDirect(directMemSize); - long baseSize = MetricsSystem.getDirectMemUsed(); - ByteBuffer.allocateDirect(directMemSize); - ServerIndicator serverIndicator1 - = ServerIndicator.createFromMetrics(0); - Thread.sleep(2000); - Assert.assertEquals(directMemSize + baseSize, serverIndicator1.getDirectMemUsed()); - ServerIndicator serverIndicator2 - = ServerIndicator.createFromMetrics(0); - Assert.assertEquals(directMemSize + baseSize, serverIndicator2.getDirectMemUsed()); - Assert.assertEquals(true, serverIndicator1.getPitTimeMS() - < serverIndicator2.getPitTimeMS()); - Assert.assertEquals(true, serverIndicator1.getTotalJVMPauseTimeMS() - <= serverIndicator2.getTotalJVMPauseTimeMS()); - Assert.assertEquals(serverIndicator1.getDirectMemUsed(), serverIndicator2.getDirectMemUsed()); - Assert.assertEquals(serverIndicator1.getHeapMax(), serverIndicator2.getHeapMax()); - Assert.assertEquals(serverIndicator1.getRpcQueueSize(), serverIndicator2.getRpcQueueSize()); - } - - @Test - public void basicIndicatorComparisonTest() throws InterruptedException { - long directMemUsed = 8; - long headMax = 1000; - long headUsed = 800; - double cpuLoad = 0.3; - long totalJVMPauseTimeMS = 200; - long pitTotalJVMPauseTime = 30; - long rpcQueueSize = 50; - long snapshotTimeMS = 33; - ServerIndicator serverIndicator = new ServerIndicator(directMemUsed, headMax, headUsed, cpuLoad, - totalJVMPauseTimeMS, pitTotalJVMPauseTime, rpcQueueSize, snapshotTimeMS); - - int times = 3; - ServerIndicator serverIndicatorM = new ServerIndicator(serverIndicator, times); - Assert.assertEquals(serverIndicatorM.getHeapUsed(), - (serverIndicator.getHeapUsed() * times)); - Assert.assertEquals(serverIndicatorM.getCpuLoad(), - (serverIndicator.getCpuLoad() * times), 0.0000001); - Assert.assertEquals(serverIndicatorM.getDirectMemUsed(), - (serverIndicator.getDirectMemUsed() * times)); - Assert.assertEquals(serverIndicatorM.getTotalJVMPauseTimeMS(), - (serverIndicator.getTotalJVMPauseTimeMS() * times)); - Assert.assertEquals(serverIndicatorM.getRpcQueueSize(), - (serverIndicator.getRpcQueueSize() * times)); - - long baseDeltaLong = 4; - double baseDeltaDouble = 0.1; - long deltaLong = baseDeltaLong; - double deltaDouble = baseDeltaDouble; - ServerIndicator serverIndicator1Delta = new ServerIndicator(directMemUsed + deltaLong, - headMax, - headUsed + deltaLong, cpuLoad + deltaDouble, - totalJVMPauseTimeMS + deltaLong, pitTotalJVMPauseTime + deltaLong, - rpcQueueSize + deltaLong, snapshotTimeMS); - - ServerIndicator deltaIndicator = new ServerIndicator(serverIndicator1Delta); - deltaIndicator.reduction(serverIndicator); - Assert.assertEquals(deltaLong, deltaIndicator.getRpcQueueSize()); - Assert.assertEquals(deltaDouble, deltaIndicator.getCpuLoad(), 0.00001); - Assert.assertEquals(deltaLong, deltaIndicator.getDirectMemUsed()); - Assert.assertEquals(deltaLong, deltaIndicator.getHeapUsed()); - Assert.assertEquals(deltaLong, deltaIndicator.getTotalJVMPauseTimeMS()); - - deltaDouble += deltaDouble; - deltaLong += deltaLong; - ServerIndicator serverIndicator2Delta = new ServerIndicator(directMemUsed + deltaLong, - headMax, - headUsed + deltaLong, cpuLoad + deltaDouble, - totalJVMPauseTimeMS + deltaLong, pitTotalJVMPauseTime + deltaLong, - rpcQueueSize + deltaLong, snapshotTimeMS); - deltaIndicator = new ServerIndicator(serverIndicator2Delta); - deltaIndicator.reduction(serverIndicator); - Assert.assertEquals(deltaIndicator.getRpcQueueSize(), deltaLong); - Assert.assertEquals(deltaIndicator.getCpuLoad(), deltaDouble, 0.00001); - Assert.assertEquals(deltaIndicator.getDirectMemUsed(), deltaLong); - Assert.assertEquals(deltaIndicator.getHeapUsed(), deltaLong); - Assert.assertEquals(deltaIndicator.getTotalJVMPauseTimeMS(), deltaLong); - - ServerIndicator serverAgg = new ServerIndicator(serverIndicator); - serverAgg.addition(serverIndicator); - serverAgg.addition(serverIndicator); - serverAgg.reduction(serverIndicator); - int aggTimes = 2; - Assert.assertEquals(aggTimes * serverIndicator.getRpcQueueSize(), serverAgg.getRpcQueueSize()); - Assert.assertEquals(aggTimes * serverIndicator.getDirectMemUsed(), - serverAgg.getDirectMemUsed()); - Assert.assertEquals(aggTimes * serverIndicator.getHeapUsed(), serverAgg.getHeapUsed()); - Assert.assertEquals(aggTimes * serverIndicator.getTotalJVMPauseTimeMS(), - serverAgg.getTotalJVMPauseTimeMS()); - Assert.assertEquals(aggTimes * serverIndicator.getCpuLoad(), - serverAgg.getCpuLoad(), 0.00001); - } -} - diff --git a/core/server/pom.xml b/core/server/pom.xml deleted file mode 100644 index 66525d88a161..000000000000 --- a/core/server/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - 4.0.0 - - alluxio-core - org.alluxio - 2.10.0-SNAPSHOT - - alluxio-core-server - pom - Alluxio Core - Server - Alluxio core services - - - common - master - proxy - worker - - - - - ${project.parent.parent.basedir}/build - - - diff --git a/core/server/proxy/pom.xml b/core/server/proxy/pom.xml deleted file mode 100644 index 010b5ac5b9c2..000000000000 --- a/core/server/proxy/pom.xml +++ /dev/null @@ -1,145 +0,0 @@ - - - 4.0.0 - - alluxio-core-server - org.alluxio - 2.10.0-SNAPSHOT - - alluxio-core-server-proxy - jar - Alluxio Core - Server - Proxy - Alluxio proxy service - - - - - ${project.parent.parent.parent.basedir}/build - - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - - - com.google.guava - guava - - - commons-io - commons-io - - - javax.servlet - javax.servlet-api - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - - - org.eclipse.jetty - jetty-server - - - org.eclipse.jetty - jetty-servlet - - - org.glassfish.jersey.containers - jersey-container-servlet-core - - - org.glassfish.jersey.core - jersey-server - - - org.glassfish.jersey.inject - jersey-hk2 - - - org.apache.kerby - kerby-util - - - - org.glassfish.jersey.media - jersey-media-json-jackson - - - - - org.alluxio - alluxio-core-common - ${project.version} - - - org.alluxio - alluxio-core-client-fs - ${project.version} - - - org.alluxio - alluxio-core-server-common - ${project.version} - - - - - - - - false - com.github.kongchen - swagger-maven-plugin - - - - false - http - [Alluxio Proxy Hostname] - /api/v1 - - v1 - Alluxio Proxy REST API Documentation - - - The Alluxio Proxy acts as a REST gateway for clients to communicate with the Alluxio system. There are three different endpoints: - -1. The Proxy endpoint gives general info about the proxy service. -1. The Paths endpoint provides a RESTful gateway to the Alluxio file system for metadata operations. -1. The Streams endpoint provides a RESTful gateway to the Alluxio file system for data operations. - - - - alluxio.proxy.AlluxioProxyRestServiceHandler - alluxio.proxy.PathsRestServiceHandler - alluxio.proxy.StreamsRestServiceHandler - - ${project.parent.parent.parent.basedir}/templates/strapdown.html.hbs - ${project.parent.parent.parent.basedir}/generated/proxy/index.html - ${project.parent.parent.parent.basedir}/generated/proxy/swagger-ui - - - - - - - diff --git a/core/server/proxy/src/main/java/alluxio/proxy/AlluxioProxyProcess.java b/core/server/proxy/src/main/java/alluxio/proxy/AlluxioProxyProcess.java deleted file mode 100644 index cb1f90f01963..000000000000 --- a/core/server/proxy/src/main/java/alluxio/proxy/AlluxioProxyProcess.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.proxy; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.util.network.NetworkAddressUtils.ServiceType; -import alluxio.web.ProxyWebServer; -import alluxio.web.WebServer; - -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.HttpClientBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeoutException; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class encapsulates the different worker services that are configured to run. - */ -@NotThreadSafe -public final class AlluxioProxyProcess implements ProxyProcess { - private static final Logger LOG = LoggerFactory.getLogger(AlluxioProxyProcess.class); - - /** The web server. */ - private WebServer mWebServer = null; - - /** Worker start time in milliseconds. */ - private final long mStartTimeMs; - - private final CountDownLatch mLatch; - - /** - * Creates an instance of {@link AlluxioProxy}. - */ - AlluxioProxyProcess() { - mStartTimeMs = System.currentTimeMillis(); - mLatch = new CountDownLatch(1); - } - - @Override - public int getWebLocalPort() { - return mWebServer.getLocalPort(); - } - - @Override - public long getStartTimeMs() { - return mStartTimeMs; - } - - @Override - public long getUptimeMs() { - return System.currentTimeMillis() - mStartTimeMs; - } - - @Override - public void start() throws Exception { - mWebServer = new ProxyWebServer(ServiceType.PROXY_WEB.getServiceName(), - NetworkAddressUtils.getBindAddress(ServiceType.PROXY_WEB, Configuration.global()), - this); - // reset proxy web port - Configuration.set(PropertyKey.PROXY_WEB_PORT, - mWebServer.getLocalPort()); - mWebServer.start(); - mLatch.await(); - } - - @Override - public void stop() throws Exception { - if (mWebServer != null) { - mWebServer.stop(); - mWebServer = null; - } - mLatch.countDown(); - } - - @Override - public boolean waitForReady(int timeoutMs) { - try { - CommonUtils.waitFor(this + " to start", () -> { - if (mWebServer == null || !mWebServer.getServer().isRunning()) { - return false; - } - HttpClient client = HttpClientBuilder.create().build(); - HttpPost method = new HttpPost(String - .format("http://%s:%d%s/%s/%s/%s", mWebServer.getBindHost(), mWebServer.getLocalPort(), - Constants.REST_API_PREFIX, PathsRestServiceHandler.SERVICE_PREFIX, "%2f", - PathsRestServiceHandler.EXISTS)); - try { - HttpResponse response = client.execute(method); - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - return true; - } - LOG.debug(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); - return false; - } catch (IOException e) { - LOG.debug("Exception: ", e); - return false; - } - }, WaitForOptions.defaults().setTimeoutMs(timeoutMs)); - return true; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } catch (TimeoutException e) { - return false; - } - } - - @Override - public String toString() { - return "Alluxio Proxy"; - } -} diff --git a/core/server/proxy/src/main/java/alluxio/proxy/s3/DeleteObjectsResult.java b/core/server/proxy/src/main/java/alluxio/proxy/s3/DeleteObjectsResult.java deleted file mode 100644 index 27132a63362e..000000000000 --- a/core/server/proxy/src/main/java/alluxio/proxy/s3/DeleteObjectsResult.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.proxy.s3; - -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -import java.util.ArrayList; -import java.util.List; - -/** - * An object representing the response to a DeleteObjects request. See - * https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html for more information about - * this type of request. - */ -@JacksonXmlRootElement(localName = "DeleteResult") -public class DeleteObjectsResult { - - @JacksonXmlProperty(localName = "Deleted") - @JacksonXmlElementWrapper(useWrapping = false) - private List mDeleted = new ArrayList<>(); - - @JacksonXmlProperty(localName = "Error") - @JacksonXmlElementWrapper(useWrapping = false) - private List mErrored = new ArrayList<>(); - - /** - * @return the successfully deleted objects - */ - @JacksonXmlProperty(localName = "Deleted") - public List getDeleted() { - return mDeleted; - } - - /** - * @return the objects that errored when deleting - */ - @JacksonXmlProperty(localName = "Error") - public List getErrored() { - return mErrored; - } - - /** - * Sets the set of deleted objects. - * - * @param deleted the set to include in the response - */ - @JacksonXmlProperty(localName = "Deleted") - public void setDeleted(List deleted) { - mDeleted = deleted; - } - - /** - * Sets the set of errored objects in the response. - * - * @param errored the set of errored objects - */ - @JacksonXmlProperty(localName = "Error") - public void setErrored(List errored) { - mErrored = errored; - } - - @JacksonXmlRootElement(localName = "Deleted") - static class DeletedObject { - - @JacksonXmlProperty(localName = "Key") - private String mKey; - - @JacksonXmlProperty(localName = "DeleteMarker") - private String mDeleteMarker; - - @JacksonXmlProperty(localName = "DeleteMarkerVersionId") - private String mDeleteMarkerVersionId; - - @JacksonXmlProperty(localName = "VersionId") - private String mVersionId; - - public DeletedObject() { } - - public DeletedObject(String key) { - mKey = key; - } - - @JacksonXmlProperty(localName = "Key") - public String getKey() { - return mKey; - } - - @JacksonXmlProperty(localName = "Key") - public void setKey(String key) { - mKey = key; - } - - @JacksonXmlProperty(localName = "DeleteMarker") - public String getDeleteMarker() { - return mDeleteMarker; - } - - @JacksonXmlProperty(localName = "DeleteMarker") - public void setDeleteMarker(String marker) { - mDeleteMarker = marker; - } - - @JacksonXmlProperty(localName = "DeleteMarkerVersionId") - public String getDeleteMarkerVersionId() { - return mDeleteMarkerVersionId; - } - - @JacksonXmlProperty(localName = "DeleteMarkerVersionId") - public void setDeleteMarkerVersionId(String deleteMarkerVersionId) { - mDeleteMarkerVersionId = deleteMarkerVersionId; - } - - @JacksonXmlProperty(localName = "VersionId") - public String getVersionId() { - return mVersionId; - } - - @JacksonXmlProperty(localName = "VersionId") - public void setVersionId(String versionId) { - mVersionId = versionId; - } - } - - @JacksonXmlRootElement(localName = "Error") - static class ErrorObject { - public String mKey; - public String mCode; - public String mMessage; - public String mVersionId; - - public ErrorObject() { - mKey = ""; - mCode = ""; - mMessage = ""; - mVersionId = ""; - } - - @JacksonXmlProperty(localName = "Key") - public String getKey() { - return mKey; - } - - @JacksonXmlProperty(localName = "Code") - public String getCode() { - return mCode; - } - - @JacksonXmlProperty(localName = "Message") - public String getMessage() { - return mMessage; - } - - @JacksonXmlProperty(localName = "VersionId") - public String getVersionId() { - return mVersionId; - } - - @JacksonXmlProperty(localName = "Key") - public void setKey(String key) { - mKey = key; - } - - @JacksonXmlProperty(localName = "Code") - public void setCode(String code) { - mCode = code; - } - - @JacksonXmlProperty(localName = "Message") - public void setMessage(String message) { - mMessage = message; - } - - @JacksonXmlProperty(localName = "VersionId") - public void setVersionId(String versionId) { - mVersionId = versionId; - } - } -} diff --git a/core/server/proxy/src/main/java/alluxio/proxy/s3/ListAllMyBucketsResult.java b/core/server/proxy/src/main/java/alluxio/proxy/s3/ListAllMyBucketsResult.java deleted file mode 100644 index 05401c878e5c..000000000000 --- a/core/server/proxy/src/main/java/alluxio/proxy/s3/ListAllMyBucketsResult.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.proxy.s3; - -import alluxio.client.file.URIStatus; - -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * Implementation of ListAllMyBucketsResult according to https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html. - */ -@JacksonXmlRootElement(localName = "ListAllMyBucketsResult") -public class ListAllMyBucketsResult { - private List mBuckets; - - /** - * Creates a {@link ListAllMyBucketsResult}. - * - * @param names names of all of the buckets - */ - public ListAllMyBucketsResult(List names) { - mBuckets = - names.stream().map((uriStatus) -> new Bucket(uriStatus.getName(), - S3RestUtils.toS3Date(uriStatus.getCreationTimeMs()))) - .collect(Collectors.toList()); - } - - /** - * @return the list of buckets - */ - @JacksonXmlProperty(localName = "Bucket") - @JacksonXmlElementWrapper(localName = "Buckets") - public List getBuckets() { - return mBuckets; - } - - /** - * The Bucket object. - */ - @JacksonXmlRootElement(localName = "Bucket") - public class Bucket { - private String mName; - private String mCreationDate; - - private Bucket(String name, String creationDate) { - mName = name; - mCreationDate = creationDate; - } - - /** - * @return the name of the bucket - */ - @JacksonXmlProperty(localName = "Name") - public String getName() { - return mName; - } - - /** - * @return the creation timestamp for the bucket - */ - @JacksonXmlProperty(localName = "CreationDate") - public String getCreationDate() { - return mCreationDate; - } - } -} diff --git a/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ErrorResponse.java b/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ErrorResponse.java deleted file mode 100644 index 67b541f438be..000000000000 --- a/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ErrorResponse.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.proxy.s3; - -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.exception.runtime.NotFoundRuntimeException; -import alluxio.exception.status.AlluxioStatusException; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.IOException; -import javax.ws.rs.core.Response; - -/** - * Utilities for creating HTTP Responses for the S3 API. - */ -public class S3ErrorResponse { - private static final Logger LOG = LoggerFactory.getLogger(S3ErrorResponse.class); - - /** - * Creates an error response using the given exception. - * - * @param e a {@link Throwable} object - * @param resource an S3 resource key - * @return response Http {@link Response} - */ - public static Response createErrorResponse(Throwable e, String resource) { - if (e instanceof AlluxioStatusException) { - return createErrorResponse((AlluxioStatusException) e, resource); - } else if (e instanceof AlluxioRuntimeException) { - return createErrorResponse((AlluxioRuntimeException) e, resource); - } else if (e instanceof S3Exception) { - return createErrorResponse((S3Exception) e, resource); - } else if (e instanceof IOException) { - return createErrorResponse((IOException) e, resource); - } else { - return Response.status(Response.Status.INTERNAL_SERVER_ERROR) - .entity(e.getMessage()).build(); - } - } - - /** - * Creates an error response using the given exception. - * - * @param e the exception to be converted into {@link Error} and encoded into XML - * @param resource resource - * @return the response - */ - private static Response createErrorResponse(S3Exception e, String resource) { - S3Error errorResponse = new S3Error(resource, e.getErrorCode()); - // Need to explicitly encode the string as XML because Jackson will not do it automatically. - XmlMapper mapper = new XmlMapper(); - try { - return Response.status(e.getErrorCode().getStatus()) - .entity(mapper.writeValueAsString(errorResponse)).build(); - } catch (JsonProcessingException e2) { - return Response.status(Response.Status.INTERNAL_SERVER_ERROR) - .entity("Failed to encode XML: " + e2.getMessage()).build(); - } - } - - /** - * convert the AlluxioStatusException to the HTTP Response. - * @param e AlluxioStatusException - * @param resource resource - * @return response Http Response - */ - private static Response createErrorResponse(AlluxioStatusException e, String resource) { - XmlMapper mapper = new XmlMapper(); - S3ErrorCode s3ErrorCode; - // TODO(WYY): we need to handle more exception in the future. - if (e instanceof alluxio.exception.status.NotFoundException) { - // 404 - s3ErrorCode = S3ErrorCode.NO_SUCH_KEY; - } else if (e instanceof alluxio.exception.status.InvalidArgumentException) { - // 400 - s3ErrorCode = S3ErrorCode.INVALID_ARGUMENT; - } else if (e instanceof alluxio.exception.status.PermissionDeniedException) { - // 403 - s3ErrorCode = S3ErrorCode.ACCESS_DENIED_ERROR; - } else if (e instanceof alluxio.exception.status.FailedPreconditionException) { - // 412 - s3ErrorCode = S3ErrorCode.PRECONDITION_FAILED; - } else { - // 500 - s3ErrorCode = S3ErrorCode.INTERNAL_ERROR; - } - S3Error errorResponse = new S3Error(resource, s3ErrorCode); - errorResponse.setMessage(e.getMessage()); - try { - return Response.status(s3ErrorCode.getStatus()) - .entity(mapper.writeValueAsString(errorResponse)).build(); - } catch (JsonProcessingException e2) { - return Response.status(Response.Status.INTERNAL_SERVER_ERROR) - .entity("Failed to encode XML: " + e2.getMessage()).build(); - } finally { - LOG.warn("mapper convert exception {} to {}.", e.getClass().getName(), - s3ErrorCode.getStatus().toString()); - } - } - - /** - * convert the IOException to the HTTP Response. - * @param e IOException - * @param resource resource - * @return response Http Response - */ - private static Response createErrorResponse(IOException e, String resource) { - XmlMapper mapper = new XmlMapper(); - S3ErrorCode s3ErrorCode; - // TODO(WYY): we need to handle more exception in the future. - if (e instanceof FileNotFoundException) { - // 404 - s3ErrorCode = S3ErrorCode.NO_SUCH_KEY; - } else { - // 500 - s3ErrorCode = S3ErrorCode.INTERNAL_ERROR; - } - S3Error errorResponse = new S3Error(resource, s3ErrorCode); - errorResponse.setMessage(e.getMessage()); - try { - return Response.status(s3ErrorCode.getStatus()) - .entity(mapper.writeValueAsString(errorResponse)).build(); - } catch (JsonProcessingException e2) { - return Response.status(Response.Status.INTERNAL_SERVER_ERROR) - .entity("Failed to encode XML: " + e2.getMessage()).build(); - } finally { - LOG.warn("mapper convert exception {} to {}.", e.getClass().getName(), - s3ErrorCode.getStatus().toString()); - } - } - - /** - * convert the IOException to the HTTP Response. - * @param e AlluxioRuntimeException - * @param resource resource - * @return response Http Response - */ - private static Response createErrorResponse(AlluxioRuntimeException e, String resource) { - XmlMapper mapper = new XmlMapper(); - S3ErrorCode s3ErrorCode; - // TODO(WYY): we need to handle more exception in the future. - if (e instanceof NotFoundRuntimeException) { - // 404 - s3ErrorCode = S3ErrorCode.NO_SUCH_KEY; - } else { - // 500 - s3ErrorCode = S3ErrorCode.INTERNAL_ERROR; - } - S3Error errorResponse = new S3Error(resource, s3ErrorCode); - errorResponse.setMessage(e.getMessage()); - try { - return Response.status(s3ErrorCode.getStatus()) - .entity(mapper.writeValueAsString(errorResponse)).build(); - } catch (JsonProcessingException e2) { - return Response.status(Response.Status.INTERNAL_SERVER_ERROR) - .entity("Failed to encode XML: " + e2.getMessage()).build(); - } finally { - LOG.warn("mapper convert exception {} to {}.", e.getClass().getName(), - s3ErrorCode.getStatus().toString()); - } - } -} diff --git a/core/server/proxy/src/main/java/alluxio/proxy/s3/signature/AwsSignatureProcessor.java b/core/server/proxy/src/main/java/alluxio/proxy/s3/signature/AwsSignatureProcessor.java deleted file mode 100644 index 4bcaea62dec9..000000000000 --- a/core/server/proxy/src/main/java/alluxio/proxy/s3/signature/AwsSignatureProcessor.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.proxy.s3.signature; - -import static alluxio.proxy.s3.S3Constants.S3_SIGN_DATE; - -import alluxio.proxy.s3.S3ErrorCode; -import alluxio.proxy.s3.S3Exception; -import alluxio.proxy.s3.S3RestUtils; -import alluxio.proxy.s3.auth.AwsAuthInfo; -import alluxio.proxy.s3.signature.utils.AwsAuthV2HeaderParserUtils; -import alluxio.proxy.s3.signature.utils.AwsAuthV4HeaderParserUtils; -import alluxio.proxy.s3.signature.utils.AwsAuthV4QueryParserUtils; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; -import javax.ws.rs.container.ContainerRequestContext; - -/** - * Parser to process AWS V2 & V4 auth request. Creates string to sign and auth - * header. For more details refer to AWS documentation https://docs.aws - * .amazon.com/general/latest/gr/sigv4-create-canonical-request.html. - **/ - -public class AwsSignatureProcessor { - private static final Logger LOG = - LoggerFactory.getLogger(AwsSignatureProcessor.class); - private static final String AUTHORIZATION = "Authorization"; - - private final ContainerRequestContext mContext; - - /** - * Create a new {@link AwsSignatureProcessor}. - * - * @param context ContainerRequestContext - */ - public AwsSignatureProcessor(ContainerRequestContext context) { - mContext = context; - } - - /** - * Extract signature info from request. - * @return SignatureInfo - * @throws S3Exception - */ - public SignatureInfo parseSignature() throws S3Exception { - Map headers = S3RestUtils.fromMultiValueToSingleValueMap( - mContext.getHeaders(), true); - String authHeader = headers.get(AUTHORIZATION); - String dateHeader = headers.get(S3_SIGN_DATE); - Map queryParameters = S3RestUtils.fromMultiValueToSingleValueMap( - mContext.getUriInfo().getQueryParameters(), false); - - SignatureInfo signatureInfo; - if ((signatureInfo = - AwsAuthV4HeaderParserUtils.parseSignature(authHeader, dateHeader)) != null - || (signatureInfo = - AwsAuthV2HeaderParserUtils.parseSignature(authHeader)) != null - || (signatureInfo = - AwsAuthV4QueryParserUtils.parseSignature(queryParameters)) != null) { - return signatureInfo; - } else { - LOG.error("Can not parse signature from header information."); - throw new S3Exception("Can not parse signature from header information.", - S3ErrorCode.ACCESS_DENIED_ERROR); - } - } - - /** - * Convert SignatureInfo to AwsAuthInfo. - * @return AwsAuthInfo - * @throws S3Exception - */ - public AwsAuthInfo getAuthInfo() throws S3Exception { - try { - SignatureInfo signatureInfo = parseSignature(); - String stringToSign = ""; - if (signatureInfo.getVersion() == SignatureInfo.Version.V4) { - stringToSign = - StringToSignProducer.createSignatureBase(signatureInfo, mContext); - } - String awsAccessId = signatureInfo.getAwsAccessId(); - // ONLY validate aws access id when needed. - if (StringUtils.isEmpty(awsAccessId)) { - LOG.debug("Malformed s3 header. awsAccessID is empty"); - throw new S3Exception("awsAccessID is empty", S3ErrorCode.ACCESS_DENIED_ERROR); - } - - return new AwsAuthInfo(awsAccessId, - stringToSign, - signatureInfo.getSignature() - ); - } catch (S3Exception ex) { - LOG.debug("Error during signature parsing: ", ex); - throw ex; - } catch (Exception e) { - // For any other critical errors during object creation throw Internal - // error. - LOG.debug("Error during signature parsing: ", e); - throw new S3Exception(e, "Context is invalid", S3ErrorCode.INTERNAL_ERROR); - } - } -} diff --git a/core/server/proxy/src/main/java/alluxio/web/ProxyWebServer.java b/core/server/proxy/src/main/java/alluxio/web/ProxyWebServer.java deleted file mode 100644 index f876d736de7a..000000000000 --- a/core/server/proxy/src/main/java/alluxio/web/ProxyWebServer.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.web; - -import alluxio.Constants; -import alluxio.StreamCache; -import alluxio.client.file.FileSystem; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.audit.AsyncUserAccessAuditLogWriter; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proxy.ProxyProcess; -import alluxio.proxy.s3.CompleteMultipartUploadHandler; -import alluxio.proxy.s3.S3RestExceptionMapper; -import alluxio.util.io.PathUtils; - -import com.google.common.base.Stopwatch; -import org.eclipse.jetty.servlet.ServletHolder; -import org.glassfish.jersey.server.ResourceConfig; -import org.glassfish.jersey.servlet.ServletContainer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Collections; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import javax.annotation.concurrent.NotThreadSafe; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** - * The Alluxio proxy web server. - */ -@NotThreadSafe -public final class ProxyWebServer extends WebServer { - private static final Logger LOG = LoggerFactory.getLogger(ProxyWebServer.class); - public static final String ALLUXIO_PROXY_SERVLET_RESOURCE_KEY = "Alluxio Proxy"; - public static final String FILE_SYSTEM_SERVLET_RESOURCE_KEY = "File System"; - public static final String STREAM_CACHE_SERVLET_RESOURCE_KEY = "Stream Cache"; - - public static final String SERVER_CONFIGURATION_RESOURCE_KEY = "Server Configuration"; - public static final String ALLUXIO_PROXY_AUDIT_LOG_WRITER_KEY = "Alluxio Proxy Audit Log Writer"; - - private final FileSystem mFileSystem; - - private AsyncUserAccessAuditLogWriter mAsyncAuditLogWriter; - - /** - * Creates a new instance of {@link ProxyWebServer}. - * - * @param serviceName the service name - * @param address the service address - * @param proxyProcess the Alluxio proxy process - */ - public ProxyWebServer(String serviceName, InetSocketAddress address, - final ProxyProcess proxyProcess) { - super(serviceName, address); - - // REST configuration - ResourceConfig config = new ResourceConfig().packages("alluxio.proxy", "alluxio.proxy.s3", - "alluxio.proxy.s3.logging") - .register(JacksonProtobufObjectMapperProvider.class) - .register(S3RestExceptionMapper.class); - - mFileSystem = FileSystem.Factory.create(Configuration.global()); - - if (Configuration.getBoolean(PropertyKey.PROXY_AUDIT_LOGGING_ENABLED)) { - mAsyncAuditLogWriter = new AsyncUserAccessAuditLogWriter("PROXY_AUDIT_LOG"); - mAsyncAuditLogWriter.start(); - MetricsSystem.registerGaugeIfAbsent( - MetricKey.PROXY_AUDIT_LOG_ENTRIES_SIZE.getName(), - () -> mAsyncAuditLogWriter != null - ? mAsyncAuditLogWriter.getAuditLogEntriesSize() : -1); - } - - ServletContainer servlet = new ServletContainer(config) { - private static final long serialVersionUID = 7756010860672831556L; - - @Override - public void init() throws ServletException { - super.init(); - getServletContext().setAttribute(ALLUXIO_PROXY_SERVLET_RESOURCE_KEY, proxyProcess); - getServletContext() - .setAttribute(FILE_SYSTEM_SERVLET_RESOURCE_KEY, mFileSystem); - getServletContext().setAttribute(STREAM_CACHE_SERVLET_RESOURCE_KEY, - new StreamCache(Configuration.getMs(PropertyKey.PROXY_STREAM_CACHE_TIMEOUT_MS))); - getServletContext().setAttribute(ALLUXIO_PROXY_AUDIT_LOG_WRITER_KEY, mAsyncAuditLogWriter); - } - - @Override - public void service(final ServletRequest req, final ServletResponse res) - throws ServletException, IOException { - Stopwatch stopWatch = Stopwatch.createStarted(); - super.service(req, res); - if ((req instanceof HttpServletRequest) && (res instanceof HttpServletResponse)) { - HttpServletRequest httpReq = (HttpServletRequest) req; - HttpServletResponse httpRes = (HttpServletResponse) res; - logAccess(httpReq, httpRes, stopWatch); - } - } - }; - ServletHolder servletHolder = new ServletHolder("Alluxio Proxy Web Service", servlet); - mServletContextHandler - .addServlet(servletHolder, PathUtils.concatPath(Constants.REST_API_PREFIX, "*")); - // TODO(czhu): Move S3 API logging out of CompleteMultipartUploadHandler into a logging handler - addHandler(new CompleteMultipartUploadHandler(mFileSystem, Constants.REST_API_PREFIX)); - } - - @Override - public void stop() throws Exception { - if (mAsyncAuditLogWriter != null) { - mAsyncAuditLogWriter.stop(); - mAsyncAuditLogWriter = null; - } - mFileSystem.close(); - super.stop(); - } - - /** - * Log the access of every single http request. - * @param request - * @param response - * @param stopWatch - */ - public static void logAccess(HttpServletRequest request, HttpServletResponse response, - Stopwatch stopWatch) { - String contentLenStr = "None"; - if (request.getHeader("x-amz-decoded-content-length") != null) { - contentLenStr = request.getHeader("x-amz-decoded-content-length"); - } else if (request.getHeader("Content-Length") != null) { - contentLenStr = request.getHeader("Content-Length"); - } - String accessLog = String.format("[ACCESSLOG] Request:%s - Status:%d " - + "- ContentLength:%s - Elapsed(ms):%d", - request, response.getStatus(), - contentLenStr, stopWatch.elapsed(TimeUnit.MILLISECONDS)); - if (LOG.isDebugEnabled()) { - String requestHeaders = Collections.list(request.getHeaderNames()).stream() - .map(x -> x + ":" + request.getHeader(x)) - .collect(Collectors.joining("\n")); - String responseHeaders = response.getHeaderNames().stream() - .map(x -> x + ":" + response.getHeader(x)) - .collect(Collectors.joining("\n")); - String moreInfoStr = String.format("%n[RequestHeader]:%n%s%n[ResponseHeader]:%n%s", - requestHeaders, responseHeaders); - LOG.debug(accessLog + " " + moreInfoStr); - } else { - LOG.info(accessLog); - } - } -} diff --git a/core/server/worker/pom.xml b/core/server/worker/pom.xml deleted file mode 100644 index d68ac2327586..000000000000 --- a/core/server/worker/pom.xml +++ /dev/null @@ -1,167 +0,0 @@ - - - 4.0.0 - - alluxio-core-server - org.alluxio - 2.10.0-SNAPSHOT - - alluxio-core-server-worker - jar - Alluxio Core - Server - Worker - Alluxio worker service - - - - - ${project.parent.parent.parent.basedir}/build - - - - - - com.google.guava - guava - - - commons-io - commons-io - - - io.dropwizard.metrics - metrics-core - - - javax.servlet - javax.servlet-api - - - org.apache.commons - commons-lang3 - - - org.eclipse.jetty - jetty-server - - - org.eclipse.jetty - jetty-servlet - - - org.glassfish.jersey.containers - jersey-container-servlet-core - - - org.glassfish.jersey.core - jersey-server - - - org.glassfish.jersey.inject - jersey-hk2 - - - org.glassfish.jersey.media - jersey-media-json-jackson - provided - - - - - org.alluxio - alluxio-core-client-fs - ${project.version} - - - org.alluxio - alluxio-core-common - ${project.version} - - - org.alluxio - alluxio-core-transport - ${project.version} - - - org.alluxio - alluxio-core-server-common - ${project.version} - - - org.alluxio - alluxio-integration-fuse - ${project.version} - - - - - org.hamcrest - hamcrest - test - - - - - org.alluxio - alluxio-core-common - ${project.version} - test-jar - test - - - - org.alluxio - alluxio-underfs-local - ${project.version} - test - - - org.alluxio - alluxio-underfs-hdfs - ${project.version} - test - - - - - - - - false - com.github.kongchen - swagger-maven-plugin - - - - false - http - [Alluxio Worker Hostname] - /api/v1 - - v1 - Alluxio Worker REST API Documentation - - The Alluxio Workers are processes which provide clients access to the data exposed by the Alluxio System. - - - alluxio.worker.AlluxioWorkerRestServiceHandler - ${project.parent.parent.parent.basedir}/templates/strapdown.html.hbs - ${project.parent.parent.parent.basedir}/generated/worker/index.html - ${project.parent.parent.parent.basedir}/generated/worker/swagger-ui - - - - - - - diff --git a/core/server/worker/src/main/java/alluxio/worker/AlluxioWorker.java b/core/server/worker/src/main/java/alluxio/worker/AlluxioWorker.java deleted file mode 100644 index 050cf0a1d032..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/AlluxioWorker.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker; - -import alluxio.ProcessUtils; -import alluxio.RuntimeConstants; -import alluxio.conf.Configuration; -import alluxio.grpc.Scope; -import alluxio.master.MasterInquireClient; -import alluxio.retry.RetryUtils; -import alluxio.security.user.ServerUserState; -import alluxio.util.CommonUtils; -import alluxio.util.ConfigurationUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Entry point for the Alluxio worker. - */ -@ThreadSafe -public final class AlluxioWorker { - private static final Logger LOG = LoggerFactory.getLogger(AlluxioWorker.class); - - /** - * Starts the Alluxio worker. - * - * @param args command line arguments, should be empty - */ - public static void main(String[] args) { - if (args.length != 0) { - LOG.info("java -cp {} {}", RuntimeConstants.ALLUXIO_JAR, - AlluxioWorker.class.getCanonicalName()); - System.exit(-1); - } - - if (!ConfigurationUtils.masterHostConfigured(Configuration.global())) { - ProcessUtils.fatalError(LOG, - ConfigurationUtils.getMasterHostNotConfiguredMessage("Alluxio worker")); - } - - CommonUtils.PROCESS_TYPE.set(CommonUtils.ProcessType.WORKER); - MasterInquireClient masterInquireClient = - MasterInquireClient.Factory.create(Configuration.global(), ServerUserState.global()); - try { - RetryUtils.retry("load cluster default configuration with master", () -> { - InetSocketAddress masterAddress = masterInquireClient.getPrimaryRpcAddress(); - Configuration.loadClusterDefaults(masterAddress, Scope.WORKER); - }, RetryUtils.defaultWorkerMasterClientRetry()); - } catch (IOException e) { - ProcessUtils.fatalError(LOG, - "Failed to load cluster default configuration for worker. Please make sure that Alluxio " - + "master is running: %s", e.toString()); - } - WorkerProcess process; - try { - process = WorkerProcess.Factory.create(); - } catch (Throwable t) { - ProcessUtils.fatalError(LOG, t, "Failed to create worker process"); - // fatalError will exit, so we shouldn't reach here. - throw t; - } - - ProcessUtils.stopProcessOnShutdown(process); - ProcessUtils.run(process); - } - - private AlluxioWorker() {} // prevent instantiation -} diff --git a/core/server/worker/src/main/java/alluxio/worker/AlluxioWorkerProcess.java b/core/server/worker/src/main/java/alluxio/worker/AlluxioWorkerProcess.java deleted file mode 100644 index 88d3170351b2..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/AlluxioWorkerProcess.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.network.ChannelType; -import alluxio.underfs.UfsManager; -import alluxio.underfs.WorkerUfsManager; -import alluxio.util.CommonUtils; -import alluxio.util.JvmPauseMonitor; -import alluxio.util.WaitForOptions; -import alluxio.util.io.FileUtils; -import alluxio.util.io.PathUtils; -import alluxio.util.network.NettyUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.util.network.NetworkAddressUtils.ServiceType; -import alluxio.web.WebServer; -import alluxio.web.WorkerWebServer; -import alluxio.wire.TieredIdentity; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.BlockWorker; -import alluxio.worker.grpc.GrpcDataServer; - -import io.netty.channel.unix.DomainSocketAddress; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.util.ArrayList; -import java.util.List; -import java.util.ServiceLoader; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeoutException; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class encapsulates the different worker services that are configured to run. - */ -@NotThreadSafe -public final class AlluxioWorkerProcess implements WorkerProcess { - private static final Logger LOG = LoggerFactory.getLogger(AlluxioWorkerProcess.class); - - private final TieredIdentity mTieredIdentitiy; - - /** Server for data requests and responses. */ - private final DataServer mDataServer; - - /** If started (i.e. not null), this server is used to serve local data transfer. */ - private DataServer mDomainSocketDataServer; - - /** The worker registry. */ - private final WorkerRegistry mRegistry; - - /** Worker Web UI server. */ - private final WebServer mWebServer; - - /** Used for auto binding. **/ - private ServerSocket mBindSocket; - - /** The bind address for the rpc server. */ - private final InetSocketAddress mRpcBindAddress; - - /** The connect address for the rpc server. */ - private final InetSocketAddress mRpcConnectAddress; - - /** Worker start time in milliseconds. */ - private final long mStartTimeMs; - - /** The manager for all ufs. */ - private final UfsManager mUfsManager; - - /** The jvm monitor.*/ - private JvmPauseMonitor mJvmPauseMonitor; - - /** - * Creates a new instance of {@link AlluxioWorkerProcess}. - */ - AlluxioWorkerProcess(TieredIdentity tieredIdentity) { - mTieredIdentitiy = tieredIdentity; - try { - mStartTimeMs = System.currentTimeMillis(); - mUfsManager = new WorkerUfsManager(); - mRegistry = new WorkerRegistry(); - List> callables = new ArrayList<>(); - for (final WorkerFactory factory : ServiceLoader.load(WorkerFactory.class, - WorkerFactory.class.getClassLoader())) { - callables.add(() -> { - if (factory.isEnabled()) { - factory.create(mRegistry, mUfsManager); - } - return null; - }); - } - CommonUtils.invokeAll(callables, - Configuration.getMs(PropertyKey.WORKER_STARTUP_TIMEOUT)); - - // Setup web server - mWebServer = - new WorkerWebServer(NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, - Configuration.global()), this, - mRegistry.get(BlockWorker.class)); - - // Random port binding. - int bindPort; - InetSocketAddress configuredBindAddress = - NetworkAddressUtils.getBindAddress(ServiceType.WORKER_RPC, - Configuration.global()); - if (configuredBindAddress.getPort() == 0) { - mBindSocket = new ServerSocket(0); - bindPort = mBindSocket.getLocalPort(); - } else { - bindPort = configuredBindAddress.getPort(); - } - mRpcBindAddress = new InetSocketAddress(configuredBindAddress.getHostName(), bindPort); - mRpcConnectAddress = NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, - Configuration.global()); - if (mBindSocket != null) { - // Socket opened for auto bind. - // Close it. - mBindSocket.close(); - } - // Setup Data server - mDataServer = new GrpcDataServer(mRpcConnectAddress.getHostName(), mRpcBindAddress, this); - - // Setup domain socket data server - if (isDomainSocketEnabled()) { - String domainSocketPath = - Configuration.getString(PropertyKey.WORKER_DATA_SERVER_DOMAIN_SOCKET_ADDRESS); - if (Configuration.getBoolean(PropertyKey.WORKER_DATA_SERVER_DOMAIN_SOCKET_AS_UUID)) { - domainSocketPath = - PathUtils.concatPath(domainSocketPath, UUID.randomUUID().toString()); - } - LOG.info("Domain socket data server is enabled at {}.", domainSocketPath); - mDomainSocketDataServer = new GrpcDataServer(mRpcConnectAddress.getHostName(), - new DomainSocketAddress(domainSocketPath), this); - // Share domain socket so that clients can access it. - FileUtils.changeLocalFileToFullPermission(domainSocketPath); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public long getStartTimeMs() { - return mStartTimeMs; - } - - @Override - public long getUptimeMs() { - return System.currentTimeMillis() - mStartTimeMs; - } - - @Override - public String getDataBindHost() { - return ((InetSocketAddress) mDataServer.getBindAddress()).getHostString(); - } - - @Override - public int getDataLocalPort() { - return ((InetSocketAddress) mDataServer.getBindAddress()).getPort(); - } - - @Override - public String getDataDomainSocketPath() { - if (mDomainSocketDataServer != null) { - return ((DomainSocketAddress) mDomainSocketDataServer.getBindAddress()).path(); - } - return ""; - } - - @Override - public String getWebBindHost() { - return mWebServer.getBindHost(); - } - - @Override - public int getWebLocalPort() { - return mWebServer.getLocalPort(); - } - - @Override - public T getWorker(Class clazz) { - return mRegistry.get(clazz); - } - - @Override - public UfsManager getUfsManager() { - return mUfsManager; - } - - @Override - public InetSocketAddress getRpcAddress() { - return mRpcBindAddress; - } - - @Override - public void start() throws Exception { - // NOTE: the order to start different services is sensitive. If you change it, do it cautiously. - - // Start serving metrics system, this will not block - MetricsSystem.startSinks(Configuration.getString(PropertyKey.METRICS_CONF_FILE)); - - // Start each worker. This must be done before starting the web or RPC servers. - // Requirement: NetAddress set in WorkerContext, so block worker can initialize BlockMasterSync - // Consequence: worker id is granted - startWorkers(); - - // Start serving the web server, this will not block. - mWebServer.start(); - - // Start monitor jvm - if (Configuration.getBoolean(PropertyKey.WORKER_JVM_MONITOR_ENABLED)) { - mJvmPauseMonitor = - new JvmPauseMonitor( - Configuration.getMs(PropertyKey.JVM_MONITOR_SLEEP_INTERVAL_MS), - Configuration.getMs(PropertyKey.JVM_MONITOR_WARN_THRESHOLD_MS), - Configuration.getMs(PropertyKey.JVM_MONITOR_INFO_THRESHOLD_MS)); - mJvmPauseMonitor.start(); - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.TOTAL_EXTRA_TIME.getName()), - mJvmPauseMonitor::getTotalExtraTime); - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.INFO_TIME_EXCEEDED.getName()), - mJvmPauseMonitor::getInfoTimeExceeded); - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.WARN_TIME_EXCEEDED.getName()), - mJvmPauseMonitor::getWarnTimeExceeded); - } - - // Start serving RPC, this will block - LOG.info("Alluxio worker started. id={}, bindHost={}, connectHost={}, rpcPort={}, webPort={}", - mRegistry.get(BlockWorker.class).getWorkerId(), - NetworkAddressUtils.getBindHost(ServiceType.WORKER_RPC, Configuration.global()), - NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, Configuration.global()), - NetworkAddressUtils.getPort(ServiceType.WORKER_RPC, Configuration.global()), - NetworkAddressUtils.getPort(ServiceType.WORKER_WEB, Configuration.global())); - - mDataServer.awaitTermination(); - - LOG.info("Alluxio worker ended"); - } - - @Override - public void stop() throws Exception { - if (isServing()) { - stopServing(); - if (mJvmPauseMonitor != null) { - mJvmPauseMonitor.stop(); - } - } - stopWorkers(); - } - - private boolean isServing() { - return mDataServer != null && !mDataServer.isClosed(); - } - - private void startWorkers() throws Exception { - mRegistry.start(getAddress()); - } - - private void stopWorkers() throws Exception { - mRegistry.stop(); - } - - private void stopServing() throws Exception { - mDataServer.close(); - if (mDomainSocketDataServer != null) { - mDomainSocketDataServer.close(); - mDomainSocketDataServer = null; - } - mUfsManager.close(); - try { - mWebServer.stop(); - } catch (Exception e) { - LOG.error("Failed to stop {} web server", this, e); - } - MetricsSystem.stopSinks(); - } - - /** - * @return true if domain socket is enabled - */ - private boolean isDomainSocketEnabled() { - return NettyUtils.getWorkerChannel(Configuration.global()) == ChannelType.EPOLL - && Configuration.isSet(PropertyKey.WORKER_DATA_SERVER_DOMAIN_SOCKET_ADDRESS); - } - - @Override - public boolean waitForReady(int timeoutMs) { - try { - CommonUtils.waitFor(this + " to start", - () -> isServing() && mRegistry.get(BlockWorker.class).getWorkerId() != null - && mWebServer != null && mWebServer.getServer().isRunning(), - WaitForOptions.defaults().setTimeoutMs(timeoutMs)); - return true; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } catch (TimeoutException e) { - return false; - } - } - - @Override - public WorkerNetAddress getAddress() { - return new WorkerNetAddress() - .setHost(NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, - Configuration.global())) - .setContainerHost(Configuration.global() - .getOrDefault(PropertyKey.WORKER_CONTAINER_HOSTNAME, "")) - .setRpcPort(mRpcBindAddress.getPort()) - .setDataPort(getDataLocalPort()) - .setDomainSocketPath(getDataDomainSocketPath()) - .setWebPort(mWebServer.getLocalPort()) - .setTieredIdentity(mTieredIdentitiy); - } - - @Override - public String toString() { - return "Alluxio worker @" + mRpcConnectAddress; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/AlluxioWorkerRestServiceHandler.java b/core/server/worker/src/main/java/alluxio/worker/AlluxioWorkerRestServiceHandler.java deleted file mode 100644 index 602feb6bac11..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/AlluxioWorkerRestServiceHandler.java +++ /dev/null @@ -1,628 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.RestUtils; -import alluxio.RuntimeConstants; -import alluxio.client.file.FileSystem; -import alluxio.client.file.URIStatus; -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.ConfigurationValueOptions; -import alluxio.conf.PropertyKey; -import alluxio.exception.AlluxioException; -import alluxio.exception.FileDoesNotExistException; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.grpc.ConfigProperty; -import alluxio.grpc.GetConfigurationPOptions; -import alluxio.master.block.BlockId; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.util.ConfigurationUtils; -import alluxio.util.FormatUtils; -import alluxio.util.LogUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.util.webui.UIFileBlockInfo; -import alluxio.util.webui.UIFileInfo; -import alluxio.util.webui.UIStorageDir; -import alluxio.util.webui.UIUsageOnTier; -import alluxio.util.webui.UIWorkerInfo; -import alluxio.util.webui.WebUtils; -import alluxio.web.WorkerWebServer; -import alluxio.wire.AlluxioWorkerInfo; -import alluxio.wire.Capacity; -import alluxio.wire.WorkerWebUIBlockInfo; -import alluxio.wire.WorkerWebUIConfiguration; -import alluxio.wire.WorkerWebUIInit; -import alluxio.wire.WorkerWebUILogs; -import alluxio.wire.WorkerWebUIMetrics; -import alluxio.wire.WorkerWebUIOverview; -import alluxio.worker.block.BlockStoreMeta; -import alluxio.worker.block.BlockWorker; -import alluxio.worker.block.DefaultBlockWorker; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Metric; -import com.codahale.metrics.MetricRegistry; -import com.google.common.collect.Sets; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.commons.lang3.tuple.Triple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.TreeSet; -import javax.annotation.concurrent.NotThreadSafe; -import javax.servlet.ServletContext; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -/** - * This class is a REST handler for requesting general worker information. - */ -@NotThreadSafe -@Api(value = "/worker", description = "Alluxio Worker Rest Service") -@Path(AlluxioWorkerRestServiceHandler.SERVICE_PREFIX) -@Produces(MediaType.APPLICATION_JSON) -public final class AlluxioWorkerRestServiceHandler { - private static final Logger LOG = LoggerFactory.getLogger(AlluxioWorkerRestServiceHandler.class); - - public static final String SERVICE_PREFIX = "worker"; - - // endpoints - public static final String GET_INFO = "info"; - - // webui endpoints // TODO(william): DRY up these enpoints - public static final String WEBUI_INIT = "webui_init"; - public static final String WEBUI_OVERVIEW = "webui_overview"; - public static final String WEBUI_LOGS = "webui_logs"; - public static final String WEBUI_BLOCKINFO = "webui_blockinfo"; - public static final String WEBUI_METRICS = "webui_metrics"; - public static final String WEBUI_CONFIG = "webui_config"; - - // queries - public static final String QUERY_RAW_CONFIGURATION = "raw_configuration"; - - // log - public static final String LOG_LEVEL = "logLevel"; - public static final String LOG_ARGUMENT_NAME = "logName"; - public static final String LOG_ARGUMENT_LEVEL = "level"; - - private final WorkerProcess mWorkerProcess; - private final BlockStoreMeta mStoreMeta; - private final DefaultBlockWorker mBlockWorker; - private final FileSystem mFsClient; - - /** - * @param context context for the servlet - */ - public AlluxioWorkerRestServiceHandler(@Context ServletContext context) { - mWorkerProcess = - (WorkerProcess) context.getAttribute(WorkerWebServer.ALLUXIO_WORKER_SERVLET_RESOURCE_KEY); - mBlockWorker = (DefaultBlockWorker) mWorkerProcess.getWorker(BlockWorker.class); - mStoreMeta = mBlockWorker.getStoreMeta(); - mFsClient = - (FileSystem) context.getAttribute(WorkerWebServer.ALLUXIO_FILESYSTEM_CLIENT_RESOURCE_KEY); - } - - /** - * @summary get the Alluxio master information - * @param rawConfiguration if it's true, raw configuration values are returned, - * otherwise, they are looked up; if it's not provided in URL queries, then - * it is null, which means false. - * @return the response object - */ - @GET - @Path(GET_INFO) - @ApiOperation(value = "Get general Alluxio Worker service information", - response = alluxio.wire.AlluxioWorkerInfo.class) - public Response getInfo(@QueryParam(QUERY_RAW_CONFIGURATION) final Boolean rawConfiguration) { - // TODO(jiri): Add a mechanism for retrieving only a subset of the fields. - return RestUtils.call(() -> { - boolean rawConfig = false; - if (rawConfiguration != null) { - rawConfig = rawConfiguration; - } - return new AlluxioWorkerInfo().setCapacity(getCapacityInternal()) - .setConfiguration(getConfigurationInternal(rawConfig)).setMetrics(getMetricsInternal()) - .setRpcAddress(mWorkerProcess.getRpcAddress().toString()) - .setStartTimeMs(mWorkerProcess.getStartTimeMs()) - .setTierCapacity(getTierCapacityInternal()).setTierPaths(getTierPathsInternal()) - .setUptimeMs(mWorkerProcess.getUptimeMs()).setVersion(RuntimeConstants.VERSION); - }, Configuration.global()); - } - - /** - * Gets Web UI initialization data. - * - * @return the response object - */ - @GET - @Path(WEBUI_INIT) - public Response getWebUIInit() { - return RestUtils.call(() -> { - WorkerWebUIInit response = new WorkerWebUIInit(); - - response.setDebug(Configuration.getBoolean(PropertyKey.DEBUG)) - .setWebFileInfoEnabled(Configuration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) - .setSecurityAuthorizationPermissionEnabled( - Configuration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED)) - .setMasterHostname(NetworkAddressUtils - .getConnectHost(NetworkAddressUtils.ServiceType.MASTER_WEB, - Configuration.global())) - .setMasterPort(Configuration.getInt(PropertyKey.MASTER_WEB_PORT)) - .setRefreshInterval((int) Configuration.getMs(PropertyKey.WEB_REFRESH_INTERVAL)); - - return response; - }, Configuration.global()); - } - - /** - * Gets web ui overview page data. - * - * @return the response object - */ - @GET - @Path(WEBUI_OVERVIEW) - public Response getWebUIOverview() { - return RestUtils.call(() -> { - WorkerWebUIOverview response = new WorkerWebUIOverview(); - - response.setWorkerInfo(new UIWorkerInfo(mWorkerProcess.getRpcAddress().toString(), - mWorkerProcess.getStartTimeMs(), - Configuration.getString(PropertyKey.USER_DATE_FORMAT_PATTERN))); - BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull(); - long capacityBytes = 0L; - long usedBytes = 0L; - Map capacityBytesOnTiers = storeMeta.getCapacityBytesOnTiers(); - Map usedBytesOnTiers = storeMeta.getUsedBytesOnTiers(); - List usageOnTiers = new ArrayList<>(); - for (Map.Entry entry : capacityBytesOnTiers.entrySet()) { - String tier = entry.getKey(); - long capacity = entry.getValue(); - Long nullableUsed = usedBytesOnTiers.get(tier); - long used = nullableUsed == null ? 0 : nullableUsed; - - capacityBytes += capacity; - usedBytes += used; - - usageOnTiers.add(new UIUsageOnTier(tier, capacity, used)); - } - - response.setCapacityBytes(FormatUtils.getSizeFromBytes(capacityBytes)) - .setUsedBytes(FormatUtils.getSizeFromBytes(usedBytes)).setUsageOnTiers(usageOnTiers) - .setBlockCount(Long.toString(storeMeta.getNumberOfBlocks())) - .setVersion(RuntimeConstants.VERSION); - - List storageDirs = new ArrayList<>(storeMeta.getCapacityBytesOnDirs().size()); - for (Pair tierAndDirPath : storeMeta.getCapacityBytesOnDirs().keySet()) { - storageDirs.add(new UIStorageDir(tierAndDirPath.getFirst(), tierAndDirPath.getSecond(), - storeMeta.getCapacityBytesOnDirs().get(tierAndDirPath), - storeMeta.getUsedBytesOnDirs().get(tierAndDirPath))); - } - - response.setStorageDirs(storageDirs); - - return response; - }, Configuration.global()); - } - - /** - * Gets web ui block info page data. - * - * @param requestPath the request path - * @param requestOffset the request offset - * @param requestLimit the request limit - * @return the response object - */ - @GET - @Path(WEBUI_BLOCKINFO) - public Response getWebUIBlockInfo(@QueryParam("path") String requestPath, - @DefaultValue("0") @QueryParam("offset") String requestOffset, - @DefaultValue("20") @QueryParam("limit") String requestLimit) { - return RestUtils.call(() -> { - WorkerWebUIBlockInfo response = new WorkerWebUIBlockInfo(); - - if (!Configuration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) { - return response; - } - response.setFatalError("").setInvalidPathError(""); - if (!(requestPath == null || requestPath.isEmpty())) { - // Display file block info - try { - URIStatus status = mFsClient.getStatus(new AlluxioURI(requestPath)); - UIFileInfo uiFileInfo = new UIFileInfo(status, Configuration.global(), - mStoreMeta.getStorageTierAssoc().getOrderedStorageAliases()); - for (long blockId : status.getBlockIds()) { - // The block last access time is not available. Use -1 for now. - // It's not necessary to show location information here since - // we are viewing at the context of this worker. - mBlockWorker.getBlockStore().getVolatileBlockMeta(blockId) - .ifPresent(meta -> uiFileInfo.addBlock(meta.getBlockLocation().tierAlias(), - blockId, meta.getBlockSize(), -1)); - } - List>> fileBlocksOnTier = new ArrayList<>(); - for (Map.Entry> e : uiFileInfo.getBlocksOnTier() - .entrySet()) { - fileBlocksOnTier.add(new ImmutablePair<>(e.getKey(), e.getValue())); - } - - response.setFileBlocksOnTier(fileBlocksOnTier) - .setBlockSizeBytes(uiFileInfo.getBlockSizeBytes()).setPath(requestPath); - } catch (FileDoesNotExistException e) { - response.setFatalError("Error: Invalid Path " + e.getMessage()); - } catch (IOException e) { - response.setInvalidPathError( - "Error: File " + requestPath + " is not available " + e.getMessage()); - } catch (AlluxioException e) { - response.setFatalError("Error: alluxio exception. " + e.getMessage()); - } - catch (AlluxioRuntimeException e) { - response.setFatalError("Error: alluxio run time exception. " + e.getMessage()); - } - } - - Set unsortedFileIds = new HashSet<>(); - BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull(); - for (List blockIds : storeMeta.getBlockList().values()) { - for (long blockId : blockIds) { - unsortedFileIds.add(BlockId.getFileId(blockId)); - } - } - List fileIds = new ArrayList<>(unsortedFileIds); - Collections.sort(fileIds); - response.setNTotalFile(unsortedFileIds.size()) - .setOrderedTierAliases(mStoreMeta.getStorageTierAssoc().getOrderedStorageAliases()); - - try { - int offset = Integer.parseInt(requestOffset); - int limit = Integer.parseInt(requestLimit); - // make the limit the total number of files if request limit is > than what is available - limit = offset == 0 && limit > fileIds.size() ? fileIds.size() : limit; - // offset+limit can't be greater than the size of the list - limit = offset + limit > fileIds.size() ? fileIds.size() - offset : limit; - int sum = Math.addExact(offset, limit); - List subFileIds = fileIds.subList(offset, sum); - List uiFileInfos = new ArrayList<>(subFileIds.size()); - for (long fileId : subFileIds) { - try { - URIStatus status = new URIStatus(mBlockWorker.getFileInfo(fileId)); - UIFileInfo uiFileInfo = new UIFileInfo(status, Configuration.global(), - mStoreMeta.getStorageTierAssoc().getOrderedStorageAliases()); - for (long blockId : status.getBlockIds()) { - // The block last access time is not available. Use -1 for now. - // It's not necessary to show location information here since - // we are viewing at the context of this worker. - mBlockWorker.getBlockStore().getVolatileBlockMeta(blockId) - .ifPresent(meta -> uiFileInfo.addBlock(meta.getBlockLocation().tierAlias(), - blockId, meta.getBlockSize(), -1)); - } - if (!uiFileInfo.getBlockIds().isEmpty()) { - uiFileInfos.add(uiFileInfo); - } - } catch (Exception e) { - // The file might have been deleted, log a warning and ignore this file. - LOG.warn("Unable to get file info for fileId {}. {}", fileId, e.toString()); - } - } - response.setFileInfos(uiFileInfos); - } catch (NumberFormatException e) { - response.setFatalError("Error: offset or limit parse error, " + e.getLocalizedMessage()); - } catch (ArithmeticException e) { - response.setFatalError( - "Error: offset or offset + limit is out ofbound, " + e.getLocalizedMessage()); - } catch (Exception e) { - response.setFatalError(e.getLocalizedMessage()); - } - - return response; - }, Configuration.global()); - } - - /** - * Gets web ui metrics page data. - * - * @return the response object - */ - @GET - @Path(WEBUI_METRICS) - public Response getWebUIMetrics() { - return RestUtils.call(() -> { - WorkerWebUIMetrics response = new WorkerWebUIMetrics(); - - MetricRegistry mr = MetricsSystem.METRIC_REGISTRY; - SortedMap gauges = mr.getGauges(); - - Long workerCapacityTotal = (Long) gauges.get(MetricsSystem - .getMetricName(MetricKey.WORKER_CAPACITY_TOTAL.getName())).getValue(); - Long workerCapacityUsed = (Long) gauges.get(MetricsSystem - .getMetricName(MetricKey.WORKER_CAPACITY_USED.getName())).getValue(); - - int workerCapacityUsedPercentage = - (workerCapacityTotal > 0) ? (int) (100L * workerCapacityUsed / workerCapacityTotal) : 0; - - response.setWorkerCapacityUsedPercentage(workerCapacityUsedPercentage); - response.setWorkerCapacityFreePercentage(100L - workerCapacityUsedPercentage); - - Map operations = new TreeMap<>(); - // Remove the instance name from the metrics. - for (Map.Entry entry : mr.getCounters().entrySet()) { - operations.put(MetricsSystem.stripInstanceAndHost(entry.getKey()), entry.getValue()); - } - - response.setOperationMetrics(operations); - - return response; - }, Configuration.global()); - } - - /** - * Gets web ui logs page data. - * - * @param requestPath the request path - * @param requestOffset the request offset - * @param requestEnd the request end - * @param requestLimit the request limit - * @return the response object - */ - @GET - @Path(WEBUI_LOGS) - public Response getWebUILogs(@DefaultValue("") @QueryParam("path") String requestPath, - @DefaultValue("0") @QueryParam("offset") String requestOffset, - @QueryParam("end") String requestEnd, - @DefaultValue("20") @QueryParam("limit") String requestLimit) { - return RestUtils.call(() -> { - FilenameFilter filenameFilter = (dir, name) -> name.toLowerCase().endsWith(".log"); - WorkerWebUILogs response = new WorkerWebUILogs(); - - if (!Configuration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) { - return response; - } - response.setDebug(Configuration.getBoolean(PropertyKey.DEBUG)).setInvalidPathError("") - .setViewingOffset(0).setCurrentPath(""); - - String logsPath = Configuration.getString(PropertyKey.LOGS_DIR); - File logsDir = new File(logsPath); - String requestFile = requestPath; - - if (requestFile == null || requestFile.isEmpty()) { - // List all log files in the log/ directory. - - List fileInfos = new ArrayList<>(); - File[] logFiles = logsDir.listFiles(filenameFilter); - if (logFiles != null) { - for (File logFile : logFiles) { - String logFileName = logFile.getName(); - fileInfos.add(new UIFileInfo( - new UIFileInfo.LocalFileInfo(logFileName, logFileName, logFile.length(), - UIFileInfo.LocalFileInfo.EMPTY_CREATION_TIME, logFile.lastModified(), - logFile.isDirectory()), Configuration.global(), - mStoreMeta.getStorageTierAssoc().getOrderedStorageAliases())); - } - } - Collections.sort(fileInfos, UIFileInfo.PATH_STRING_COMPARE); - response.setNTotalFile(fileInfos.size()); - - try { - int offset = Integer.parseInt(requestOffset); - int limit = Integer.parseInt(requestLimit); - limit = offset == 0 && limit > fileInfos.size() ? fileInfos.size() : limit; - limit = offset + limit > fileInfos.size() ? fileInfos.size() - offset : limit; - int sum = Math.addExact(offset, limit); - fileInfos = fileInfos.subList(offset, sum); - response.setFileInfos(fileInfos); - } catch (NumberFormatException e) { - response.setFatalError("Error: offset or limit parse error, " + e.getLocalizedMessage()); - return response; - } catch (ArithmeticException e) { - response.setFatalError( - "Error: offset or offset + limit is out of bound, " + e.getLocalizedMessage()); - return response; - } catch (IllegalArgumentException e) { - response.setFatalError(e.getLocalizedMessage()); - return response; - } - } else { - // Request a specific log file. - - // Only allow filenames as the path, to avoid arbitrary local path lookups. - requestFile = new File(requestFile).getName(); - response.setCurrentPath(requestFile); - - File logFile = new File(logsDir, requestFile); - - try { - long fileSize = logFile.length(); - long relativeOffset = 0; - long offset; - try { - if (requestOffset != null) { - relativeOffset = Long.parseLong(requestOffset); - } - } catch (NumberFormatException e) { - // pass - } - // If no param "end" presents, the offset is relative to the beginning; otherwise, it is - // relative to the end of the file. - if (requestEnd == null) { - offset = relativeOffset; - } else { - offset = fileSize - relativeOffset; - } - if (offset < 0) { - offset = 0; - } else if (offset > fileSize) { - offset = fileSize; - } - - String fileData; - try (InputStream is = new FileInputStream(logFile)) { - fileSize = logFile.length(); - int len = (int) Math.min(5L * Constants.KB, fileSize - offset); - byte[] data = new byte[len]; - long skipped = is.skip(offset); - if (skipped < 0) { - // Nothing was skipped. - fileData = "Unable to traverse to offset; is file empty?"; - } else if (skipped < offset) { - // Couldn't skip all the way to offset. - fileData = "Unable to traverse to offset; is offset larger than the file?"; - } else { - // Read may not read up to len, so only convert what was read. - int read = is.read(data, 0, len); - if (read < 0) { - // Stream couldn't read anything, skip went to EOF? - fileData = "Unable to read file"; - } else { - fileData = WebUtils.convertByteArrayToStringWithoutEscape(data, 0, read); - } - } - } - response.setFileData(fileData).setViewingOffset(offset); - } catch (IOException e) { - response.setInvalidPathError( - "Error: File " + logFile + " is not available " + e.getMessage()); - } - } - - return response; - }, Configuration.global()); - } - - /** - * Gets Web UI Configuration page data. - * - * @return the response object - */ - @GET - @Path(WEBUI_CONFIG) - public Response getWebUIConfiguration() { - return RestUtils.call(() -> { - WorkerWebUIConfiguration response = new WorkerWebUIConfiguration(); - response.setWhitelist(mBlockWorker.getWhiteList()); - - TreeSet> sortedProperties = new TreeSet<>(); - Set alluxioConfExcludes = Sets.newHashSet(PropertyKey.WORKER_WHITELIST.toString()); - for (ConfigProperty configProperty : mBlockWorker - .getConfiguration(GetConfigurationPOptions.newBuilder().setRawValue(true).build()) - .toProto().getClusterConfigsList()) { - String confName = configProperty.getName(); - if (!alluxioConfExcludes.contains(confName)) { - sortedProperties.add(new ImmutableTriple<>(confName, - ConfigurationUtils.valueAsString(configProperty.getValue()), - configProperty.getSource())); - } - } - - response.setConfiguration(sortedProperties); - - return response; - }, Configuration.global()); - } - - private Capacity getCapacityInternal() { - return new Capacity().setTotal(mStoreMeta.getCapacityBytes()) - .setUsed(mStoreMeta.getUsedBytes()); - } - - private Map getConfigurationInternal(boolean raw) { - return new TreeMap<>(Configuration - .toMap(ConfigurationValueOptions.defaults().useDisplayValue(true).useRawValue(raw))); - } - - private Map getMetricsInternal() { - MetricRegistry metricRegistry = MetricsSystem.METRIC_REGISTRY; - - // Get all counters. - Map counters = metricRegistry.getCounters(); - - // Only the gauge for cached blocks is retrieved here, other gauges are statistics of - // free/used spaces, those statistics can be gotten via other REST apis. - String blocksCachedProperty = - MetricsSystem.getMetricName(MetricKey.WORKER_BLOCKS_CACHED.getName()); - @SuppressWarnings("unchecked") Gauge blocksCached = - (Gauge) metricRegistry.getGauges().get(blocksCachedProperty); - - // Get values of the counters and gauges and put them into a metrics map. - SortedMap metrics = new TreeMap<>(); - for (Map.Entry counter : counters.entrySet()) { - metrics.put(counter.getKey(), counter.getValue().getCount()); - } - metrics.put(blocksCachedProperty, blocksCached.getValue().longValue()); - - return metrics; - } - - private Comparator getTierAliasComparator() { - return (tier1, tier2) -> { - int ordinal1 = mStoreMeta.getStorageTierAssoc().getOrdinal(tier1); - int ordinal2 = mStoreMeta.getStorageTierAssoc().getOrdinal(tier2); - return Integer.compare(ordinal1, ordinal2); - }; - } - - private Map getTierCapacityInternal() { - SortedMap tierCapacity = new TreeMap<>(getTierAliasComparator()); - Map capacityBytesOnTiers = mStoreMeta.getCapacityBytesOnTiers(); - Map usedBytesOnTiers = mStoreMeta.getUsedBytesOnTiers(); - for (Map.Entry entry : capacityBytesOnTiers.entrySet()) { - tierCapacity.put(entry.getKey(), - new Capacity().setTotal(entry.getValue()).setUsed(usedBytesOnTiers.get(entry.getKey()))); - } - return tierCapacity; - } - - private Map> getTierPathsInternal() { - SortedMap> tierToDirPaths = new TreeMap<>(getTierAliasComparator()); - tierToDirPaths.putAll(mStoreMeta.getDirectoryPathsOnTiers()); - return tierToDirPaths; - } - - /** - * @summary set the Alluxio log information - * @param logName the log's name - * @param level the log level - * @return the response object - */ - @POST - @Path(LOG_LEVEL) - public Response logLevel(@QueryParam(LOG_ARGUMENT_NAME) final String logName, - @QueryParam(LOG_ARGUMENT_LEVEL) final String level) { - return RestUtils.call(() -> LogUtils.setLogLevel(logName, level), Configuration.global()); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/WorkerProcess.java b/core/server/worker/src/main/java/alluxio/worker/WorkerProcess.java deleted file mode 100644 index b679917de586..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/WorkerProcess.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker; - -import alluxio.Process; -import alluxio.conf.Configuration; -import alluxio.network.TieredIdentityFactory; -import alluxio.underfs.UfsManager; -import alluxio.wire.TieredIdentity; -import alluxio.wire.WorkerNetAddress; - -import java.net.InetSocketAddress; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A worker in the Alluxio system. - */ -public interface WorkerProcess extends Process { - /** - * Factory for creating {@link WorkerProcess}. - */ - @ThreadSafe - final class Factory { - /** - * @return a new instance of {@link WorkerProcess} - */ - public static WorkerProcess create() { - return create(TieredIdentityFactory.localIdentity(Configuration.global())); - } - - /** - * @param tieredIdentity tiered identity for the worker process - * @return a new instance of {@link WorkerProcess} - */ - public static WorkerProcess create(TieredIdentity tieredIdentity) { - return new AlluxioWorkerProcess(tieredIdentity); - } - - private Factory() {} // prevent instantiation - } - - /** - * @return the connect information for this worker - */ - WorkerNetAddress getAddress(); - - /** - * @return the block worker for this Alluxio worker - */ - UfsManager getUfsManager(); - - /** - * @return the worker's data service bind host (used by unit test only) - */ - String getDataBindHost(); - - /** - * @return the worker's data service port (used by unit test only) - */ - int getDataLocalPort(); - - /** - * @return the worker's data service domain socket path if available or "" if not available - */ - String getDataDomainSocketPath(); - - /** - * @return this worker's rpc address - */ - InetSocketAddress getRpcAddress(); - - /** - * @return the start time of the worker in milliseconds - */ - long getStartTimeMs(); - - /** - * @return the uptime of the worker in milliseconds - */ - long getUptimeMs(); - - /** - * @return the worker web service bind host (used by unit test only) - */ - String getWebBindHost(); - - /** - * @return the worker web service port (used by unit test only) - */ - int getWebLocalPort(); - - /** - * @param clazz the class of the worker to get - * @param the type of the worker to get - - * @return the given worker - */ - T getWorker(Class clazz); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/AbstractBlockStoreEventListener.java b/core/server/worker/src/main/java/alluxio/worker/block/AbstractBlockStoreEventListener.java deleted file mode 100644 index dc99e9406b7d..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/AbstractBlockStoreEventListener.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A block store event listener base class implementing the {@link BlockStoreEventListener} - * interface with empty methods bodies. - */ -@NotThreadSafe -public abstract class AbstractBlockStoreEventListener implements BlockStoreEventListener { - - @Override - public void onAccessBlock(long blockId) {} - - @Override - public void onAccessBlock(long blockId, BlockStoreLocation location) {} - - @Override - public void onAbortBlock(long blockId) {} - - @Override - public void onCommitBlock(long blockId, BlockStoreLocation location) {} - - @Override - public void onMoveBlockByClient(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) {} - - @Override - public void onMoveBlockByWorker(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) {} - - @Override - public void onRemoveBlockByClient(long blockId) {} - - @Override - public void onRemoveBlockByWorker(long blockId) {} - - @Override - public void onRemoveBlock(long blockId, BlockStoreLocation location) {} - - @Override - public void onBlockLost(long blockId) {} - - @Override - public void onStorageLost(String tierAlias, String dirPath) {} - - @Override - public void onStorageLost(BlockStoreLocation dirLocation) {} -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/AsyncBlockRemover.java b/core/server/worker/src/main/java/alluxio/worker/block/AsyncBlockRemover.java deleted file mode 100644 index 0ac70439f13c..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/AsyncBlockRemover.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.Sessions; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.util.ThreadFactoryUtils; - -import com.codahale.metrics.Counter; -import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Asynchronous block removal service. - */ -@ThreadSafe -public class AsyncBlockRemover { - private static final Logger LOG = LoggerFactory.getLogger(AsyncBlockRemover.class); - - private static final int DEFAULT_BLOCK_REMOVER_POOL_SIZE = 10; - - private final BlockWorker mBlockWorker; - /** This list is used for queueing blocks to be removed by BlockWorker. */ - private final BlockingQueue mBlocksToRemove; - /** This set is used for recording blocks in BlockRemover. */ - private final Set mRemovingBlocks; - private final ExecutorService mRemoverPool; - private final Counter mTryRemoveCount; - private final Counter mRemovedCount; - private volatile boolean mShutdown = false; - - /** - * Constructor of AsyncBlockRemover. - * @param worker block worker - */ - public AsyncBlockRemover(BlockWorker worker) { - this(worker, DEFAULT_BLOCK_REMOVER_POOL_SIZE, new LinkedBlockingQueue<>(), - Collections.newSetFromMap(new ConcurrentHashMap<>())); - } - - /** - * Constructor of AsyncBlockRemover. - * - * @param worker block worker - * @param threads number of threads - * @param blocksToRemove blocks to remove - * @param removingBlocks blocks being removed - */ - @VisibleForTesting - public AsyncBlockRemover(BlockWorker worker, int threads, BlockingQueue blocksToRemove, - Set removingBlocks) { - mBlockWorker = worker; - mBlocksToRemove = blocksToRemove; - mRemovingBlocks = removingBlocks; - mTryRemoveCount = MetricsSystem.counter( - MetricKey.WORKER_BLOCK_REMOVER_TRY_REMOVE_COUNT.getName()); - mRemovedCount = MetricsSystem.counter( - MetricKey.WORKER_BLOCK_REMOVER_REMOVED_COUNT.getName()); - MetricsSystem.registerGaugeIfAbsent( - MetricKey.WORKER_BLOCK_REMOVER_TRY_REMOVE_BLOCKS_SIZE.getName(), mBlocksToRemove::size); - MetricsSystem.registerGaugeIfAbsent( - MetricKey.WORKER_BLOCK_REMOVER_REMOVING_BLOCKS_SIZE.getName(), mRemovingBlocks::size); - - mRemoverPool = Executors.newFixedThreadPool(threads, - ThreadFactoryUtils.build("block-removal-service-%d", true)); - for (int i = 0; i < threads; i++) { - mRemoverPool.execute(new BlockRemover()); - } - } - - /** - * Put blocks into async block remover. This method will take care of the duplicate blocks. - * @param blocks blocks to be deleted - */ - public void addBlocksToDelete(List blocks) { - for (long id : blocks) { - if (mRemovingBlocks.contains(id)) { - LOG.debug("{} is being removed. Current queue size is {}.", id, mBlocksToRemove.size()); - continue; - } - try { - mBlocksToRemove.put(id); - mRemovingBlocks.add(id); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.warn("AsyncBlockRemover got interrupted while it was putting block {}.", id); - } - } - } - - /** - * Shutdown async block remover. - */ - public void shutDown() { - mShutdown = true; - mRemoverPool.shutdownNow(); - } - - private class BlockRemover implements Runnable { - @Override - public void run() { - String threadName = Thread.currentThread().getName(); - while (true) { - Long blockToBeRemoved = null; - try { - blockToBeRemoved = mBlocksToRemove.take(); - mTryRemoveCount.inc(); - mBlockWorker.removeBlock(Sessions.MASTER_COMMAND_SESSION_ID, blockToBeRemoved); - mRemovedCount.inc(); - LOG.debug("Block {} is removed in thread {}.", blockToBeRemoved, threadName); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - // Only log warning if interrupted not due to a shutdown. - if (!mShutdown) { - LOG.warn("{} got interrupted while it was cleaning block {}.", threadName, - blockToBeRemoved); - } - break; - } catch (Exception e) { - LOG.warn("Failed to remove block {} instructed by master. This is best-effort and " - + "will be tried later. threadName {}, error {}", blockToBeRemoved, - threadName, e.getMessage()); - } finally { - if (blockToBeRemoved != null) { - mRemovingBlocks.remove(blockToBeRemoved); - } - } - } - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockHeartbeatReporter.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockHeartbeatReporter.java deleted file mode 100644 index 5c0c7b52923c..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockHeartbeatReporter.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import com.google.common.collect.Lists; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Represents the delta of the block store within one heartbeat period. For now, newly committed - * blocks do not pass through this master communication mechanism, instead it is synchronized - * through {@link alluxio.worker.block.BlockWorker#commitBlock(long, long)}. - */ -@ThreadSafe -public final class BlockHeartbeatReporter extends AbstractBlockStoreEventListener { - /** Lock for operations on the removed and added block collections. */ - private final Object mLock; - - /** List of blocks that were removed in the last heartbeat period. */ - private final List mRemovedBlocks; - - /** - * Map of block store locations to a list of blocks that were added in the last - * heartbeat period. - */ - private final Map> mAddedBlocks; - - /** - * Map of storage tier alias to a list of storage paths - * that were lost in the last hearbeat period. - */ - private final Map> mLostStorage; - - /** - * Creates a new instance of {@link BlockHeartbeatReporter}. - */ - public BlockHeartbeatReporter() { - mLock = new Object(); - mRemovedBlocks = new ArrayList<>(100); - mAddedBlocks = new HashMap<>(20); - mLostStorage = new HashMap<>(); - } - - /** - * Generates the report of the block store delta in the last heartbeat period. Calling this method - * marks the end of a period and the start of a new heartbeat period. - * - * @return the block store delta report for the last heartbeat period - */ - public BlockHeartbeatReport generateReport() { - synchronized (mLock) { - BlockHeartbeatReport report - = new BlockHeartbeatReport(mAddedBlocks, mRemovedBlocks, mLostStorage); - // Clear added and removed blocks - mAddedBlocks.clear(); - mRemovedBlocks.clear(); - mLostStorage.clear(); - return report; - } - } - - @Override - public void onMoveBlockByClient(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) { - synchronized (mLock) { - // Remove the block from our list of added blocks in this heartbeat, if it was added, to - // prevent adding the block twice. - removeBlockFromAddedBlocks(blockId); - // Add the block back with the new tier - addBlockToAddedBlocks(blockId, newLocation); - } - } - - @Override - public void onRemoveBlockByClient(long blockId) { - synchronized (mLock) { - removeBlockInternal(blockId); - } - } - - @Override - public void onRemoveBlockByWorker(long blockId) { - synchronized (mLock) { - removeBlockInternal(blockId); - } - } - - @Override - public void onMoveBlockByWorker(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) { - synchronized (mLock) { - // Remove the block from our list of added blocks in this heartbeat, if it was added, to - // prevent adding the block twice. - removeBlockFromAddedBlocks(blockId); - // Add the block back with the new storagedir. - addBlockToAddedBlocks(blockId, newLocation); - } - } - - @Override - public void onBlockLost(long blockId) { - synchronized (mLock) { - removeBlockInternal(blockId); - } - } - - @Override - public void onStorageLost(String tierAlias, String dirPath) { - synchronized (mLock) { - List storagesList = mLostStorage.getOrDefault(tierAlias, new ArrayList<>()); - storagesList.add(dirPath); - mLostStorage.put(tierAlias, storagesList); - } - } - - private void removeBlockInternal(long blockId) { - // Remove the block from list of added blocks, in case it was added in this heartbeat period. - removeBlockFromAddedBlocks(blockId); - // Add to the list of removed blocks in this heartbeat period. - if (!mRemovedBlocks.contains(blockId)) { - mRemovedBlocks.add(blockId); - } - } - - /** - * Adds a block to the list of added blocks in this heartbeat period. - * - * @param blockId the id of the block to add - * @param location BlockStoreLocation containing the blockid - */ - private void addBlockToAddedBlocks(long blockId, BlockStoreLocation location) { - if (mAddedBlocks.containsKey(location)) { - mAddedBlocks.get(location).add(blockId); - } else { - mAddedBlocks.put(location, Lists.newArrayList(blockId)); - } - } - - /** - * Removes the block from the added blocks map, if it exists. - * - * @param blockId the block to remove - */ - private void removeBlockFromAddedBlocks(long blockId) { - Iterator>> iterator = mAddedBlocks.entrySet().iterator(); - while (iterator.hasNext()) { - Entry> entry = iterator.next(); - List blockList = entry.getValue(); - if (blockList.contains(blockId)) { - blockList.remove(blockId); - if (blockList.isEmpty()) { - iterator.remove(); - } - // exit the loop when already find and remove block id from mAddedBlocks - break; - } - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockLockManager.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockLockManager.java deleted file mode 100644 index 2e897e684094..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockLockManager.java +++ /dev/null @@ -1,464 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.collections.IndexDefinition; -import alluxio.collections.IndexedSet; -import alluxio.collections.Pair; -import alluxio.concurrent.ClientRWLock; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.resource.ResourcePool; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import javax.annotation.Nullable; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Handle all block locks. - */ -@ThreadSafe -public final class BlockLockManager { - private static final Logger LOG = LoggerFactory.getLogger(BlockLockManager.class); - - /** The unique id of each lock. */ - private static final AtomicLong LOCK_ID_GEN = new AtomicLong(0); - private static final int MAX_READERS = Configuration.getInt( - PropertyKey.WORKER_TIERED_STORE_BLOCK_LOCK_READERS); - - /** A pool of read write locks. */ - private final ResourcePool mLockPool = new ResourcePool( - Configuration.getInt(PropertyKey.WORKER_TIERED_STORE_BLOCK_LOCKS)) { - @Override - public void close() {} - - @Override - public ClientRWLock createNewResource() { - return new ClientRWLock(MAX_READERS); - } - }; - - /** A map from block id to the read write lock used to guard that block. */ - // todo(bowen): set concurrency level? - private final ConcurrentHashMap mLocks = new ConcurrentHashMap<>(); - - /** - * The permits are effectively unlimited for concurrently adding new records which only - * acquires 1 permit, but session cleanup takes all permits here to be mutually exclusive to - * adding new records. - */ - private static final int SESSION_SEMAPHORE_PERMITS = Integer.MAX_VALUE; - - /** - * A semaphore used to prevent concurrent adding and removing of records when - * session clean-up is in progress. During session cleaning, to avoid race conditions, - * no new lock records should be added to {@link #mLockRecords}, but retrievals and removals - * are OK since the underlying concurrent map handles that atomically. - */ - private final Semaphore mSessionCleaning = new Semaphore(SESSION_SEMAPHORE_PERMITS); - - /** - * Records of locks currently held by clients. New entries added to the set must be protected - * from concurrent removals due to session clean-ups. - */ - private final IndexedSet mLockRecords = - new IndexedSet<>(INDEX_LOCK_ID, INDEX_BLOCK_ID, INDEX_SESSION_ID, INDEX_SESSION_BLOCK_ID); - - private static final IndexDefinition> INDEX_SESSION_BLOCK_ID = - IndexDefinition.ofNonUnique( - lockRecord -> new Pair<>(lockRecord.getSessionId(), lockRecord.getBlockId())); - - private static final IndexDefinition INDEX_BLOCK_ID = - IndexDefinition.ofNonUnique(LockRecord::getBlockId); - - private static final IndexDefinition INDEX_LOCK_ID = - IndexDefinition.ofUnique(LockRecord::getLockId); - - private static final IndexDefinition INDEX_SESSION_ID = - IndexDefinition.ofNonUnique(LockRecord::getSessionId); - - /** - * Constructs a new {@link BlockLockManager}. - */ - public BlockLockManager() {} - - /** - * Locks a block. Note that even if this block does not exist, a closable lock is still returned. - * - * If all {@link PropertyKey#WORKER_TIERED_STORE_BLOCK_LOCKS} are already in use and no lock has - * been allocated for the specified block, this method will need to wait until a lock can be - * acquired from the lock pool. - * - * @param sessionId the session id - * @param blockId the block id - * @param blockLockType {@link BlockLockType#READ} or {@link BlockLockType#WRITE} - * @return closable block lock - */ - public BlockLock acquireBlockLock(long sessionId, long blockId, BlockLockType blockLockType) { - OptionalLong lockId = lockBlockInternal(sessionId, blockId, blockLockType, true, null, null); - Preconditions.checkState(lockId.isPresent(), "lockBlock should always return a lockId"); - return new BlockLock(lockId.getAsLong(), this::unlockBlock); - } - - /** - * Tries to lock a block within the given time. - * Note that even if this block does not exist, a lock id is still returned. - * - * If all {@link PropertyKey#WORKER_TIERED_STORE_BLOCK_LOCKS} are already in use and no lock has - * been allocated for the specified block, this method will need to wait until a lock can be - * acquired from the lock pool. - * - * @param sessionId the session id - * @param blockId the block id - * @param blockLockType {@link BlockLockType#READ} or {@link BlockLockType#WRITE} - * @param time the maximum time to wait for the lock - * @param unit the time unit of the {@code time} argument - * @return lock id or INVALID_LOCK_ID if not able to lock within the given time - */ - public Optional tryAcquireBlockLock(long sessionId, long blockId, - BlockLockType blockLockType, - long time, TimeUnit unit) { - OptionalLong lockId = lockBlockInternal(sessionId, blockId, blockLockType, false, time, unit); - return lockId.isPresent() ? Optional.of(new BlockLock(lockId.getAsLong(), this::unlockBlock)) : - Optional.empty(); - } - - private OptionalLong lockBlockInternal(long sessionId, long blockId, BlockLockType blockLockType, - boolean blocking, @Nullable Long time, @Nullable TimeUnit unit) { - ClientRWLock blockLock = getBlockLock(blockId); - Lock lock = blockLockType == BlockLockType.READ ? blockLock.readLock() : blockLock.writeLock(); - // Make sure the session isn't already holding the block lock. - // todo(bowen): This is a best-effort check and is subject to race condition. - // Remove this check and implement better error signaling or retry. - // When the said race condition occurs, the write lock will block indefinitely until the - // reader releases the lock. In case the reader panics while holding the lock, this will lead - // to a deadlock. This can happen e.g. when worker tries to move a block to a new location while - // a client is actively reading it. - if (blockLockType == BlockLockType.WRITE && sessionHoldsLock(sessionId, blockId)) { - throw new IllegalStateException(String - .format("Session %s attempted to take a write lock on block %s, but the session already" - + " holds a lock on the block", sessionId, blockId)); - } - if (blocking) { - lock.lock(); - } else { - Preconditions.checkNotNull(time, "time"); - Preconditions.checkNotNull(unit, "unit"); - try { - if (!lock.tryLock(time, unit)) { - LOG.warn("Failed to acquire lock for block {} after {} {}. " - + "session: {}, blockLockType: {}, lock reference count = {}", - blockId, time, unit, sessionId, blockLockType, - blockLock.getReferenceCount()); - return OptionalLong.empty(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return OptionalLong.empty(); - } - } - long lockId = LOCK_ID_GEN.getAndIncrement(); - LockRecord record = new LockRecord(sessionId, blockId, lockId, lock); - try { - try { - mSessionCleaning.acquire(); - } catch (InterruptedException e) { - LOG.warn("Interrupted while waiting for session clean up, sessionId={}, blockId={}", - sessionId, blockId); - unlock(lock, blockId); - Thread.currentThread().interrupt(); - return OptionalLong.empty(); - } - mLockRecords.add(record); - mSessionCleaning.release(); - return OptionalLong.of(lockId); - } catch (Throwable e) { - // If an unexpected exception occurs, we should release the lock to be conservative. - mLockRecords.remove(record); - unlock(lock, blockId); - throw e; - } - } - - /** - * @param sessionId the session id to check - * @param blockId the block id to check - * @return whether the specified session holds a lock on the specified block - */ - private boolean sessionHoldsLock(long sessionId, long blockId) { - Set sessionRecords = - mLockRecords.getByField(INDEX_SESSION_BLOCK_ID, new Pair<>(sessionId, blockId)); - return !sessionRecords.isEmpty(); - } - - /** - * Returns the block lock for the given block id, acquiring such a lock if it doesn't exist yet. - * - * If all locks have been allocated, this method will block until one can be acquired. - * - * @param blockId the block id to get the lock for - * @return the block lock - */ - private ClientRWLock getBlockLock(long blockId) { - // Loop until we either find the block lock in the mLocks map, or successfully acquire a new - // block lock from the lock pool. - while (true) { - // Check whether a lock has already been allocated for the block id. - ClientRWLock reuseExistingLock = mLocks.computeIfPresent( - blockId, - (blkid, lock) -> { - lock.addReference(); - return lock; - } - ); - if (reuseExistingLock != null) { - return reuseExistingLock; - } - // Since a block lock hasn't already been allocated, try to acquire a new one from the pool. - // We shouldn't wait indefinitely in acquire because the another lock for this block could be - // allocated to another thread, in which case we could just use that lock. - ClientRWLock newlyAcquiredLock = mLockPool.acquire(1, TimeUnit.SECONDS); - if (newlyAcquiredLock != null) { - int referenceCount = newlyAcquiredLock.getReferenceCount(); - if (referenceCount != 0) { - LOG.error("A block lock was not cleanly released as newly acquired locks should have 0 " - + "references, but got {}", referenceCount); - } - ClientRWLock computed = mLocks.compute(blockId, (id, lock) -> { - // Check if someone else acquired a block lock for blockId while we were acquiring one. - if (lock != null) { - // reuse someone else's lock and release newlyAcquiredLock later - // Instead of releasing it immediately here, we release it outside mLock.compute - // as ResourcePool.release has an internal lock, and may block due to concurrent calls - // to ResourcePool.acquire, thus blocking other access to mLocks if called in - // mLock.compute. - lock.addReference(); - return lock; - } else { - newlyAcquiredLock.addReference(); - return newlyAcquiredLock; - } - }); - if (computed != newlyAcquiredLock) { - // reuse someone else's lock and release the unused lock - mLockPool.release(newlyAcquiredLock); - } - return computed; - } - } - } - - /** - * Releases the lock with the specified lock id. - * - * @param lockId the id of the lock to release - */ - private void unlockBlock(long lockId) { - LockRecord record = mLockRecords.getFirstByField(INDEX_LOCK_ID, lockId); - if (record == null) { - return; - } - // the record may have been removed by someone else - // after we retrieved it, so a check is necessary - if (mLockRecords.remove(record)) { - unlock(record.getLock(), record.getBlockId()); - } - } - - /** - * Validates the lock is hold by the given session for the given block. - * - * @param sessionId the session id - * @param blockId the block id - * @param lockId the lock id - * @return hold or not - */ - @VisibleForTesting - public boolean checkLock(long sessionId, long blockId, long lockId) { - LockRecord record = mLockRecords.getFirstByField(INDEX_LOCK_ID, lockId); - return record != null && record.getSessionId() == sessionId && record.getBlockId() == blockId; - } - - /** - * Cleans up the locks currently hold by a specific session. - * - * @param sessionId the id of the session to cleanup - */ - public void cleanupSession(long sessionId) { - // acquire all permits of the semaphore so that no new lock records can be added - mSessionCleaning.acquireUninterruptibly(SESSION_SEMAPHORE_PERMITS); - try { - Set records = mLockRecords.getByField(INDEX_SESSION_ID, sessionId); - if (records == null) { - return; - } - // NOTE: iterating through an ConcurrentHashSet is not done atomically - // this section must be protected from concurrently adding new records that belong to - // the same session - for (LockRecord record : records) { - mLockRecords.remove(record); - unlock(record.getLock(), record.getBlockId()); - } - } finally { - mSessionCleaning.release(SESSION_SEMAPHORE_PERMITS); - } - } - - /** - * Gets a snapshot of currently locked blocks. - * - * @return a set of locked blocks - */ - public Set getLockedBlocks() { - Set set = new HashSet<>(); - // NOTE: iterating through an IndexedSet is not done atomically - // we may end up with stale information in the resulting set - // but the set is merely meant to be a snapshot of the locked blocks - // and is stale as soon as this method returns - for (LockRecord lockRecord : mLockRecords) { - set.add(lockRecord.getBlockId()); - } - return set; - } - - /** - * Unlocks the given lock and releases the block lock for the given block id if the lock no longer - * in use. - * - * @param lock the lock to unlock - * @param blockId the block id for which to potentially release the block lock - */ - private void unlock(Lock lock, long blockId) { - lock.unlock(); - releaseBlockLockIfUnused(blockId); - } - - /** - * Checks whether anyone is using the block lock for the given block id, returning the lock to - * the lock pool if it is unused. - * - * @param blockId the block id for which to potentially release the block lock - */ - private void releaseBlockLockIfUnused(long blockId) { - mLocks.computeIfPresent( - blockId, - (blkid, lock) -> { - // If we were the last worker with a reference to the lock, clean it up. - if (lock.dropReference() == 0) { - mLockPool.release(lock); - return null; - } - return lock; - } - ); - } - - /** - * Checks the internal state of the manager to make sure invariants hold. - * - * This method is intended for testing purposes. A runtime exception will be thrown if invalid - * state is encountered. This method should only be called when there are no other concurrent - * threads accessing this manager. - */ - @VisibleForTesting - public void validate() { - // Compute block lock reference counts based off of lock records - ConcurrentMap blockLockReferenceCounts = new ConcurrentHashMap<>(); - // NOTE: iterating through an IndexedSet is not done atomically - // the counts are valid only when the caller ensures no concurrent access from other threads - for (LockRecord record : mLockRecords) { - blockLockReferenceCounts.putIfAbsent(record.getBlockId(), new AtomicInteger(0)); - blockLockReferenceCounts.get(record.getBlockId()).incrementAndGet(); - } - - // Check that the reference count for each block lock matches the lock record counts. - for (Entry entry : mLocks.entrySet()) { - long blockId = entry.getKey(); - ClientRWLock lock = entry.getValue(); - Integer recordCount = blockLockReferenceCounts.get(blockId).get(); - Integer referenceCount = lock.getReferenceCount(); - if (!Objects.equal(recordCount, referenceCount)) { - throw new IllegalStateException("There are " + recordCount + " lock records for block" - + " id " + blockId + ", but the reference count is " + referenceCount); - } - } - } - - /** - * Inner class to keep record of a lock. - */ - @ThreadSafe - private static final class LockRecord { - private final long mSessionId; - private final long mBlockId; - private final long mLockId; - private final Lock mLock; - - /** Creates a new instance of {@link LockRecord}. - * - * @param sessionId the session id - * @param blockId the block id - * @param lock the lock - */ - LockRecord(long sessionId, long blockId, long lockId, Lock lock) { - mSessionId = sessionId; - mBlockId = blockId; - mLockId = lockId; - mLock = lock; - } - - /** - * @return the session id - */ - long getSessionId() { - return mSessionId; - } - - /** - * @return the block id - */ - long getBlockId() { - return mBlockId; - } - - /** - * @return the lock id - */ - long getLockId() { - return mLockId; - } - - /** - * @return the lock - */ - Lock getLock() { - return mLock; - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockLockType.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockLockType.java deleted file mode 100644 index 9bd0e942ac3f..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockLockType.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * A read-write lock to guard one block. There should be only one lock per block. - */ -@ThreadSafe -public enum BlockLockType { - READ(0), // A read lock - WRITE(1), // A write lock - ; - - private final int mValue; - - BlockLockType(int value) { - mValue = value; - } - - /** - * @return the value - */ - public int getValue() { - return mValue; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterClient.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterClient.java deleted file mode 100644 index ced8f6efe6db..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterClient.java +++ /dev/null @@ -1,362 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.AbstractMasterClient; -import alluxio.Constants; -import alluxio.ProjectConstants; -import alluxio.conf.PropertyKey; -import alluxio.exception.FailedToAcquireRegisterLeaseException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.grpc.BlockHeartbeatPOptions; -import alluxio.grpc.BlockHeartbeatPRequest; -import alluxio.grpc.BlockIdList; -import alluxio.grpc.BlockMasterWorkerServiceGrpc; -import alluxio.grpc.BlockStoreLocationProto; -import alluxio.grpc.BuildVersion; -import alluxio.grpc.Command; -import alluxio.grpc.CommitBlockInUfsPRequest; -import alluxio.grpc.CommitBlockPRequest; -import alluxio.grpc.ConfigProperty; -import alluxio.grpc.GetRegisterLeasePRequest; -import alluxio.grpc.GetRegisterLeasePResponse; -import alluxio.grpc.GetWorkerIdPRequest; -import alluxio.grpc.GrpcUtils; -import alluxio.grpc.LocationBlockIdListEntry; -import alluxio.grpc.Metric; -import alluxio.grpc.RegisterWorkerPOptions; -import alluxio.grpc.RegisterWorkerPRequest; -import alluxio.grpc.ServiceType; -import alluxio.grpc.StorageList; -import alluxio.master.MasterClientContext; -import alluxio.retry.RetryPolicy; -import alluxio.wire.WorkerNetAddress; - -import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A wrapper for the gRPC client to interact with the block master, used by alluxio worker. - *

- */ -@ThreadSafe -public class BlockMasterClient extends AbstractMasterClient { - private static final Logger LOG = LoggerFactory.getLogger(BlockMasterClient.class); - public BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceBlockingStub mClient = null; - public BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub mAsyncClient = null; - - /** - * Creates a new instance of {@link BlockMasterClient} for the worker. - * - * @param conf master client configuration - */ - public BlockMasterClient(MasterClientContext conf) { - super(conf); - } - - @Override - protected ServiceType getRemoteServiceType() { - return ServiceType.BLOCK_MASTER_WORKER_SERVICE; - } - - @Override - protected String getServiceName() { - return Constants.BLOCK_MASTER_WORKER_SERVICE_NAME; - } - - @Override - protected long getServiceVersion() { - return Constants.BLOCK_MASTER_WORKER_SERVICE_VERSION; - } - - @Override - protected void afterConnect() { - mClient = BlockMasterWorkerServiceGrpc.newBlockingStub(mChannel); - mAsyncClient = BlockMasterWorkerServiceGrpc.newStub(mChannel); - } - - /** - * Commits a block on a worker. - * - * @param workerId the worker id committing the block - * @param usedBytesOnTier the amount of used bytes on the tier the block is committing to - * @param tierAlias the alias of the tier the block is being committed to - * @param mediumType the medium type the block is being committed to - * @param blockId the block id being committed - * @param length the length of the block being committed - */ - public void commitBlock(final long workerId, final long usedBytesOnTier, - final String tierAlias, final String mediumType, - final long blockId, final long length) throws AlluxioStatusException { - retryRPC(() -> { - CommitBlockPRequest request = - CommitBlockPRequest.newBuilder().setWorkerId(workerId).setUsedBytesOnTier(usedBytesOnTier) - .setTierAlias(tierAlias).setMediumType(mediumType) - .setBlockId(blockId).setLength(length).build(); - mClient.commitBlock(request); - return null; - }, LOG, "CommitBlock", - "workerId=%d,usedBytesOnTier=%d,tierAlias=%s,mediumType=%s,blockId=%d,length=%d", - workerId, usedBytesOnTier, tierAlias, mediumType, blockId, length); - } - - /** - * Commits a block in Ufs. - * - * @param blockId the block id being committed - * @param length the length of the block being committed - */ - public void commitBlockInUfs(final long blockId, final long length) - throws AlluxioStatusException { - retryRPC(() -> { - CommitBlockInUfsPRequest request = - CommitBlockInUfsPRequest.newBuilder().setBlockId(blockId).setLength(length).build(); - mClient.commitBlockInUfs(request); - return null; - }, LOG, "CommitBlockInUfs", "blockId=%d,length=%d", blockId, length); - } - - /** - * Returns a worker id for a workers net address. - * - * @param address the net address to get a worker id for - * @return a worker id - */ - public long getId(final WorkerNetAddress address) throws IOException { - return retryRPC(() -> { - GetWorkerIdPRequest request = - GetWorkerIdPRequest.newBuilder().setWorkerNetAddress(GrpcUtils.toProto(address)).build(); - return mClient.getWorkerId(request).getWorkerId(); - }, LOG, "GetId", "address=%s", address); - } - - /** - * Converts the block list map to a proto list. - * Because the list is flattened from a map, in the list no two {@link LocationBlockIdListEntry} - * instances shall have the same {@link BlockStoreLocationProto}. - * The uniqueness of {@link BlockStoreLocationProto} is determined by tier alias and medium type. - * That means directories with the same tier alias and medium type will be merged into the same - * {@link LocationBlockIdListEntry}. - * - * @param blockListOnLocation a map from block location to the block list - * @return a flattened and deduplicated list - */ - @VisibleForTesting - public List convertBlockListMapToProto( - Map> blockListOnLocation) { - final List entryList = new ArrayList<>(); - - Map> tierToBlocks = new HashMap<>(); - for (Map.Entry> entry : blockListOnLocation.entrySet()) { - BlockStoreLocation loc = entry.getKey(); - BlockStoreLocationProto locationProto = BlockStoreLocationProto.newBuilder() - .setTierAlias(loc.tierAlias()) - .setMediumType(loc.mediumType()) - .build(); - if (tierToBlocks.containsKey(locationProto)) { - tierToBlocks.get(locationProto).addAll(entry.getValue()); - } else { - List blockList = new ArrayList<>(entry.getValue()); - tierToBlocks.put(locationProto, blockList); - } - } - for (Map.Entry> entry : tierToBlocks.entrySet()) { - BlockIdList blockIdList = BlockIdList.newBuilder().addAllBlockId(entry.getValue()).build(); - LocationBlockIdListEntry listEntry = LocationBlockIdListEntry.newBuilder() - .setKey(entry.getKey()).setValue(blockIdList).build(); - entryList.add(listEntry); - } - - return entryList; - } - - /** - * The method the worker should periodically execute to heartbeat back to the master. - * - * @param workerId the worker id - * @param capacityBytesOnTiers a mapping from storage tier alias to capacity bytes - * @param usedBytesOnTiers a mapping from storage tier alias to used bytes - * @param removedBlocks a list of block removed from this worker - * @param addedBlocks a mapping from storage tier alias to added blocks - * @param lostStorage a mapping from storage tier alias to a list of lost storage paths - * @param metrics a list of worker metrics - * @return an optional command for the worker to execute - */ - public synchronized Command heartbeat(final long workerId, - final Map capacityBytesOnTiers, final Map usedBytesOnTiers, - final List removedBlocks, final Map> addedBlocks, - final Map> lostStorage, final List metrics) - throws IOException { - final BlockHeartbeatPOptions options = BlockHeartbeatPOptions.newBuilder() - .addAllMetrics(metrics).putAllCapacityBytesOnTiers(capacityBytesOnTiers).build(); - - final List entryList = convertBlockListMapToProto(addedBlocks); - - final Map lostStorageMap = lostStorage.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, - e -> StorageList.newBuilder().addAllStorage(e.getValue()).build())); - - final BlockHeartbeatPRequest request = BlockHeartbeatPRequest.newBuilder().setWorkerId(workerId) - .putAllUsedBytesOnTiers(usedBytesOnTiers).addAllRemovedBlockIds(removedBlocks) - .addAllAddedBlocks(entryList).setOptions(options) - .putAllLostStorage(lostStorageMap).build(); - - return retryRPC(() -> mClient.withDeadlineAfter(mContext.getClusterConf() - .getMs(PropertyKey.WORKER_MASTER_PERIODICAL_RPC_TIMEOUT), TimeUnit.MILLISECONDS) - .blockHeartbeat(request).getCommand(), LOG, "Heartbeat", "workerId=%d", workerId); - } - - private GetRegisterLeasePResponse acquireRegisterLease( - final long workerId, final int estimatedBlockCount) throws IOException { - return retryRPC(() -> { - LOG.info("Requesting lease with workerId {}, blockCount {}", workerId, estimatedBlockCount); - return mClient.requestRegisterLease(GetRegisterLeasePRequest.newBuilder() - .setWorkerId(workerId).setBlockCount(estimatedBlockCount).build()); - }, LOG, "GetRegisterLease", "workerId=%d, estimatedBlockCount=%d", - workerId, estimatedBlockCount); - } - - /** - * Acquires a {@link alluxio.wire.RegisterLease} from the master with - * the {@link RetryPolicy} specified. - * If all the retry attempts have been exhaused, a {@link FailedToAcquireRegisterLeaseException} - * will be thrown. - * - * @param workerId the worker ID - * @param estimatedBlockCount the number of blocks this worker currently holds - * There is a gap between acquiring a lease and generating the {@link RegisterWorkerPRequest} - * so the real block count may be different. But this is a good hint for the master to - * guess how much resource the registration will take. - * @param retry a retry policy - */ - public void acquireRegisterLeaseWithBackoff( - final long workerId, final int estimatedBlockCount, final RetryPolicy retry) - throws IOException, FailedToAcquireRegisterLeaseException { - boolean leaseAcquired = false; - GetRegisterLeasePResponse response = null; - while (!leaseAcquired && retry.attempt()) { - LOG.debug("Worker {} attempting to grant registration lease from the master, iter {}", - workerId, retry.getAttemptCount()); - response = acquireRegisterLease(workerId, estimatedBlockCount); - LOG.debug("Worker {} lease response: {}", workerId, response); - leaseAcquired = response.getAllowed(); - } - - if (response == null || !response.getAllowed()) { - throw new FailedToAcquireRegisterLeaseException( - String.format("Failed to acquire a register lease from master after %d attempts", - retry.getAttemptCount())); - } - - LOG.info("Worker {} acquired lease after {} attempts: {}", - workerId, retry.getAttemptCount(), response); - } - - /** - * The method the worker should execute to register with the block master. - * - * @param workerId the worker id of the worker registering - * @param storageTierAliases a list of storage tier aliases in ordinal order - * @param totalBytesOnTiers mapping from storage tier alias to total bytes - * @param usedBytesOnTiers mapping from storage tier alias to used bytes - * @param currentBlocksOnLocation mapping from storage tier alias to the list of list of blocks - * @param lostStorage mapping from storage tier alias to the list of lost storage paths - * @param configList a list of configurations - */ - // TODO(yupeng): rename to workerBlockReport or workerInitialize? - public void register(final long workerId, final List storageTierAliases, - final Map totalBytesOnTiers, final Map usedBytesOnTiers, - final Map> currentBlocksOnLocation, - final Map> lostStorage, - final List configList) throws IOException { - - final BuildVersion buildVersion = BuildVersion.newBuilder() - .setVersion(ProjectConstants.VERSION) - .setRevision(ProjectConstants.REVISION).build(); - - final RegisterWorkerPOptions options = - RegisterWorkerPOptions.newBuilder().addAllConfigs(configList) - .setBuildVersion(buildVersion).build(); - - final List currentBlocks - = convertBlockListMapToProto(currentBlocksOnLocation); - - final Map lostStorageMap = lostStorage.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, - e -> StorageList.newBuilder().addAllStorage(e.getValue()).build())); - - final RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId) - .addAllStorageTiers(storageTierAliases).putAllTotalBytesOnTiers(totalBytesOnTiers) - .putAllUsedBytesOnTiers(usedBytesOnTiers) - .addAllCurrentBlocks(currentBlocks) - .putAllLostStorage(lostStorageMap) - .setOptions(options).build(); - - retryRPC(() -> { - mClient.registerWorker(request); - return null; - }, LOG, "Register", "workerId=%d", workerId); - } - - /** - * Registers with the master in a stream. - * - * @param workerId the worker ID - * @param storageTierAliases storage/tier setup from the configuration - * @param totalBytesOnTiers the capacity of each tier - * @param usedBytesOnTiers the current usage of each tier - * @param currentBlocksOnLocation the blocks in each tier/dir - * @param lostStorage the lost storage paths - * @param configList the configuration properties - */ - public void registerWithStream(final long workerId, final List storageTierAliases, - final Map totalBytesOnTiers, final Map usedBytesOnTiers, - final Map> currentBlocksOnLocation, - final Map> lostStorage, - final List configList) throws IOException { - AtomicReference ioe = new AtomicReference<>(); - // The retry logic only takes care of connection issues. - // If the master side sends back an error, - // no retry will be attempted and the worker will quit. - retryRPC(() -> { - // The gRPC stream lifecycle is managed internal to the RegisterStreamer - // When an exception is thrown, the stream has been closed and error propagated - // to the other side, so no extra handling is required here. - RegisterStreamer stream = new RegisterStreamer(mAsyncClient, - workerId, storageTierAliases, totalBytesOnTiers, usedBytesOnTiers, - currentBlocksOnLocation, lostStorage, configList); - try { - stream.registerWithMaster(); - } catch (IOException e) { - ioe.set(e); - } catch (InterruptedException e) { - ioe.set(new IOException(e)); - } - return null; - }, LOG, "Register", "workerId=%d", workerId); - - if (ioe.get() != null) { - throw ioe.get(); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterClientPool.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterClientPool.java deleted file mode 100644 index e9ece84e2fc9..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterClientPool.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.ClientContext; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.MasterClientContext; -import alluxio.resource.ResourcePool; - -import com.google.common.io.Closer; - -import java.io.IOException; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Class for managing block master clients. After obtaining a client with - * {@link ResourcePool#acquire()}, {@link ResourcePool#release(Object)} must be called when the - * thread is done using the client. - */ -@ThreadSafe -public class BlockMasterClientPool extends ResourcePool { - private final Queue mClientList; - private final MasterClientContext mMasterContext; - - /** - * Creates a new block master client pool. - */ - public BlockMasterClientPool() { - super(Configuration.getInt(PropertyKey.WORKER_BLOCK_MASTER_CLIENT_POOL_SIZE)); - mClientList = new ConcurrentLinkedQueue<>(); - mMasterContext = MasterClientContext - .newBuilder(ClientContext.create(Configuration.global())).build(); - } - - @Override - public void close() throws IOException { - BlockMasterClient client; - Closer closer = Closer.create(); - while ((client = mClientList.poll()) != null) { - closer.register(client); - } - closer.close(); - } - - @Override - public BlockMasterClient createNewResource() { - BlockMasterClient client = new BlockMasterClient(mMasterContext); - mClientList.add(client); - return client; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterSync.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterSync.java deleted file mode 100644 index da898ccd9405..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockMasterSync.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.ProcessUtils; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ConnectionFailedException; -import alluxio.exception.FailedToAcquireRegisterLeaseException; -import alluxio.grpc.Command; -import alluxio.grpc.ConfigProperty; -import alluxio.grpc.Scope; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.metrics.MetricsSystem; -import alluxio.retry.ExponentialTimeBoundedRetry; -import alluxio.retry.RetryPolicy; -import alluxio.wire.WorkerNetAddress; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Duration; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Task that carries out the necessary block worker to master communications, including register and - * heartbeat. This class manages its own {@link BlockMasterClient}. - * - * When running, this task first requests a block report from the - * {@link alluxio.worker.block.BlockWorker}, then sends it to the master. The master may - * respond to the heartbeat with a command which will be executed. After which, the task will wait - * for the elapsed time since its last heartbeat has reached the heartbeat interval. Then the cycle - * will continue. - * - * If the task fails to heartbeat to the master, it will destroy its old master client and recreate - * it before retrying. - */ -@NotThreadSafe -public final class BlockMasterSync implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(BlockMasterSync.class); - private static final long ACQUIRE_LEASE_WAIT_BASE_SLEEP_MS = - Configuration.getMs(PropertyKey.WORKER_REGISTER_LEASE_RETRY_SLEEP_MIN); - private static final long ACQUIRE_LEASE_WAIT_MAX_SLEEP_MS = - Configuration.getMs(PropertyKey.WORKER_REGISTER_LEASE_RETRY_SLEEP_MAX); - private static final long ACQUIRE_LEASE_WAIT_MAX_DURATION = - Configuration.getMs(PropertyKey.WORKER_REGISTER_LEASE_RETRY_MAX_DURATION); - private static final int HEARTBEAT_TIMEOUT_MS = - (int) Configuration.getMs(PropertyKey.WORKER_BLOCK_HEARTBEAT_TIMEOUT_MS); - - /** The block worker responsible for interacting with Alluxio and UFS storage. */ - private final BlockWorker mBlockWorker; - - /** The worker ID for the worker. This may change if the master asks the worker to re-register. */ - private final AtomicReference mWorkerId; - - /** The net address of the worker. */ - private final WorkerNetAddress mWorkerAddress; - - /** Client-pool for all master communication. */ - private final BlockMasterClientPool mMasterClientPool; - /** Client for all master communication. */ - private final BlockMasterClient mMasterClient; - - /** An async service to remove block. */ - private final AsyncBlockRemover mAsyncBlockRemover; - - /** Last System.currentTimeMillis() timestamp when a heartbeat successfully completed. */ - private long mLastSuccessfulHeartbeatMs; - - /** - * Creates a new instance of {@link BlockMasterSync}. - * - * @param blockWorker the {@link BlockWorker} this syncer is updating to - * @param workerId the worker id of the worker, assigned by the block master - * @param workerAddress the net address of the worker - * @param masterClientPool the Alluxio master client pool - */ - public BlockMasterSync(BlockWorker blockWorker, AtomicReference workerId, - WorkerNetAddress workerAddress, BlockMasterClientPool masterClientPool) throws IOException { - mBlockWorker = blockWorker; - mWorkerId = workerId; - mWorkerAddress = workerAddress; - mMasterClientPool = masterClientPool; - mMasterClient = mMasterClientPool.acquire(); - mAsyncBlockRemover = new AsyncBlockRemover(mBlockWorker); - - registerWithMaster(); - mLastSuccessfulHeartbeatMs = System.currentTimeMillis(); - } - - /** - * Gets the default retry policy for acquiring a {@link alluxio.wire.RegisterLease} - * from the BlockMaster. - * - * @return the policy to use - */ - public static RetryPolicy getDefaultAcquireLeaseRetryPolicy() { - return ExponentialTimeBoundedRetry.builder() - .withMaxDuration(Duration.of(ACQUIRE_LEASE_WAIT_MAX_DURATION, ChronoUnit.MILLIS)) - .withInitialSleep(Duration.of(ACQUIRE_LEASE_WAIT_BASE_SLEEP_MS, ChronoUnit.MILLIS)) - .withMaxSleep(Duration.of(ACQUIRE_LEASE_WAIT_MAX_SLEEP_MS, ChronoUnit.MILLIS)) - .withSkipInitialSleep() - .build(); - } - - /** - * Registers with the Alluxio master. This should be called before the - * continuous heartbeat thread begins. - */ - private void registerWithMaster() throws IOException { - BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull(); - List configList = - Configuration.getConfiguration(Scope.WORKER); - - boolean leaseRequired = Configuration.getBoolean(PropertyKey.WORKER_REGISTER_LEASE_ENABLED); - if (leaseRequired) { - LOG.info("Acquiring a RegisterLease from the master before registering"); - try { - mMasterClient.acquireRegisterLeaseWithBackoff(mWorkerId.get(), - storeMeta.getNumberOfBlocks(), - getDefaultAcquireLeaseRetryPolicy()); - LOG.info("Lease acquired"); - } catch (FailedToAcquireRegisterLeaseException e) { - mMasterClient.disconnect(); - if (Configuration.getBoolean(PropertyKey.TEST_MODE)) { - throw new RuntimeException(String.format("Master register lease timeout exceeded: %dms", - ACQUIRE_LEASE_WAIT_MAX_DURATION)); - } - ProcessUtils.fatalError(LOG, "Master register lease timeout exceeded: %dms", - ACQUIRE_LEASE_WAIT_MAX_DURATION); - } - } - - boolean useStreaming = Configuration.getBoolean(PropertyKey.WORKER_REGISTER_STREAM_ENABLED); - if (useStreaming) { - mMasterClient.registerWithStream(mWorkerId.get(), - storeMeta.getStorageTierAssoc().getOrderedStorageAliases(), - storeMeta.getCapacityBytesOnTiers(), - storeMeta.getUsedBytesOnTiers(), storeMeta.getBlockListByStorageLocation(), - storeMeta.getLostStorage(), configList); - } else { - mMasterClient.register(mWorkerId.get(), - storeMeta.getStorageTierAssoc().getOrderedStorageAliases(), - storeMeta.getCapacityBytesOnTiers(), - storeMeta.getUsedBytesOnTiers(), storeMeta.getBlockListByStorageLocation(), - storeMeta.getLostStorage(), configList); - } - // If the worker registers with master successfully, the lease will be recycled on the - // master side. No need to manually request for recycle on the worker side. - } - - /** - * Heartbeats to the master node about the change in the worker's managed space. - */ - @Override - public void heartbeat() { - // Prepare metadata for the next heartbeat - BlockHeartbeatReport blockReport = mBlockWorker.getReport(); - BlockStoreMeta storeMeta = mBlockWorker.getStoreMeta(); - - // Send the heartbeat and execute the response - Command cmdFromMaster = null; - List metrics = MetricsSystem.reportWorkerMetrics(); - - try { - cmdFromMaster = mMasterClient.heartbeat(mWorkerId.get(), storeMeta.getCapacityBytesOnTiers(), - storeMeta.getUsedBytesOnTiers(), blockReport.getRemovedBlocks(), - blockReport.getAddedBlocks(), blockReport.getLostStorage(), metrics); - handleMasterCommand(cmdFromMaster); - mLastSuccessfulHeartbeatMs = System.currentTimeMillis(); - } catch (IOException | ConnectionFailedException e) { - // An error occurred, log and ignore it or error if heartbeat timeout is reached - if (cmdFromMaster == null) { - LOG.error("Failed to receive master heartbeat command.", e); - } else { - LOG.error("Failed to receive or execute master heartbeat command: {}", cmdFromMaster, e); - } - mMasterClient.disconnect(); - if (HEARTBEAT_TIMEOUT_MS > 0) { - if (System.currentTimeMillis() - mLastSuccessfulHeartbeatMs >= HEARTBEAT_TIMEOUT_MS) { - if (Configuration.getBoolean(PropertyKey.TEST_MODE)) { - throw new RuntimeException( - String.format("Master heartbeat timeout exceeded: %s", HEARTBEAT_TIMEOUT_MS)); - } - // TODO(andrew): Propagate the exception to the main thread and exit there. - ProcessUtils.fatalError(LOG, "Master heartbeat timeout exceeded: %d", - HEARTBEAT_TIMEOUT_MS); - } - } - } - } - - @Override - public void close() { - mAsyncBlockRemover.shutDown(); - mMasterClientPool.release(mMasterClient); - } - - /** - * Handles a master command. The command is one of Unknown, Nothing, Register, Free, or Delete. - * This call will block until the command is complete. - * - * @param cmd the command to execute - * @throws IOException if I/O errors occur - * @throws ConnectionFailedException if connection fails - */ - // TODO(calvin): Evaluate the necessity of each command. - private void handleMasterCommand(Command cmd) throws IOException, ConnectionFailedException { - if (cmd == null) { - return; - } - switch (cmd.getCommandType()) { - // Currently unused - case Delete: - break; - // Master requests blocks to be removed from Alluxio managed space. - case Free: - mAsyncBlockRemover.addBlocksToDelete(cmd.getDataList()); - break; - // No action required - case Nothing: - break; - // Master requests re-registration - case Register: - mWorkerId.set(mMasterClient.getId(mWorkerAddress)); - registerWithMaster(); - break; - // Unknown request - case Unknown: - LOG.error("Master heartbeat sends unknown command {}", cmd); - break; - default: - throw new RuntimeException("Un-recognized command from master " + cmd); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataAllocatorView.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataAllocatorView.java deleted file mode 100644 index fa65e54359b5..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataAllocatorView.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.worker.block.meta.StorageTier; -import alluxio.worker.block.meta.StorageTierAllocatorView; - -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class exposes a narrower read-only view of block metadata to allocators. - */ -@NotThreadSafe -public class BlockMetadataAllocatorView extends BlockMetadataView { - - /** - * Creates a new instance of {@link BlockMetadataAllocatorView}. - * - * @param manager which the view should be constructed from - * @param useReservedSpace include reserved space in available bytes - */ - public BlockMetadataAllocatorView(BlockMetadataManager manager, boolean useReservedSpace) { - super(manager, useReservedSpace); - } - - @Override - public void initializeView() { - // iteratively create all StorageTierViews and StorageDirViews - for (StorageTier tier : mMetadataManager.getTiers()) { - StorageTierAllocatorView tierView = new StorageTierAllocatorView(tier, mUseReservedSpace); - mTierViews.add(tierView); - mAliasToTierViews.put(tier.getTierAlias(), tierView); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataEvictorView.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataEvictorView.java deleted file mode 100644 index 5facfd7f0cd5..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataEvictorView.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.master.block.BlockId; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.StorageDirEvictorView; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTier; -import alluxio.worker.block.meta.StorageTierEvictorView; -import alluxio.worker.block.meta.StorageTierView; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class exposes a narrower view of {@link BlockMetadataManager} to Evictors, - * filtering out un-evictable blocks and un-allocatable space internally, so that evictors and - * allocators can be developed with much simpler logic, without worrying about various constraints, - * e.g. pinned files, locked blocks, etc. - * - * TODO(cc): Filter un-allocatable space. - */ -@NotThreadSafe -public class BlockMetadataEvictorView extends BlockMetadataView { - private static final Logger LOG = LoggerFactory.getLogger(BlockMetadataEvictorView.class); - - /** A list of pinned inodes, including inodes which are scheduled for async persist. */ - private final Set mPinnedInodes = new HashSet<>(); - - /** Indices of locks that are being used. */ - private final Set mInUseBlocks = new HashSet<>(); - - /** - * Creates a new instance of {@link BlockMetadataEvictorView}. Now we always create a new view - * before freespace. - * - * @param manager which the view should be constructed from - * @param pinnedInodes a set of pinned inodes - * @param lockedBlocks a set of locked blocks - */ - // TODO(qifan): Incrementally update the view. - public BlockMetadataEvictorView(BlockMetadataManager manager, Set pinnedInodes, - Set lockedBlocks) { - super(manager); - mPinnedInodes.addAll(Preconditions.checkNotNull(pinnedInodes, "pinnedInodes")); - Preconditions.checkNotNull(lockedBlocks, "lockedBlocks"); - mInUseBlocks.addAll(lockedBlocks); - } - - @Override - protected void initializeView() { - // iteratively create all StorageTierViews and StorageDirViews - for (StorageTier tier : mMetadataManager.getTiers()) { - StorageTierEvictorView tierView = new StorageTierEvictorView(tier, this); - mTierViews.add(tierView); - mAliasToTierViews.put(tier.getTierAlias(), tierView); - } - } - - /** - * Gets all dir views that belongs to a given location. - * - * @param location the location - * @return the list of dir views - */ - public List getDirs(BlockStoreLocation location) { - List dirs = new LinkedList<>(); - for (StorageTierView tier : mTierViews) { - for (StorageDirView dir : tier.getDirViews()) { - if (dir.toBlockStoreLocation().belongsTo(location)) { - dirs.add(dir); - } - } - } - return dirs; - } - - /** - * Clears all marks of blocks to move in/out in all dir views. - */ - public void clearBlockMarks() { - for (StorageTierView tierView : mTierViews) { - for (StorageDirView dirView : tierView.getDirViews()) { - ((StorageDirEvictorView) dirView).clearBlockMarks(); - } - } - } - - /** - * Tests if the block is pinned either explicitly or because it is scheduled for async persist. - * - * @param blockId to be tested - * @return boolean, true if block is pinned - */ - public boolean isBlockPinned(long blockId) { - return mPinnedInodes.contains(BlockId.getFileId(blockId)); - } - - /** - * Tests if the block is locked. - * - * @param blockId to be tested - * @return boolean, true if block is locked - */ - public boolean isBlockLocked(long blockId) { - return mInUseBlocks.contains(blockId); - } - - /** - * Tests if the block is evictable. - * - * @param blockId to be tested - * @return boolean, true if the block can be evicted - */ - public boolean isBlockEvictable(long blockId) { - boolean pinned = isBlockPinned(blockId); - boolean locked = isBlockLocked(blockId); - boolean marked = isBlockMarked(blockId); - boolean isEvictable = !pinned && !locked && !marked; - if (!isEvictable) { - LOG.debug("Block not evictable: {}. Pinned: {}, Locked: {}, Marked: {}", blockId, pinned, - locked, marked); - } - - return isEvictable; - } - - /** - * Tests if the block is marked to move out of its current dir in this view. - * - * @param blockId the id of the block - * @return boolean, true if the block is marked to move out - */ - public boolean isBlockMarked(long blockId) { - for (StorageTierView tierView : mTierViews) { - for (StorageDirView dirView : tierView.getDirViews()) { - if (((StorageDirEvictorView) dirView).isMarkedToMoveOut(blockId)) { - return true; - } - } - } - return false; - } - - /** - * Gets available bytes given certain location - * {@link BlockMetadataManager#getAvailableBytes(BlockStoreLocation)}. Throws an - * {@link IllegalArgumentException} if the location does not belong to tiered storage. - * - * @param location location the check available bytes - * @return available bytes - */ - public long getAvailableBytes(BlockStoreLocation location) { - return mMetadataManager.getAvailableBytes(location); - } - - /** - * Returns Optional.empty() if block is pinned or currently being locked, otherwise returns - * {@link BlockMetadataManager#getBlockMeta(long)}. - * - * @param blockId the block id - * @return metadata of the block or null - */ - public Optional getBlockMeta(long blockId) { - if (isBlockEvictable(blockId)) { - return mMetadataManager.getBlockMeta(blockId); - } else { - return Optional.empty(); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataManager.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataManager.java deleted file mode 100644 index 5c170199003c..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataManager.java +++ /dev/null @@ -1,435 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static java.util.function.Function.identity; - -import alluxio.DefaultStorageTierAssoc; -import alluxio.StorageTierAssoc; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.worker.block.allocator.Allocator; -import alluxio.worker.block.annotator.BlockAnnotator; -import alluxio.worker.block.annotator.BlockIterator; -import alluxio.worker.block.annotator.DefaultBlockIterator; -import alluxio.worker.block.annotator.EmulatingBlockIterator; -import alluxio.worker.block.evictor.Evictor; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.DefaultBlockMeta; -import alluxio.worker.block.meta.DefaultStorageTier; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageTier; -import alluxio.worker.block.meta.TempBlockMeta; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.IntStream; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Manages the metadata of all blocks in managed space. This information is used by the - * {@link TieredBlockStore}, {@link Allocator} and {@link Evictor}. - *

- * All operations on block metadata such as {@link StorageTier}, {@link StorageDir} should go - * through this class. - */ -@NotThreadSafe -// TODO(bin): consider how to better expose information to Evictor and Allocator. -public final class BlockMetadataManager { - private static final Logger LOG = LoggerFactory.getLogger(BlockMetadataManager.class); - public static final StorageTierAssoc WORKER_STORAGE_TIER_ASSOC = - new DefaultStorageTierAssoc( - PropertyKey.WORKER_TIERED_STORE_LEVELS, - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS); - - /** A list of managed {@link StorageTier}, in order from lowest tier ordinal to greatest. */ - private final List mTiers; - - /** A map from tier alias to {@link StorageTier}. */ - private final Map mAliasToTiers; - - /** Used to get iterators per locations. */ - private final BlockIterator mBlockIterator; - - /** Deprecated evictors. */ - private static final String DEPRECATED_LRU_EVICTOR = "alluxio.worker.block.evictor.LRUEvictor"; - private static final String DEPRECATED_PARTIAL_LRUEVICTOR = - "alluxio.worker.block.evictor.PartialLRUEvictor"; - private static final String DEPRECATED_LRFU_EVICTOR = "alluxio.worker.block.evictor.LRFUEvictor"; - private static final String DEPRECATED_GREEDY_EVICTOR = - "alluxio.worker.block.evictor.GreedyEvictor"; - - private BlockMetadataManager() { - mTiers = IntStream.range(0, WORKER_STORAGE_TIER_ASSOC.size()).mapToObj( - tierOrdinal -> DefaultStorageTier.newStorageTier( - WORKER_STORAGE_TIER_ASSOC.getAlias(tierOrdinal), - tierOrdinal, - WORKER_STORAGE_TIER_ASSOC.size() > 1)) - .parallel() - .collect(toImmutableList()); - mAliasToTiers = mTiers.stream().collect(toImmutableMap(StorageTier::getTierAlias, identity())); - // Create the block iterator. - if (Configuration.isSet(PropertyKey.WORKER_EVICTOR_CLASS)) { - LOG.warn(String.format("Evictor is being emulated. Please use %s instead.", - PropertyKey.Name.WORKER_BLOCK_ANNOTATOR_CLASS)); - String evictorType = Configuration.getString(PropertyKey.WORKER_EVICTOR_CLASS); - switch (evictorType) { - case DEPRECATED_LRU_EVICTOR: - case DEPRECATED_PARTIAL_LRUEVICTOR: - case DEPRECATED_GREEDY_EVICTOR: - LOG.warn("Evictor is deprecated, switching to LRUAnnotator"); - Configuration.set(PropertyKey.WORKER_BLOCK_ANNOTATOR_CLASS, - "alluxio.worker.block.annotator.LRUAnnotator"); - mBlockIterator = new DefaultBlockIterator(this, BlockAnnotator.Factory.create()); - break; - case DEPRECATED_LRFU_EVICTOR: - LOG.warn("Evictor is deprecated, switching to LRFUAnnotator"); - Configuration.set(PropertyKey.WORKER_BLOCK_ANNOTATOR_CLASS, - "alluxio.worker.block.annotator.LRFUAnnotator"); - mBlockIterator = new DefaultBlockIterator(this, BlockAnnotator.Factory.create()); - break; - default: - //For user defined evictor - BlockMetadataEvictorView initManagerView = new BlockMetadataEvictorView(this, - Collections.emptySet(), Collections.emptySet()); - mBlockIterator = new EmulatingBlockIterator(this, - Evictor.Factory.create(initManagerView, Allocator.Factory.create(initManagerView))); - } - } else { - // Create default block iterator - mBlockIterator = new DefaultBlockIterator(this, BlockAnnotator.Factory.create()); - } - } - - /** - * @return the configured iteration provider - */ - public BlockIterator getBlockIterator() { - return mBlockIterator; - } - - /** - * Creates a new instance of {@link BlockMetadataManager}. - * - * @return a {@link BlockMetadataManager} instance - */ - public static BlockMetadataManager createBlockMetadataManager() { - return new BlockMetadataManager(); - } - - /** - * Aborts a temp block. - * - * @param tempBlockMeta the metadata of the temp block to add - */ - public void abortTempBlockMeta(TempBlockMeta tempBlockMeta) { - tempBlockMeta.getParentDir().removeTempBlockMeta(tempBlockMeta); - } - - /** - * Adds a temp block. - * - * @param tempBlockMeta the metadata of the temp block to add - */ - public void addTempBlockMeta(TempBlockMeta tempBlockMeta) { - StorageDir dir = tempBlockMeta.getParentDir(); - dir.addTempBlockMeta(tempBlockMeta); - } - - /** - * Commits a temp block. - * - * @param tempBlockMeta the metadata of the temp block to commit - */ - public void commitTempBlockMeta(TempBlockMeta tempBlockMeta) { - long blockId = tempBlockMeta.getBlockId(); - if (hasBlockMeta(blockId)) { - throw new IllegalStateException(ExceptionMessage.ADD_EXISTING_BLOCK.getMessage(blockId, - getBlockMeta(blockId).get().getBlockLocation().tierAlias())); - } - BlockMeta block = new DefaultBlockMeta(Preconditions.checkNotNull(tempBlockMeta)); - StorageDir dir = tempBlockMeta.getParentDir(); - dir.removeTempBlockMeta(tempBlockMeta); - dir.addBlockMeta(block); - } - - /** - * Cleans up the metadata of the given temp block ids. - * - * @param sessionId the id of the client associated with the temp blocks - * @param tempBlockIds the list of temporary block ids to be cleaned up, non temporary block ids - * will be ignored. - * @deprecated As of version 0.8. - */ - @Deprecated - public void cleanupSessionTempBlocks(long sessionId, List tempBlockIds) { - for (StorageTier tier : mTiers) { - for (StorageDir dir : tier.getStorageDirs()) { - dir.cleanupSessionTempBlocks(sessionId, tempBlockIds); - } - } - } - - /** - * Gets the amount of available space of given location in bytes. Master queries the total number - * of bytes available on each tier of the worker, and Evictor/Allocator often cares about the - * bytes at a {@link StorageDir}. Throws an {@link IllegalArgumentException} when the location - * does not belong to the tiered storage. - * - * @param location location the check available bytes - * @return available bytes - */ - public long getAvailableBytes(BlockStoreLocation location) { - long spaceAvailable = 0; - - if (location.hasNoRestriction()) { - for (StorageTier tier : mTiers) { - spaceAvailable += tier.getAvailableBytes(); - } - return spaceAvailable; - } - - if (!location.isAnyMedium() && location.isAnyDir() && location.isAnyTier()) { - for (StorageTier tier : mTiers) { - for (StorageDir dir : tier.getStorageDirs()) { - if (dir.getDirMedium().equals(location.mediumType())) { - spaceAvailable += dir.getAvailableBytes(); - } - } - } - return spaceAvailable; - } - - String tierAlias = location.tierAlias(); - StorageTier tier = getTier(tierAlias); - // TODO(calvin): This should probably be max of the capacity bytes in the dirs? - if (location.isAnyDir()) { - return tier.getAvailableBytes(); - } - - int dirIndex = location.dir(); - StorageDir dir = tier.getDir(dirIndex); - return dir == null ? 0 : dir.getAvailableBytes(); - } - - /** - * Gets the metadata of a block given its block id. - * - * @param blockId the block id - * @return metadata of the block - */ - public Optional getBlockMeta(long blockId) { - for (StorageTier tier : mTiers) { - for (StorageDir dir : tier.getStorageDirs()) { - if (dir.hasBlockMeta(blockId)) { - return dir.getBlockMeta(blockId); - } - } - } - return Optional.empty(); - } - - /** - * Gets a summary of the metadata. - * - * @return the metadata of this block store - */ - public BlockStoreMeta getBlockStoreMeta() { - return new DefaultBlockStoreMeta(this, false); - } - - /** - * Gets a full summary of block store metadata. This is an expensive operation. - * - * @return the full metadata of this block store - */ - public BlockStoreMeta getBlockStoreMetaFull() { - return new DefaultBlockStoreMeta(this, true); - } - - /** - * Gets the {@link StorageDir} given its location in the store. Throws an - * {@link IllegalArgumentException} if the location is not a specific dir or the location is - * invalid. - * - * @param location Location of the dir - * @return the {@link StorageDir} object - */ - public StorageDir getDir(BlockStoreLocation location) { - checkArgument(!(location.isAnyTier() || location.isAnyDir()), - MessageFormat.format("Cannot get path from non-specific dir {0}", location)); - return getTier(location.tierAlias()).getDir(location.dir()); - } - - /** - * Gets the metadata of a temp block. - * - * @param blockId the id of the temp block - * @return metadata of the block - */ - public Optional getTempBlockMeta(long blockId) { - for (StorageTier tier : mTiers) { - for (StorageDir dir : tier.getStorageDirs()) { - if (dir.hasTempBlockMeta(blockId)) { - return dir.getTempBlockMeta(blockId); - } - } - } - return Optional.empty(); - } - - /** - * Gets the {@link StorageTier} given its tierAlias. Throws an {@link IllegalArgumentException} if - * the tierAlias is not found. - * - * @param tierAlias the alias of this tier - * @return the {@link StorageTier} object associated with the alias - */ - public StorageTier getTier(String tierAlias) { - StorageTier tier = mAliasToTiers.get(tierAlias); - if (tier == null) { - throw new IllegalArgumentException( - ExceptionMessage.TIER_ALIAS_NOT_FOUND.getMessage(tierAlias)); - } - return tier; - } - - /** - * Gets the list of {@link StorageTier} managed. - * - * @return the list of {@link StorageTier}s - */ - public List getTiers() { - return mTiers; - } - - /** - * Gets the list of {@link StorageTier} below the tier with the given tierAlias. Throws an - * {@link IllegalArgumentException} if the tierAlias is not found. - * - * @param tierAlias the alias of a tier - * @return the list of {@link StorageTier} - */ - public List getTiersBelow(String tierAlias) { - int ordinal = getTier(tierAlias).getTierOrdinal(); - return mTiers.subList(ordinal + 1, mTiers.size()); - } - - /** - * Gets all the temporary blocks associated with a session, empty list is returned if the session - * has no temporary blocks. - * - * @param sessionId the id of the session - * @return A list of temp blocks associated with the session - */ - public List getSessionTempBlocks(long sessionId) { - List sessionTempBlocks = new ArrayList<>(); - for (StorageTier tier : mTiers) { - for (StorageDir dir : tier.getStorageDirs()) { - sessionTempBlocks.addAll(dir.getSessionTempBlocks(sessionId)); - } - } - return sessionTempBlocks; - } - - /** - * Checks if the storage has a given block. - * - * @param blockId the block id - * @return true if the block is contained, false otherwise - */ - public boolean hasBlockMeta(long blockId) { - for (StorageTier tier : mTiers) { - for (StorageDir dir : tier.getStorageDirs()) { - if (dir.hasBlockMeta(blockId)) { - return true; - } - } - } - return false; - } - - /** - * Checks if the storage has a given temp block. - * - * @param blockId the temp block id - * @return true if the block is contained, false otherwise - */ - public boolean hasTempBlockMeta(long blockId) { - for (StorageTier tier : mTiers) { - for (StorageDir dir : tier.getStorageDirs()) { - if (dir.hasTempBlockMeta(blockId)) { - return true; - } - } - } - return false; - } - - /** - * Moves an existing block to another location currently hold by a temp block. - * - * @param blockMeta the metadata of the block to move - * @param tempBlockMeta a placeholder in the destination directory - * @return the new block metadata if success, absent otherwise - */ - public BlockMeta moveBlockMeta(BlockMeta blockMeta, TempBlockMeta tempBlockMeta) { - StorageDir srcDir = blockMeta.getParentDir(); - StorageDir dstDir = tempBlockMeta.getParentDir(); - srcDir.removeBlockMeta(blockMeta); - BlockMeta newBlockMeta = - new DefaultBlockMeta(blockMeta.getBlockId(), blockMeta.getBlockSize(), dstDir); - dstDir.removeTempBlockMeta(tempBlockMeta); - dstDir.addBlockMeta(newBlockMeta); - return newBlockMeta; - } - - /** - * Removes the metadata of a specific block. - * - * @param block the metadata of the block to remove - */ - public void removeBlockMeta(BlockMeta block) { - StorageDir dir = block.getParentDir(); - dir.removeBlockMeta(block); - } - - /** - * Modifies the size of a temp block. - * - * @param tempBlockMeta the temp block to modify - * @param newSize new size in bytes - */ - public void resizeTempBlockMeta(TempBlockMeta tempBlockMeta, long newSize) { - StorageDir dir = tempBlockMeta.getParentDir(); - dir.resizeTempBlockMeta(tempBlockMeta, newSize); - } - - /** - * @return the storage tier mapping - */ - public StorageTierAssoc getStorageTierAssoc() { - return WORKER_STORAGE_TIER_ASSOC; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataView.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataView.java deleted file mode 100644 index 7f253ce2ca09..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetadataView.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.exception.ExceptionMessage; -import alluxio.worker.block.meta.StorageTier; -import alluxio.worker.block.meta.StorageTierView; - -import com.google.common.base.Preconditions; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.annotation.Nullable; - -/** - * This class is an abstract class for allocators and evictors to extend to provide - * limited access to block metadata. - */ -public abstract class BlockMetadataView { - /** The {@link BlockMetadataManager} this view is derived from. */ - protected final BlockMetadataManager mMetadataManager; - protected final boolean mUseReservedSpace; - - /** - * A list of {@link StorageTierView}, derived from {@link StorageTier}s from the - * {@link BlockMetadataManager}. - */ - final List mTierViews = new ArrayList<>(); - - /** A map from tier alias to {@link StorageTierView}. */ - Map mAliasToTierViews = new HashMap<>(); - - /** - * Creates a new instance of {@link BlockMetadataView}. - * - * @param manager which the view should be constructed from - */ - public BlockMetadataView(BlockMetadataManager manager) { - this(manager, false); - } - - /** - * Creates a new instance of {@link BlockMetadataView}. - * - * @param manager which the view should be constructed from - * @param useReservedSpace whether to include reserved space in available bytes - */ - public BlockMetadataView(BlockMetadataManager manager, boolean useReservedSpace) { - mMetadataManager = Preconditions.checkNotNull(manager, "manager"); - mUseReservedSpace = useReservedSpace; - initializeView(); - } - - /** - * Provides {@link StorageTierView} given tierAlias. Throws an {@link IllegalArgumentException} if - * the tierAlias is not found. - * - * @param tierAlias the alias of this tierView - * @return the {@link StorageTierView} object associated with the alias - */ - public StorageTierView getTierView(String tierAlias) { - StorageTierView tierView = mAliasToTierViews.get(tierAlias); - if (tierView == null) { - throw new IllegalArgumentException( - ExceptionMessage.TIER_VIEW_ALIAS_NOT_FOUND.getMessage(tierAlias)); - } else { - return tierView; - } - } - - /** - * Gets all tierViews under this storage metadata view. - * - * @return the list of {@link StorageTierView}s - */ - public List getTierViews() { - return Collections.unmodifiableList(mTierViews); - } - - /** - * Gets the next storage tier view. - * - * @param tierView the storage tier view - * @return the next storage tier view, null if this is the last tier view - */ - @Nullable - public StorageTierView getNextTier(StorageTierView tierView) { - int nextOrdinal = tierView.getTierViewOrdinal() + 1; - if (nextOrdinal < mTierViews.size()) { - return mTierViews.get(nextOrdinal); - } - return null; - } - - /** - * Gets all tierViews before certain tierView. Throws an {@link IllegalArgumentException} if the - * tierAlias is not found. - * - * @param tierAlias the alias of a tierView - * @return the list of {@link StorageTierView} - */ - public List getTierViewsBelow(String tierAlias) { - int ordinal = getTierView(tierAlias).getTierViewOrdinal(); - return mTierViews.subList(ordinal + 1, mTierViews.size()); - } - - /** - * Used to initialize the view based on the current metadata status. - */ - protected abstract void initializeView(); - - /** - * Used to refresh the view based on current metadata status. - * - * @return the refreshed view instance - */ - public BlockMetadataView refreshView() { - // Clear previous views. - mTierViews.clear(); - mAliasToTierViews.clear(); - // Re-initialize the view. - initializeView(); - return this; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetricsReporter.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockMetricsReporter.java deleted file mode 100644 index 47e49cfe1051..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockMetricsReporter.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static alluxio.worker.block.BlockMetadataManager.WORKER_STORAGE_TIER_ASSOC; - -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * This class listens on block events and increases the metrics counters. - */ -@ThreadSafe -public final class BlockMetricsReporter extends AbstractBlockStoreEventListener { - private static final Counter BLOCKS_ACCESSED - = MetricsSystem.counter(MetricKey.WORKER_BLOCKS_ACCESSED.getName()); - private static final Counter BLOCKS_PROMOTED - = MetricsSystem.counter(MetricKey.WORKER_BLOCKS_PROMOTED.getName()); - private static final Counter BLOCKS_DELETED - = MetricsSystem.counter(MetricKey.WORKER_BLOCKS_DELETED.getName()); - private static final Counter BLOCKS_EVICTED - = MetricsSystem.counter(MetricKey.WORKER_BLOCKS_EVICTED.getName()); - private static final Counter BLOCKS_CANCELLED - = MetricsSystem.counter(MetricKey.WORKER_BLOCKS_CANCELLED.getName()); - private static final Counter BLOCKS_LOST - = MetricsSystem.counter(MetricKey.WORKER_BLOCKS_LOST.getName()); - - private static final Meter BLOCKS_EVICTION_RATE = - MetricsSystem.meterWithTags(MetricKey.WORKER_BLOCKS_EVICTION_RATE.getName(), - MetricKey.WORKER_BLOCKS_EVICTION_RATE.isClusterAggregated()); - - @Override - public void onAccessBlock(long blockId) { - BLOCKS_ACCESSED.inc(); - } - - @Override - public void onMoveBlockByClient(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) { - int oldTierOrdinal = WORKER_STORAGE_TIER_ASSOC.getOrdinal(oldLocation.tierAlias()); - int newTierOrdinal = WORKER_STORAGE_TIER_ASSOC.getOrdinal(newLocation.tierAlias()); - if (newTierOrdinal == 0 && oldTierOrdinal != newTierOrdinal) { - BLOCKS_PROMOTED.inc(); - } - } - - @Override - public void onRemoveBlockByClient(long blockId) { - BLOCKS_DELETED.inc(); - } - - @Override - public void onMoveBlockByWorker(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) { - int oldTierOrdinal = WORKER_STORAGE_TIER_ASSOC.getOrdinal(oldLocation.tierAlias()); - int newTierOrdinal = WORKER_STORAGE_TIER_ASSOC.getOrdinal(newLocation.tierAlias()); - if (newTierOrdinal == 0 && oldTierOrdinal != newTierOrdinal) { - BLOCKS_PROMOTED.inc(); - } - } - - @Override - public void onRemoveBlockByWorker(long blockId) { - BLOCKS_EVICTED.inc(); - BLOCKS_EVICTION_RATE.mark(); - } - - @Override - public void onAbortBlock(long blockId) { - BLOCKS_CANCELLED.inc(); - } - - @Override - public void onBlockLost(long blockId) { - BLOCKS_LOST.inc(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/BlockWorkerFactory.java b/core/server/worker/src/main/java/alluxio/worker/block/BlockWorkerFactory.java deleted file mode 100644 index 19a0e996f07a..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/BlockWorkerFactory.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.ClientContext; -import alluxio.Sessions; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.MasterClientContext; -import alluxio.underfs.UfsManager; -import alluxio.worker.WorkerFactory; -import alluxio.worker.WorkerRegistry; -import alluxio.worker.file.FileSystemMasterClient; -import alluxio.worker.page.PagedBlockStore; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.atomic.AtomicReference; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Factory to create a {@link BlockWorker} instance. - */ -@ThreadSafe -public final class BlockWorkerFactory implements WorkerFactory { - private static final Logger LOG = LoggerFactory.getLogger(BlockWorkerFactory.class); - - /** - * Constructs a new {@link BlockWorkerFactory}. - */ - public BlockWorkerFactory() {} - - @Override - public boolean isEnabled() { - return true; - } - - @Override - public BlockWorker create(WorkerRegistry registry, UfsManager ufsManager) { - BlockMasterClientPool blockMasterClientPool = new BlockMasterClientPool(); - AtomicReference workerId = new AtomicReference<>(-1L); - BlockStore blockStore; - switch (Configuration.global() - .getEnum(PropertyKey.WORKER_BLOCK_STORE_TYPE, BlockStoreType.class)) { - case PAGE: - LOG.info("Creating PagedBlockWorker"); - blockStore = PagedBlockStore.create(ufsManager, blockMasterClientPool, workerId); - break; - case FILE: - LOG.info("Creating DefaultBlockWorker"); - blockStore = - new MonoBlockStore(new TieredBlockStore(), blockMasterClientPool, ufsManager, workerId); - break; - default: - throw new UnsupportedOperationException("Unsupported block store type."); - } - BlockWorker blockWorker = new DefaultBlockWorker(blockMasterClientPool, - new FileSystemMasterClient( - MasterClientContext.newBuilder(ClientContext.create(Configuration.global())).build()), - new Sessions(), blockStore, workerId); - registry.add(BlockWorker.class, blockWorker); - return blockWorker; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/CacheRequestManager.java b/core/server/worker/src/main/java/alluxio/worker/block/CacheRequestManager.java deleted file mode 100644 index e265f5e330cc..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/CacheRequestManager.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.Constants; -import alluxio.Sessions; -import alluxio.client.file.FileSystemContext; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.AlluxioException; -import alluxio.exception.status.CancelledException; -import alluxio.grpc.CacheRequest; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proto.dataserver.Protocol; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; -import alluxio.util.io.BufferUtils; -import alluxio.util.logging.SamplingLogger; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; - -import com.codahale.metrics.Counter; -import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Objects; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicLong; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Handles client requests to synchronously/asynchronously cache blocks. Responsible for - * managing the local worker resources and intelligent pruning of duplicate or meaningless requests. - */ -@ThreadSafe -public class CacheRequestManager { - private static final Logger LOG = LoggerFactory.getLogger(CacheRequestManager.class); - private static final Logger SAMPLING_LOG = new SamplingLogger(LOG, 10L * Constants.MINUTE_MS); - private static final int NETWORK_HOST_RESOLUTION_TIMEOUT = - (int) Configuration.getMs(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS); - - /** Executor service for execute the async cache tasks. */ - private final ExecutorService mCacheExecutor; - /** The block worker. */ - private final DefaultBlockWorker mBlockWorker; - private final ConcurrentHashMap mActiveCacheRequests = - new ConcurrentHashMap<>(); - private final FileSystemContext mFsContext; - /** Keeps track of the number of rejected cache requests. */ - private final AtomicLong mNumRejected = new AtomicLong(0); - - /** - * @param service thread pool to run the background caching work - * @param blockWorker handler to the block worker - * @param fsContext context - */ - public CacheRequestManager(ExecutorService service, DefaultBlockWorker blockWorker, - FileSystemContext fsContext) { - mCacheExecutor = service; - mBlockWorker = blockWorker; - mFsContext = fsContext; - } - - /** - * Handles a request to cache a block. If it's async cache, it wouldn't throw exception. - * - * @param request the cache request fields will be available - */ - public void submitRequest(CacheRequest request) - throws AlluxioException, IOException { - CACHE_REQUESTS.inc(); - long blockId = request.getBlockId(); - boolean async = request.getAsync(); - if (mActiveCacheRequests.putIfAbsent(blockId, request) != null) { - // This block is already planned and just just return. - if (async) { - LOG.debug("request already planned: {}", request); - } else { - try { - CommonUtils.waitFor("block to be loaded", - () -> !mActiveCacheRequests.containsKey(blockId), - WaitForOptions.defaults().setTimeoutMs(30 * Constants.SECOND_MS)); - } catch (InterruptedException e) { - throw new CancelledException("Fail to finish cache request synchronously. " - + "Interrupted while waiting for block to be loaded by another request.", e); - } catch (TimeoutException e) { - throw new CancelledException("Fail to finish cache request synchronously due to timeout", - e); - } - } - return; - } - - if (!async) { - CACHE_REQUESTS_SYNC.inc(); - } else { - CACHE_REQUESTS_ASYNC.inc(); - } - Future future = null; - try { - future = mCacheExecutor.submit(new CacheTask(request)); - } catch (RejectedExecutionException e) { - // RejectedExecutionException may be thrown in extreme cases when the - // gRPC thread pool is drained due to highly concurrent caching workloads. In these cases, - // return as async caching is at best effort. - mNumRejected.incrementAndGet(); - SAMPLING_LOG.warn(String.format( - "Failed to cache block locally as the thread pool is at capacity." - + " To increase, update the parameter '%s'. numRejected: {} error: {}", - PropertyKey.Name.WORKER_NETWORK_ASYNC_CACHE_MANAGER_THREADS_MAX), mNumRejected.get(), - e.toString()); - mActiveCacheRequests.remove(blockId); - if (!async) { - throw new CancelledException(String.format( - "Fail to finish cache request synchronously as the thread pool is at capacity." - + " To increase the capacity, set the parameter '%s' and '%s' higher. ", - PropertyKey.Name.WORKER_NETWORK_ASYNC_CACHE_MANAGER_THREADS_MAX, - PropertyKey.Name.WORKER_NETWORK_ASYNC_CACHE_MANAGER_QUEUE_MAX), e); - } - } - if (future != null && !async) { - try { - future.get(); - } catch (ExecutionException e) { - CACHE_FAILED_BLOCKS.inc(); - Throwable cause = e.getCause(); - if (cause instanceof AlluxioException) { - throw new AlluxioException(cause.getMessage(), cause); - } else { - throw new IOException(cause); - } - } catch (InterruptedException e) { - throw new CancelledException( - "Fail to finish cache request synchronously. Interrupted while waiting for response.", - e); - } - } - } - - /** - * CacheTask is a callable task that can be considered equal if the blockId of the request is the - * same. - */ - @VisibleForTesting - class CacheTask implements Callable { - private final CacheRequest mRequest; - - /** - * Constructor for an CacheTask. - * - * @param request an CacheRequest - */ - CacheTask(CacheRequest request) { - mRequest = request; - } - - @Override - public int hashCode() { - // Only care about the block id being the same. - // Do not care if the source host or port is different. - return Objects.hash(mRequest.getBlockId()); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof CacheTask)) { - return false; - } - CacheTask that = ((CacheTask) obj); - if (mRequest == that.mRequest) { - return true; - } - if (this.mRequest == null || that.mRequest == null) { - return false; - } - // Only care about the block id being the same. - // Do not care if the source host or port is different. - return mRequest.getBlockId() == that.mRequest.getBlockId(); - } - - @Override - public Void call() throws IOException, AlluxioException { - long blockId = mRequest.getBlockId(); - long blockLength = mRequest.getLength(); - boolean result = false; - try { - result = cacheBlock(mRequest); - } finally { - if (result) { - CACHE_BLOCKS_SIZE.inc(blockLength); - CACHE_SUCCEEDED_BLOCKS.inc(); - } else { - CACHE_FAILED_BLOCKS.inc(); - } - mActiveCacheRequests.remove(blockId); - } - return null; - } - } - - private boolean cacheBlock(CacheRequest request) throws IOException, AlluxioException { - boolean result; - boolean isSourceLocal = NetworkAddressUtils.isLocalAddress(request.getSourceHost(), - NETWORK_HOST_RESOLUTION_TIMEOUT); - long blockId = request.getBlockId(); - long blockLength = request.getLength(); - // Check if the block has already been cached on this worker - if (mBlockWorker.getBlockStore().hasBlockMeta(blockId)) { - LOG.debug("block already cached: {}", blockId); - return true; - } - Protocol.OpenUfsBlockOptions openUfsBlockOptions = request.getOpenUfsBlockOptions(); - // Depends on the request, cache the target block from different sources - if (isSourceLocal) { - CACHE_UFS_BLOCKS.inc(); - result = cacheBlockFromUfs(blockId, blockLength, openUfsBlockOptions); - } else { - CACHE_REMOTE_BLOCKS.inc(); - InetSocketAddress sourceAddress = - new InetSocketAddress(request.getSourceHost(), request.getSourcePort()); - result = - cacheBlockFromRemoteWorker(blockId, blockLength, sourceAddress, openUfsBlockOptions); - } - LOG.debug("Result of caching block {}: {}", blockId, result); - return result; - } - - /** - * Caches the block via the local worker to read from UFS. - * - * @param blockId block ID - * @param blockSize block size - * @param openUfsBlockOptions options to open the UFS file - * @return if the block is cached - */ - private boolean cacheBlockFromUfs(long blockId, long blockSize, - Protocol.OpenUfsBlockOptions openUfsBlockOptions) throws IOException { - try (BlockReader reader = mBlockWorker.createUfsBlockReader( - Sessions.CACHE_UFS_SESSION_ID, blockId, 0, false, openUfsBlockOptions)) { - // Read the entire block, caching to block store will be handled internally in UFS block store - // when close the reader. - // Note that, we read from UFS with a smaller buffer to avoid high pressure on heap - // memory when concurrent async requests are received and thus trigger GC. - long offset = 0; - while (offset < blockSize) { - long bufferSize = Math.min(8L * Constants.MB, blockSize - offset); - reader.read(offset, bufferSize); - offset += bufferSize; - } - } - return true; - } - - /** - * Caches the block at best effort from a remote worker (possibly from UFS indirectly). - * - * @param blockId block ID - * @param blockSize block size - * @param sourceAddress the source to read the block previously by client - * @param openUfsBlockOptions options to open the UFS file - * @return if the block is cached - */ - private boolean cacheBlockFromRemoteWorker(long blockId, long blockSize, - InetSocketAddress sourceAddress, Protocol.OpenUfsBlockOptions openUfsBlockOptions) - throws IOException { - if (mBlockWorker.getBlockStore().hasBlockMeta(blockId) - || mBlockWorker.getBlockStore().hasTempBlockMeta(blockId)) { - // It is already cached - return true; - } - mBlockWorker.createBlock(Sessions.CACHE_WORKER_SESSION_ID, blockId, 0, - new CreateBlockOptions(null, "", blockSize)); - try ( - BlockReader reader = - getRemoteBlockReader(blockId, blockSize, sourceAddress, openUfsBlockOptions); - BlockWriter writer = mBlockWorker - .createBlockWriter(Sessions.CACHE_WORKER_SESSION_ID, blockId)) { - BufferUtils.transfer(reader.getChannel(), writer.getChannel()); - mBlockWorker.commitBlock(Sessions.CACHE_WORKER_SESSION_ID, blockId, false); - return true; - } catch (IllegalStateException | IOException e) { - LOG.warn("Failed to async cache block {} from remote worker ({}) on copying the block: {}", - blockId, sourceAddress, e.toString()); - try { - mBlockWorker.abortBlock(Sessions.CACHE_WORKER_SESSION_ID, blockId); - } catch (IOException ee) { - LOG.warn("Failed to abort block {}: {}", blockId, ee.toString()); - } - throw e; - } - } - - /** - * only public to test since we want to mock remoteBlockReader. - * @param blockId block ID - * @param blockSize block size - * @param sourceAddress remote source address - * @param openUfsBlockOptions options to open the UFS file - * @return remote block reader - */ - @VisibleForTesting - public RemoteBlockReader getRemoteBlockReader(long blockId, long blockSize, - InetSocketAddress sourceAddress, Protocol.OpenUfsBlockOptions openUfsBlockOptions) { - return new RemoteBlockReader(mFsContext, blockId, blockSize, sourceAddress, - openUfsBlockOptions); - } - - // Metrics - private static final Counter CACHE_REQUESTS = - MetricsSystem.counter(MetricKey.WORKER_CACHE_REQUESTS.getName()); - private static final Counter CACHE_REQUESTS_ASYNC = - MetricsSystem.counter(MetricKey.WORKER_CACHE_REQUESTS_ASYNC.getName()); - private static final Counter CACHE_REQUESTS_SYNC = - MetricsSystem.counter(MetricKey.WORKER_CACHE_REQUESTS_SYNC.getName()); - private static final Counter CACHE_FAILED_BLOCKS = - MetricsSystem.counter(MetricKey.WORKER_CACHE_FAILED_BLOCKS.getName()); - private static final Counter CACHE_REMOTE_BLOCKS = - MetricsSystem.counter(MetricKey.WORKER_CACHE_REMOTE_BLOCKS.getName()); - private static final Counter CACHE_SUCCEEDED_BLOCKS = - MetricsSystem.counter(MetricKey.WORKER_CACHE_SUCCEEDED_BLOCKS.getName()); - private static final Counter CACHE_UFS_BLOCKS = - MetricsSystem.counter(MetricKey.WORKER_CACHE_UFS_BLOCKS.getName()); - private static final Counter CACHE_BLOCKS_SIZE = - MetricsSystem.counter(MetricKey.WORKER_CACHE_BLOCKS_SIZE.getName()); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/DefaultBlockStoreMeta.java b/core/server/worker/src/main/java/alluxio/worker/block/DefaultBlockStoreMeta.java deleted file mode 100644 index 8024554b0894..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/DefaultBlockStoreMeta.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.StorageTierAssoc; -import alluxio.collections.Pair; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageTier; - -import com.google.common.base.Preconditions; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.annotation.concurrent.ThreadSafe; - -/** - * This class holds the metadata information of a block store. - *

- * TODO(bin): Use proto buf to represent this information. - */ -@ThreadSafe -public final class DefaultBlockStoreMeta implements BlockStoreMeta { - // TODO(bin): The following two fields don't need to be computed on the creation of each - // {@link BlockStoreMeta} instance. - - /** Mapping from storage tier alias to capacity bytes. */ - private final Map mCapacityBytesOnTiers = new HashMap<>(); - - /** Mapping from storage tier alias to used bytes. */ - private final Map mUsedBytesOnTiers = new HashMap<>(); - - /** Mapping from storage tier alias to block id list. */ - private final Map> mBlockIdsOnTiers; - - /** Mapping from BlockStoreLocation to block id list. */ - private final Map> mBlockIdsOnLocations; - - /** Mapping from storage dir tier and path to total capacity. */ - private final Map, Long> mCapacityBytesOnDirs = new HashMap<>(); - - /** Mapping from storage dir tier and path to used bytes. */ - private final Map, Long> mUsedBytesOnDirs = new HashMap<>(); - - private final StorageTierAssoc mStorageTierAssoc; - - /** Mapping from tier alias to lost storage paths. */ - private final Map> mLostStorage = new HashMap<>(); - - @Override - public Map> getBlockList() { - Preconditions.checkNotNull(mBlockIdsOnTiers, "mBlockIdsOnTiers"); - - return mBlockIdsOnTiers; - } - - @Override - public Map> getBlockListByStorageLocation() { - Preconditions.checkNotNull(mBlockIdsOnLocations, "mBlockIdsOnLocations"); - - return mBlockIdsOnLocations; - } - - @Override - public long getCapacityBytes() { - long capacityBytes = 0L; - for (long capacityBytesOnTier : mCapacityBytesOnTiers.values()) { - capacityBytes += capacityBytesOnTier; - } - return capacityBytes; - } - - @Override - public Map getCapacityBytesOnTiers() { - return mCapacityBytesOnTiers; - } - - @Override - public Map, Long> getCapacityBytesOnDirs() { - return mCapacityBytesOnDirs; - } - - @Override - public Map> getDirectoryPathsOnTiers() { - Map> pathsOnTiers = new HashMap<>(); - for (Pair tierPath : mCapacityBytesOnDirs.keySet()) { - String tier = tierPath.getFirst(); - pathsOnTiers.computeIfAbsent(tier, k -> new ArrayList<>()).add(tierPath.getSecond()); - } - return pathsOnTiers; - } - - @Override - public Map> getLostStorage() { - return new HashMap<>(mLostStorage); - } - - @Override - public int getNumberOfBlocks() { - Preconditions.checkNotNull(mBlockIdsOnTiers, "mBlockIdsOnTiers"); - - int numberOfBlocks = 0; - for (List blockIds : mBlockIdsOnTiers.values()) { - numberOfBlocks += blockIds.size(); - } - return numberOfBlocks; - } - - @Override - public long getUsedBytes() { - long usedBytes = 0L; - for (long usedBytesOnTier : mUsedBytesOnTiers.values()) { - usedBytes += usedBytesOnTier; - } - return usedBytes; - } - - @Override - public Map getUsedBytesOnTiers() { - return Collections.unmodifiableMap(mUsedBytesOnTiers); - } - - @Override - public Map, Long> getUsedBytesOnDirs() { - return mUsedBytesOnDirs; - } - - /** - * Creates a new instance of {@link DefaultBlockStoreMeta}. - * - * @param manager a block metadata manager handle - */ - protected DefaultBlockStoreMeta(BlockMetadataManager manager, boolean shouldIncludeBlockIds) { - Preconditions.checkNotNull(manager, "manager"); - mStorageTierAssoc = manager.getStorageTierAssoc(); - for (StorageTier tier : manager.getTiers()) { - Long capacityBytes = mCapacityBytesOnTiers.get(tier.getTierAlias()); - Long usedBytes = mUsedBytesOnTiers.get(tier.getTierAlias()); - mCapacityBytesOnTiers.put(tier.getTierAlias(), - (capacityBytes == null ? 0L : capacityBytes) + tier.getCapacityBytes()); - mUsedBytesOnTiers.put(tier.getTierAlias(), - (usedBytes == null ? 0L : usedBytes) + (tier.getCapacityBytes() - tier - .getAvailableBytes())); - for (StorageDir dir : tier.getStorageDirs()) { - Pair dirKey = - new Pair<>(tier.getTierAlias(), dir.getDirPath()); - mCapacityBytesOnDirs.put(dirKey, dir.getCapacityBytes()); - mUsedBytesOnDirs.put(dirKey, dir.getCapacityBytes() - dir.getAvailableBytes()); - } - } - - if (shouldIncludeBlockIds) { - mBlockIdsOnTiers = new HashMap<>(); - mBlockIdsOnLocations = new HashMap<>(); - for (StorageTier tier : manager.getTiers()) { - for (StorageDir dir : tier.getStorageDirs()) { - List blockIds; - BlockStoreLocation location - = new BlockStoreLocation(tier.getTierAlias(), dir.getDirIndex(), dir.getDirMedium()); - List blockIdsForLocation = new ArrayList<>(dir.getBlockIds()); - mBlockIdsOnLocations.put(location, blockIdsForLocation); - if (mBlockIdsOnTiers.containsKey(tier.getTierAlias())) { - blockIds = mBlockIdsOnTiers.get(tier.getTierAlias()); - } else { - blockIds = new ArrayList<>(); - mBlockIdsOnTiers.put(tier.getTierAlias(), blockIds); - } - blockIds.addAll(dir.getBlockIds()); - } - } - } else { - mBlockIdsOnTiers = null; - mBlockIdsOnLocations = null; - } - - for (StorageTier tier : manager.getTiers()) { - if (!tier.getLostStorage().isEmpty()) { - List lostStorages = mLostStorage - .getOrDefault(tier.getTierAlias(), new ArrayList<>()); - lostStorages.addAll(tier.getLostStorage()); - mLostStorage.put(tier.getTierAlias(), lostStorages); - } - } - } - - @Override - public StorageTierAssoc getStorageTierAssoc() { - return mStorageTierAssoc; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/DefaultBlockWorker.java b/core/server/worker/src/main/java/alluxio/worker/block/DefaultBlockWorker.java deleted file mode 100644 index cd08337e3b05..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/DefaultBlockWorker.java +++ /dev/null @@ -1,574 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static alluxio.worker.block.BlockMetadataManager.WORKER_STORAGE_TIER_ASSOC; - -import alluxio.ClientContext; -import alluxio.Constants; -import alluxio.RuntimeConstants; -import alluxio.Server; -import alluxio.Sessions; -import alluxio.annotation.SuppressFBWarnings; -import alluxio.client.file.FileSystemContext; -import alluxio.collections.PrefixList; -import alluxio.conf.Configuration; -import alluxio.conf.ConfigurationValueOptions; -import alluxio.conf.PropertyKey; -import alluxio.conf.Source; -import alluxio.exception.AlluxioException; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.grpc.AsyncCacheRequest; -import alluxio.grpc.Block; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.GetConfigurationPOptions; -import alluxio.grpc.GrpcService; -import alluxio.grpc.ServiceType; -import alluxio.grpc.UfsReadOptions; -import alluxio.heartbeat.HeartbeatContext; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.heartbeat.HeartbeatThread; -import alluxio.metrics.MetricInfo; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proto.dataserver.Protocol; -import alluxio.retry.RetryUtils; -import alluxio.security.user.ServerUserState; -import alluxio.util.executor.ExecutorServiceFactories; -import alluxio.util.io.FileUtils; -import alluxio.wire.FileInfo; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.AbstractWorker; -import alluxio.worker.SessionCleaner; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.file.FileSystemMasterClient; -import alluxio.worker.grpc.GrpcExecutors; -import alluxio.worker.page.PagedBlockStore; - -import com.codahale.metrics.Counter; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.io.Closer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicReference; -import javax.annotation.concurrent.NotThreadSafe; -import javax.annotation.concurrent.ThreadSafe; - -/** - * The class is responsible for managing all top level components of the Block Worker. - * - * This includes: - * - * Periodic Threads: {@link BlockMasterSync} (Worker to Master continuous communication) - * - * Logic: {@link DefaultBlockWorker} (Logic for all block related storage operations) - */ -@NotThreadSafe -public class DefaultBlockWorker extends AbstractWorker implements BlockWorker { - private static final Logger LOG = LoggerFactory.getLogger(DefaultBlockWorker.class); - private static final long UFS_BLOCK_OPEN_TIMEOUT_MS = - Configuration.getMs(PropertyKey.WORKER_UFS_BLOCK_OPEN_TIMEOUT_MS); - - /** Used to close resources during stop. */ - private final Closer mResourceCloser = Closer.create(); - /** - * Block master clients. commitBlock is the only reason to keep a pool of block master clients - * on each worker. We should either improve our RPC model in the master or get rid of the - * necessity to call commitBlock in the workers. - */ - private final BlockMasterClientPool mBlockMasterClientPool; - - /** Client for all file system master communication. */ - private final FileSystemMasterClient mFileSystemMasterClient; - - /** Block store delta reporter for master heartbeat. */ - private final BlockHeartbeatReporter mHeartbeatReporter; - /** Session metadata, used to keep track of session heartbeats. */ - private final Sessions mSessions; - /** Block Store manager. */ - private final BlockStore mBlockStore; - /** List of paths to always keep in memory. */ - private final PrefixList mWhitelist; - - /** - * The worker ID for this worker. This is initialized in {@link #start(WorkerNetAddress)} and may - * be updated by the block sync thread if the master requests re-registration. - */ - private final AtomicReference mWorkerId; - - private final CacheRequestManager mCacheManager; - private final FuseManager mFuseManager; - - private WorkerNetAddress mAddress; - - /** - * Constructs a default block worker. - * - * @param blockMasterClientPool a client pool for talking to the block master - * @param fileSystemMasterClient a client for talking to the file system master - * @param sessions an object for tracking and cleaning up client sessions - * @param blockStore an Alluxio block store - * @param workerId worker id - */ - @VisibleForTesting - public DefaultBlockWorker(BlockMasterClientPool blockMasterClientPool, - FileSystemMasterClient fileSystemMasterClient, Sessions sessions, BlockStore blockStore, - AtomicReference workerId) { - super(ExecutorServiceFactories.fixedThreadPool("block-worker-executor", 5)); - mBlockMasterClientPool = mResourceCloser.register(blockMasterClientPool); - mFileSystemMasterClient = mResourceCloser.register(fileSystemMasterClient); - mHeartbeatReporter = new BlockHeartbeatReporter(); - /* Metrics reporter that listens on block events and increases metrics counters. */ - BlockMetricsReporter metricsReporter = new BlockMetricsReporter(); - mSessions = sessions; - mBlockStore = mResourceCloser.register(blockStore); - mWorkerId = workerId; - mBlockStore.registerBlockStoreEventListener(mHeartbeatReporter); - mBlockStore.registerBlockStoreEventListener(metricsReporter); - FileSystemContext fsContext = mResourceCloser.register( - FileSystemContext.create(ClientContext.create(Configuration.global()), this)); - mCacheManager = new CacheRequestManager( - GrpcExecutors.CACHE_MANAGER_EXECUTOR, this, fsContext); - mFuseManager = mResourceCloser.register(new FuseManager(fsContext)); - mWhitelist = new PrefixList(Configuration.getList(PropertyKey.WORKER_WHITELIST)); - - Metrics.registerGauges(this); - } - - /** - * get the LocalBlockStore that manages local blocks. - * - * @return the LocalBlockStore that manages local blocks - * */ - @Override - public BlockStore getBlockStore() { - return mBlockStore; - } - - @Override - public Set> getDependencies() { - return new HashSet<>(); - } - - @Override - public String getName() { - return Constants.BLOCK_WORKER_NAME; - } - - @Override - public Map getServices() { - return Collections.emptyMap(); - } - - @Override - public AtomicReference getWorkerId() { - return mWorkerId; - } - - /** - * Runs the block worker. The thread must be called after all services (e.g., web, dataserver) - * started. - * - * BlockWorker doesn't support being restarted! - */ - @Override - public void start(WorkerNetAddress address) throws IOException { - super.start(address); - mAddress = address; - - // Acquire worker Id. - askForWorkerId(address); - - Preconditions.checkNotNull(mWorkerId, "mWorkerId"); - Preconditions.checkNotNull(mAddress, "mAddress"); - - // Setup BlockMasterSync - BlockMasterSync blockMasterSync = mResourceCloser - .register(new BlockMasterSync(this, mWorkerId, mAddress, mBlockMasterClientPool)); - getExecutorService() - .submit(new HeartbeatThread(HeartbeatContext.WORKER_BLOCK_SYNC, blockMasterSync, - (int) Configuration.getMs(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS), - Configuration.global(), ServerUserState.global())); - - // Setup PinListSyncer - PinListSync pinListSync = mResourceCloser.register( - new PinListSync(this, mFileSystemMasterClient)); - getExecutorService() - .submit(new HeartbeatThread(HeartbeatContext.WORKER_PIN_LIST_SYNC, pinListSync, - (int) Configuration.getMs(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS), - Configuration.global(), ServerUserState.global())); - - // Setup session cleaner - SessionCleaner sessionCleaner = mResourceCloser - .register(new SessionCleaner(mSessions, mBlockStore)); - getExecutorService().submit(sessionCleaner); - - // Setup storage checker - if (Configuration.getBoolean(PropertyKey.WORKER_STORAGE_CHECKER_ENABLED)) { - StorageChecker storageChecker = mResourceCloser.register(new StorageChecker()); - getExecutorService() - .submit(new HeartbeatThread(HeartbeatContext.WORKER_STORAGE_HEALTH, storageChecker, - (int) Configuration.getMs(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS), - Configuration.global(), ServerUserState.global())); - } - - // Mounts the embedded Fuse application - if (Configuration.getBoolean(PropertyKey.WORKER_FUSE_ENABLED)) { - mFuseManager.start(); - } - } - - /** - * Ask the master for a workerId. Should not be called outside of testing - * - * @param address the address this worker operates on - */ - @VisibleForTesting - public void askForWorkerId(WorkerNetAddress address) { - BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire(); - try { - RetryUtils.retry("create worker id", () -> mWorkerId.set(blockMasterClient.getId(address)), - RetryUtils.defaultWorkerMasterClientRetry()); - } catch (Exception e) { - throw new RuntimeException("Failed to create a worker id from block master: " - + e.getMessage()); - } finally { - mBlockMasterClientPool.release(blockMasterClient); - } - } - - /** - * Stops the block worker. This method should only be called to terminate the worker. - * - * BlockWorker doesn't support being restarted! - */ - @Override - public void stop() throws IOException { - // Stop the base. (closes executors.) - // This is intentionally called first in order to send interrupt signals to heartbeat threads. - // Otherwise, if the heartbeat threads are not interrupted then the shutdown can hang. - super.stop(); - // Stop heart-beat executors and clients. - mResourceCloser.close(); - } - - @Override - public void abortBlock(long sessionId, long blockId) throws IOException { - mBlockStore.abortBlock(sessionId, blockId); - Metrics.WORKER_ACTIVE_CLIENTS.dec(); - } - - @Override - public void commitBlock(long sessionId, long blockId, boolean pinOnCreate) { - mBlockStore.commitBlock(sessionId, blockId, pinOnCreate); - } - - @Override - public void commitBlockInUfs(long blockId, long length) { - BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire(); - try { - blockMasterClient.commitBlockInUfs(blockId, length); - } catch (AlluxioStatusException e) { - throw AlluxioRuntimeException.from(e); - } finally { - mBlockMasterClientPool.release(blockMasterClient); - } - } - - @Override - public String createBlock(long sessionId, long blockId, int tier, - CreateBlockOptions createBlockOptions) { - try { - return mBlockStore.createBlock(sessionId, blockId, tier, createBlockOptions); - } catch (ResourceExhaustedRuntimeException e) { - // mAddress is null if the worker is not started - if (mAddress == null) { - throw new ResourceExhaustedRuntimeException( - ExceptionMessage.CANNOT_REQUEST_SPACE.getMessage(mWorkerId.get(), blockId), e, false); - } - InetSocketAddress address = - InetSocketAddress.createUnresolved(mAddress.getHost(), mAddress.getRpcPort()); - throw new ResourceExhaustedRuntimeException( - ExceptionMessage.CANNOT_REQUEST_SPACE.getMessageWithUrl( - RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, address, blockId), e, false); - } - } - - @Override - public BlockWriter createBlockWriter(long sessionId, long blockId) - throws IOException { - return mBlockStore.createBlockWriter(sessionId, blockId); - } - - @Override - public BlockHeartbeatReport getReport() { - return mHeartbeatReporter.generateReport(); - } - - @Override - public BlockStoreMeta getStoreMeta() { - return mBlockStore.getBlockStoreMeta(); - } - - @Override - public BlockStoreMeta getStoreMetaFull() { - return mBlockStore.getBlockStoreMetaFull(); - } - - @Override - public List getWhiteList() { - return mWhitelist.getList(); - } - - @Override - public BlockReader createUfsBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException { - return mBlockStore.createUfsBlockReader(sessionId, blockId, offset, positionShort, options); - } - - @Override - public void removeBlock(long sessionId, long blockId) - throws IOException { - mBlockStore.removeBlock(sessionId, blockId); - } - - @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE") - @Override - public void freeWorker() throws IOException { - List paths = new ArrayList<>(); - if (Configuration.global().get(PropertyKey.WORKER_BLOCK_STORE_TYPE) == BlockStoreType.FILE) { - int tierCount = Configuration.global().getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS); - for (int i = 0; i < tierCount; i++) { - paths.addAll(Configuration.global().getList(PropertyKey - .Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(i))); - } - } else if (Configuration.global() - .get(PropertyKey.WORKER_BLOCK_STORE_TYPE) == BlockStoreType.PAGE) { - paths.addAll(Configuration.global().getList(PropertyKey.WORKER_PAGE_STORE_DIRS)); - } else { - throw new IllegalStateException("Unknown WORKER_BLOCK_STORE_TYPE."); - } - - List failDeleteDirs = new ArrayList<>(); - for (String tmpPath : paths) { - File[] files = new File(tmpPath).listFiles(); - Preconditions.checkNotNull(files, "The path does not denote a directory."); - for (File file : files) { - try { - FileUtils.deletePathRecursively(file.getPath()); - } catch (IOException ie) { - failDeleteDirs.add(file.getPath()); - } - } - } - if (!failDeleteDirs.isEmpty()) { - LOG.info("Some directories fail to be deleted: " + failDeleteDirs); - throw new IOException(failDeleteDirs.toString()); - } - LOG.info("All blocks and directories in worker {} are freed.", getWorkerId()); - } - - @Override - public void requestSpace(long sessionId, long blockId, long additionalBytes) { - mBlockStore.requestSpace(sessionId, blockId, additionalBytes); - } - - @Override - @Deprecated - public void asyncCache(AsyncCacheRequest request) { - CacheRequest cacheRequest = - CacheRequest.newBuilder().setBlockId(request.getBlockId()).setLength(request.getLength()) - .setOpenUfsBlockOptions(request.getOpenUfsBlockOptions()) - .setSourceHost(request.getSourceHost()).setSourcePort(request.getSourcePort()) - .setAsync(true).build(); - try { - mCacheManager.submitRequest(cacheRequest); - } catch (Exception e) { - LOG.warn("Failed to submit async cache request. request: {}", request, e); - } - } - - @Override - public void cache(CacheRequest request) throws AlluxioException, IOException { - // todo(bowen): paged block store handles caching from UFS automatically and on-the-fly - // this will cause an unnecessary extra read of the block - if (mBlockStore instanceof PagedBlockStore) { - return; - } - mCacheManager.submitRequest(request); - } - - @Override - public CompletableFuture> load(List blocks, UfsReadOptions options) { - return mBlockStore.load(blocks, options); - } - - @Override - public void updatePinList(Set pinnedInodes) { - mBlockStore.updatePinnedInodes(pinnedInodes); - } - - @Override - public FileInfo getFileInfo(long fileId) throws IOException { - return mFileSystemMasterClient.getFileInfo(fileId); - } - - /** - * Closes a UFS block for a client session. It also commits the block to Alluxio block store - * if the UFS block has been cached successfully. - * - * @param sessionId the session ID - * @param blockId the block ID - */ - - @Override - public BlockReader createBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException { - BlockReader reader = - mBlockStore.createBlockReader(sessionId, blockId, offset, positionShort, options); - Metrics.WORKER_ACTIVE_CLIENTS.inc(); - return reader; - } - - @Override - public void clearMetrics() { - // TODO(lu) Create a metrics worker and move this method to metrics worker - MetricsSystem.resetAllMetrics(); - } - - @Override - public alluxio.wire.Configuration getConfiguration(GetConfigurationPOptions options) { - // NOTE(cc): there is no guarantee that the returned cluster and path configurations are - // consistent snapshot of the system's state at a certain time, the path configuration might - // be in a newer state. But it's guaranteed that the hashes are respectively correspondent to - // the properties. - alluxio.wire.Configuration.Builder builder = alluxio.wire.Configuration.newBuilder(); - - if (!options.getIgnoreClusterConf()) { - for (PropertyKey key : Configuration.keySet()) { - if (key.isBuiltIn()) { - Source source = Configuration.getSource(key); - Object value = Configuration.getOrDefault(key, null, - ConfigurationValueOptions.defaults().useDisplayValue(true) - .useRawValue(options.getRawValue())); - builder.addClusterProperty(key.getName(), value, source); - } - } - // NOTE(cc): assumes that Configuration is read-only when master is running, otherwise, - // the following hash might not correspond to the above cluster configuration. - builder.setClusterConfHash(Configuration.hash()); - } - - return builder.build(); - } - - @Override - public void cleanupSession(long sessionId) { - mBlockStore.cleanupSession(sessionId); - Metrics.WORKER_ACTIVE_CLIENTS.dec(); - } - - /** - * This class contains some metrics related to the block worker. - * This class is public because the metric names are referenced in - * {@link alluxio.web.WebInterfaceAbstractMetricsServlet}. - */ - @ThreadSafe - public static final class Metrics { - public static final Counter WORKER_ACTIVE_CLIENTS = - MetricsSystem.counter(MetricKey.WORKER_ACTIVE_CLIENTS.getName()); - - /** - * Registers metric gauges. - * - * @param blockWorker the block worker handle - */ - public static void registerGauges(final BlockWorker blockWorker) { - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.WORKER_CAPACITY_TOTAL.getName()), - () -> blockWorker.getStoreMeta().getCapacityBytes()); - - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.WORKER_CAPACITY_USED.getName()), - () -> blockWorker.getStoreMeta().getUsedBytes()); - - MetricsSystem.registerGaugeIfAbsent( - MetricsSystem.getMetricName(MetricKey.WORKER_CAPACITY_FREE.getName()), - () -> blockWorker.getStoreMeta().getCapacityBytes() - blockWorker.getStoreMeta() - .getUsedBytes()); - - for (int i = 0; i < WORKER_STORAGE_TIER_ASSOC.size(); i++) { - String tier = WORKER_STORAGE_TIER_ASSOC.getAlias(i); - // TODO(lu) Add template to dynamically generate MetricKey - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_CAPACITY_TOTAL.getName() + MetricInfo.TIER + tier), - () -> blockWorker.getStoreMeta().getCapacityBytesOnTiers().getOrDefault(tier, 0L)); - - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_CAPACITY_USED.getName() + MetricInfo.TIER + tier), - () -> blockWorker.getStoreMeta().getUsedBytesOnTiers().getOrDefault(tier, 0L)); - - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_CAPACITY_FREE.getName() + MetricInfo.TIER + tier), - () -> blockWorker.getStoreMeta().getCapacityBytesOnTiers().getOrDefault(tier, 0L) - - blockWorker.getStoreMeta().getUsedBytesOnTiers().getOrDefault(tier, 0L)); - } - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCKS_CACHED.getName()), - () -> blockWorker.getStoreMetaFull().getNumberOfBlocks()); - } - - private Metrics() {} // prevent instantiation - } - - /** - * StorageChecker periodically checks the health of each storage path and report missing blocks to - * {@link BlockWorker}. - */ - @NotThreadSafe - public final class StorageChecker implements HeartbeatExecutor { - - @Override - public void heartbeat() { - try { - mBlockStore.removeInaccessibleStorage(); - } catch (Exception e) { - LOG.warn("Failed to check storage: {}", e.toString()); - LOG.debug("Exception: ", e); - } - } - - @Override - public void close() { - // Nothing to clean up - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/FuseManager.java b/core/server/worker/src/main/java/alluxio/worker/block/FuseManager.java deleted file mode 100644 index 349b166b25db..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/FuseManager.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.client.file.FileSystem; -import alluxio.client.file.FileSystemContext; -import alluxio.fuse.AlluxioFuse; -import alluxio.fuse.FuseUmountable; -import alluxio.fuse.options.FuseOptions; - -import com.google.common.io.Closer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; - -/** - * The Fuse manager that is responsible for managing the Fuse application lifecycle. - */ -public class FuseManager implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(FuseManager.class); - private final FileSystemContext mFsContext; - /** Use to umount Fuse application during stop. */ - private FuseUmountable mFuseUmountable; - /** Use to close resources during stop. */ - private final Closer mResourceCloser; - - /** - * Constructs a new {@link FuseManager}. - * - * @param fsContext fs context - */ - public FuseManager(FileSystemContext fsContext) { - mFsContext = fsContext; - mResourceCloser = Closer.create(); - } - - /** - * Starts mounting the internal Fuse applications. - */ - public void start() { - try { - // TODO(lu) consider launching fuse in a separate thread as blocking operation - // so that we can know about the fuse application status - FileSystem fileSystem = mResourceCloser.register(FileSystem.Factory.create(mFsContext)); - mFuseUmountable = AlluxioFuse.launchFuse( - mFsContext, fileSystem, FuseOptions.create(mFsContext.getClusterConf()), false); - } catch (Throwable throwable) { - // TODO(lu) for already mounted application, unmount first and then remount - LOG.error("Failed to launch worker internal Fuse application", throwable); - } - } - - @Override - public void close() throws IOException { - if (mFuseUmountable != null) { - try { - mFuseUmountable.umount(true); - } catch (Throwable throwable) { - LOG.error("Failed to umount worker internal Fuse application", throwable); - } - } - mResourceCloser.close(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/LocalBlockStore.java b/core/server/worker/src/main/java/alluxio/worker/block/LocalBlockStore.java deleted file mode 100644 index 06ff5a3cebe3..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/LocalBlockStore.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; -import alluxio.proto.dataserver.Protocol; -import alluxio.worker.SessionCleanable; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.TempBlockMeta; - -import java.io.Closeable; -import java.io.IOException; -import java.util.Optional; -import java.util.Set; - -/** - * A blob store interface to represent the local storage managing and serving all the blocks in the - * local storage. - */ -public interface LocalBlockStore - extends SessionCleanable, Closeable { - /** - * Pins the block indicating subsequent access. - * - * @param sessionId the id of the session to lock this block - * @param blockId the id of the block to lock - * @return a lock of block to conveniently unpin the block later, or empty - * if the block does not exist - */ - Optional pinBlock(long sessionId, long blockId); - - /** - * Creates the metadata of a new block and assigns a temporary path (e.g., a subdir of the final - * location named after session id) to store its data. The location can be a location with - * specific tier and dir, or {@link BlockStoreLocation#anyTier()}, or - * {@link BlockStoreLocation#anyDirInTier(String)}. - * - *

- * Before commit, all the data written to this block will be stored in the temp path and the block - * is only "visible" to its writer client. - * - * @param sessionId the id of the session - * @param blockId the id of the block to create - * @param options allocation options - * @return metadata of the temp block created - */ - TempBlockMeta createBlock(long sessionId, long blockId, AllocateOptions options); - - /** - * Gets the metadata of a block given its block id or empty if block does not exist. - * This method does not require a lock id so the block is possible to be moved or removed after it - * returns. - * - * @param blockId the block id - * @return metadata of the block - */ - Optional getVolatileBlockMeta(long blockId); - - /** - * Gets the temp metadata of a specific block from local storage. - * - * @param blockId the id of the block - * @return metadata of the block if the temp block exists - */ - Optional getTempBlockMeta(long blockId); - - /** - * Commits a temporary block to the local store. After commit, the block will be available in this - * block store for all clients to access. Since a temp block is "private" to the writer, this - * method requires no previously acquired lock. - * - * @param sessionId the id of the session - * @param blockId the id of a temp block - * @param pinOnCreate whether to pin block on create - */ - void commitBlock(long sessionId, long blockId, boolean pinOnCreate); - - /** - * Similar to {@link #commitBlock(long, long, boolean)}. It returns the block locked, - * so the caller is required to explicitly unlock the block. - * //TODO(Beinan): make this method to be private - * @param sessionId the id of the session - * @param blockId the id of a temp block - * @param pinOnCreate whether to pin block on create - * @return the lock - */ - BlockLock commitBlockLocked(long sessionId, long blockId, boolean pinOnCreate); - - /** - * Aborts a temporary block. The metadata of this block will not be added, its data will be - * deleted and the space will be reclaimed. Since a temp block is "private" to the writer, this - * requires no previously acquired lock. - * - * @param sessionId the id of the session - * @param blockId the id of a temp block - */ - void abortBlock(long sessionId, long blockId); - - /** - * Requests to increase the size of a temp block. Since a temp block is "private" to the writer - * client, this operation requires no previously acquired lock. - * - * @param sessionId the id of the session to request space - * @param blockId the id of the temp block - * @param additionalBytes the amount of more space to request in bytes, never be less than 0 - */ - void requestSpace(long sessionId, long blockId, long additionalBytes); - - /** - * Creates a writer to write data to a temp block. Since the temp block is "private" to the - * writer, this operation requires no previously acquired lock. - * - * @param sessionId the id of the session to get the writer - * @param blockId the id of the temp block - * @return a {@link BlockWriter} instance on this block - */ - BlockWriter createBlockWriter(long sessionId, long blockId); - - /** - * Creates a reader of an existing block to read data from this block. - *

- * This operation requires the lock id returned by a previously acquired - * {@link #pinBlock(long, long)}. - * - * @param sessionId the id of the session to get the reader - * @param blockId the id of an existing block - * @param offset the offset within the block - * @return a {@link BlockReader} instance on this block - * @throws BlockDoesNotExistRuntimeException if lockId is not found - */ - BlockReader createBlockReader(long sessionId, long blockId, long offset) - throws IOException; - - /** - * Creates a reader of an existing block to read data from this block. - *

- * The block reader will fetch the data from UFS when the data is not cached by the worker - * - * @param sessionId the id of the session to get the reader - * @param blockId the id of an existing block - * @param options the options for UFS fall-back - * @return a {@link BlockReader} instance on this block - */ - default BlockReader createBlockReader(long sessionId, long blockId, - Protocol.OpenUfsBlockOptions options) { - throw new UnsupportedOperationException(); - } - - /** - * Moves an existing block to a new location. - * - * @param sessionId the id of the session to move a block - * @param blockId the id of an existing block - * @param moveOptions the options for move - * block - */ - void moveBlock(long sessionId, long blockId, AllocateOptions moveOptions) - throws IOException; - - /** - * Removes an existing block. If the block can not be found in this store. - * - * @param sessionId the id of the session to remove a block - * @param blockId the id of an existing block - */ - void removeBlock(long sessionId, long blockId) throws IOException; - - /** - * Notifies the block store that a block was accessed so the block store could update accordingly - * the registered listeners such as evictor and allocator on block access. - * - * @param sessionId the id of the session to access a block - * @param blockId the id of an accessed block - */ - void accessBlock(long sessionId, long blockId); - - /** - * Gets the metadata of the entire store in a snapshot. There is no guarantee the state will be - * consistent with the snapshot after this method is called. - * This function should be cheap since it is called for every block. - * - * @return store metadata - */ - BlockStoreMeta getBlockStoreMeta(); - - /** - * Similar as {@link #getBlockStoreMeta} except that this includes - * more information about the block store (e.g. blockId list). This is an expensive operation. - * - * @return full store metadata - */ - BlockStoreMeta getBlockStoreMetaFull(); - - /** - * Checks if the storage has a given block. - * - * @param blockId the block id - * @return true if the block is contained, false otherwise - */ - boolean hasBlockMeta(long blockId); - - /** - * Checks if the storage has a given temp block. - * - * @param blockId the temp block id - * @return true if the block is contained, false otherwise - */ - boolean hasTempBlockMeta(long blockId); - - /** - * Cleans up the data associated with a specific session (typically a dead session). Clean up - * entails unlocking the block locks of this session, reclaiming space of temp blocks created by - * this session, and deleting the session temporary folder. - * - * @param sessionId the session id - */ - @Override - void cleanupSession(long sessionId); - - /** - * Registers a {@link BlockStoreEventListener} to this block store. - * - * @param listener the listener to those events - */ - void registerBlockStoreEventListener(BlockStoreEventListener listener); - - /** - * Update the pinned inodes. - * - * @param inodes a set of inodes that are currently pinned - */ - void updatePinnedInodes(Set inodes); - - /** - * Remove Storage directories that are no longer accessible. - */ - void removeInaccessibleStorage(); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/MonoBlockStore.java b/core/server/worker/src/main/java/alluxio/worker/block/MonoBlockStore.java deleted file mode 100644 index 187cbdcd7cec..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/MonoBlockStore.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static alluxio.worker.block.BlockMetadataManager.WORKER_STORAGE_TIER_ASSOC; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -import alluxio.Sessions; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; -import alluxio.exception.runtime.DeadlineExceededRuntimeException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.NotFoundException; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.Block; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.UfsReadOptions; -import alluxio.network.protocol.databuffer.NioDirectBufferPool; -import alluxio.proto.dataserver.Protocol; -import alluxio.retry.ExponentialBackoffRetry; -import alluxio.retry.RetryUtils; -import alluxio.underfs.UfsManager; -import alluxio.util.ThreadFactoryUtils; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.io.DelegatingBlockReader; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.TempBlockMeta; -import alluxio.worker.grpc.GrpcExecutors; - -import com.google.common.base.Strings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -/** - * A implementation of BlockStore. - * Each block will be stored and processed as a complete unit. - */ -public class MonoBlockStore implements BlockStore { - private static final Logger LOG = LoggerFactory.getLogger(MonoBlockStore.class); - private static final long LOAD_TIMEOUT = - Configuration.getMs(PropertyKey.USER_NETWORK_RPC_KEEPALIVE_TIMEOUT); - private final LocalBlockStore mLocalBlockStore; - private final UnderFileSystemBlockStore mUnderFileSystemBlockStore; - private final BlockMasterClientPool mBlockMasterClientPool; - private final AtomicReference mWorkerId; - private final ScheduledExecutorService mDelayer = - new ScheduledThreadPoolExecutor(1, ThreadFactoryUtils.build("LoadTimeOut", true)); - - /** - * Constructor of MonoBlockStore. - * - * @param localBlockStore - * @param blockMasterClientPool - * @param ufsManager - * @param workerId - */ - public MonoBlockStore(LocalBlockStore localBlockStore, - BlockMasterClientPool blockMasterClientPool, - UfsManager ufsManager, - AtomicReference workerId) { - mLocalBlockStore = requireNonNull(localBlockStore); - mBlockMasterClientPool = requireNonNull(blockMasterClientPool); - mUnderFileSystemBlockStore = - new UnderFileSystemBlockStore(localBlockStore, requireNonNull(ufsManager)); - mWorkerId = workerId; - } - - @Override - public void abortBlock(long sessionId, long blockId) { - mLocalBlockStore.abortBlock(sessionId, blockId); - } - - @Override - public void accessBlock(long sessionId, long blockId) { - mLocalBlockStore.accessBlock(sessionId, blockId); - } - - @Override - public void cleanupSession(long sessionId) { - mLocalBlockStore.cleanupSession(sessionId); - } - - @Override - public void commitBlock(long sessionId, long blockId, boolean pinOnCreate) { - // TODO(calvin): Reconsider how to do this without heavy locking. - // Block successfully committed, update master with new block metadata - BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire(); - try (BlockLock lock = mLocalBlockStore.commitBlockLocked(sessionId, blockId, pinOnCreate)) { - BlockMeta meta = mLocalBlockStore.getVolatileBlockMeta(blockId).get(); - BlockStoreLocation loc = meta.getBlockLocation(); - blockMasterClient.commitBlock(mWorkerId.get(), - mLocalBlockStore.getBlockStoreMeta().getUsedBytesOnTiers().get(loc.tierAlias()), - loc.tierAlias(), loc.mediumType(), blockId, meta.getBlockSize()); - } catch (AlluxioStatusException e) { - throw AlluxioRuntimeException.from(e); - } finally { - mBlockMasterClientPool.release(blockMasterClient); - DefaultBlockWorker.Metrics.WORKER_ACTIVE_CLIENTS.dec(); - } - } - - @Override - public String createBlock(long sessionId, long blockId, int tier, - CreateBlockOptions createBlockOptions) { - BlockStoreLocation loc; - String tierAlias = WORKER_STORAGE_TIER_ASSOC.getAlias(tier); - if (Strings.isNullOrEmpty(createBlockOptions.getMedium())) { - loc = BlockStoreLocation.anyDirInTier(tierAlias); - } else { - loc = BlockStoreLocation.anyDirInAnyTierWithMedium(createBlockOptions.getMedium()); - } - TempBlockMeta createdBlock; - createdBlock = mLocalBlockStore.createBlock(sessionId, blockId, - AllocateOptions.forCreate(createBlockOptions.getInitialBytes(), loc)); - DefaultBlockWorker.Metrics.WORKER_ACTIVE_CLIENTS.inc(); - return createdBlock.getPath(); - } - - @Override - public BlockReader createBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException { - BlockReader reader; - Optional blockMeta = mLocalBlockStore.getVolatileBlockMeta(blockId); - if (blockMeta.isPresent()) { - reader = mLocalBlockStore.createBlockReader(sessionId, blockId, offset); - } else { - boolean checkUfs = options != null && (options.hasUfsPath() || options.getBlockInUfsTier()); - if (!checkUfs) { - throw new BlockDoesNotExistRuntimeException(blockId); - } - // When the block does not exist in Alluxio but exists in UFS, try to open the UFS block. - reader = createUfsBlockReader(sessionId, blockId, offset, positionShort, options); - } - DefaultBlockWorker.Metrics.WORKER_ACTIVE_CLIENTS.inc(); - return reader; - } - - @Override - public BlockReader createUfsBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, - Protocol.OpenUfsBlockOptions options) - throws IOException { - try { - BlockReader reader = mUnderFileSystemBlockStore.createBlockReader(sessionId, blockId, offset, - positionShort, options); - return new DelegatingBlockReader(reader, () -> closeUfsBlock(sessionId, blockId)); - } catch (Exception e) { - try { - closeUfsBlock(sessionId, blockId); - } catch (Exception ee) { - LOG.warn("Failed to close UFS block", ee); - } - String errorMessage = format("Failed to read from UFS, sessionId=%d, " - + "blockId=%d, offset=%d, positionShort=%s, options=%s: %s", - sessionId, blockId, offset, positionShort, options, e); - if (e instanceof FileNotFoundException) { - throw new NotFoundException(errorMessage, e); - } - throw new UnavailableException(errorMessage, e); - } - } - - private void closeUfsBlock(long sessionId, long blockId) - throws IOException { - try { - mUnderFileSystemBlockStore.closeBlock(sessionId, blockId); - Optional tempBlockMeta = mLocalBlockStore.getTempBlockMeta(blockId); - if (tempBlockMeta.isPresent() && tempBlockMeta.get().getSessionId() == sessionId) { - commitBlock(sessionId, blockId, false); - } else { - // When getTempBlockMeta() return null, such as a block readType NO_CACHE writeType THROUGH. - // Counter will not be decrement in the commitblock(). - // So we should decrement counter here. - if (mUnderFileSystemBlockStore.isNoCache(sessionId, blockId)) { - DefaultBlockWorker.Metrics.WORKER_ACTIVE_CLIENTS.dec(); - } - } - } finally { - mUnderFileSystemBlockStore.releaseAccess(sessionId, blockId); - } - } - - @Override - public BlockWriter createBlockWriter(long sessionId, long blockId) throws IOException { - return mLocalBlockStore.createBlockWriter(sessionId, blockId); - } - - @Override - public BlockStoreMeta getBlockStoreMeta() { - return mLocalBlockStore.getBlockStoreMeta(); - } - - @Override - public BlockStoreMeta getBlockStoreMetaFull() { - return mLocalBlockStore.getBlockStoreMetaFull(); - } - - @Override - public Optional getTempBlockMeta(long blockId) { - return mLocalBlockStore.getTempBlockMeta(blockId); - } - - @Override - public boolean hasBlockMeta(long blockId) { - return mLocalBlockStore.hasBlockMeta(blockId); - } - - @Override - public boolean hasTempBlockMeta(long blockId) { - return mLocalBlockStore.hasTempBlockMeta(blockId); - } - - @Override - public Optional getVolatileBlockMeta(long blockId) { - return mLocalBlockStore.getVolatileBlockMeta(blockId); - } - - @Override - public void moveBlock(long sessionId, long blockId, AllocateOptions moveOptions) - throws IOException { - mLocalBlockStore.moveBlock(sessionId, blockId, moveOptions); - } - - @Override - public Optional pinBlock(long sessionId, long blockId) { - return mLocalBlockStore.pinBlock(sessionId, blockId); - } - - @Override - public void unpinBlock(BlockLock lock) { - lock.close(); - } - - @Override - public void updatePinnedInodes(Set inodes) { - mLocalBlockStore.updatePinnedInodes(inodes); - } - - @Override - public void registerBlockStoreEventListener(BlockStoreEventListener listener) { - mLocalBlockStore.registerBlockStoreEventListener(listener); - } - - @Override - public void removeBlock(long sessionId, long blockId) throws IOException { - mLocalBlockStore.removeBlock(sessionId, blockId); - } - - @Override - public void removeInaccessibleStorage() { - mLocalBlockStore.removeInaccessibleStorage(); - } - - @Override - public void requestSpace(long sessionId, long blockId, long additionalBytes) { - mLocalBlockStore.requestSpace(sessionId, blockId, additionalBytes); - } - - @Override - public CompletableFuture> load(List blocks, UfsReadOptions options) { - ArrayList> futures = new ArrayList<>(); - List errors = Collections.synchronizedList(new ArrayList<>()); - long sessionId = Sessions.LOAD_SESSION_ID; - for (Block block : blocks) { - long blockId = block.getBlockId(); - long blockSize = block.getLength(); - BlockWriter blockWriter; - UfsIOManager manager; - BlockStoreLocation loc = - BlockStoreLocation.anyDirInTier(WORKER_STORAGE_TIER_ASSOC.getAlias(0)); - try { - manager = mUnderFileSystemBlockStore.getOrAddUfsIOManager(block.getMountId()); - if (options.hasBandwidth()) { - manager.setQuota(options.getTag(), options.getBandwidth()); - } - mLocalBlockStore.createBlock(sessionId, blockId, AllocateOptions.forCreate(blockSize, loc)); - blockWriter = mLocalBlockStore.createBlockWriter(sessionId, blockId); - } catch (Exception e) { - handleException(e, block, errors, sessionId); - continue; - } - ByteBuffer buf = NioDirectBufferPool.acquire((int) blockSize); - CompletableFuture future = RetryUtils.retryCallable("read from ufs", - () -> manager.read(buf, block.getOffsetInFile(), blockSize, blockId, - block.getUfsPath(), options), - new ExponentialBackoffRetry(1000, 5000, 5)) - // use orTimeout in java 11 - .applyToEither(timeoutAfter(LOAD_TIMEOUT, TimeUnit.MILLISECONDS), d -> d) - .thenRunAsync(() -> { - buf.flip(); - blockWriter.append(buf); - }, GrpcExecutors.BLOCK_WRITER_EXECUTOR) - .thenRun(() -> { - try { - blockWriter.close(); - } catch (IOException e) { - throw AlluxioRuntimeException.from(e); - } finally { - NioDirectBufferPool.release(buf); - } - }) - .thenRun(() -> commitBlock(sessionId, blockId, false)) - .exceptionally(t -> { - handleException(t.getCause(), block, errors, sessionId); - return null; - }); - futures.add(future); - } - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .thenApply(x -> errors); - } - - private void handleException(Throwable e, Block block, List errors, long sessionId) { - LOG.warn("Load block failure: {}", block, e); - AlluxioRuntimeException exception = AlluxioRuntimeException.from(e); - BlockStatus.Builder builder = BlockStatus.newBuilder().setBlock(block) - .setCode(exception.getStatus().getCode().value()).setRetryable(exception.isRetryable()); - if (exception.getMessage() != null) { - builder.setMessage(exception.getMessage()); - } - errors.add(builder.build()); - if (hasTempBlockMeta(block.getBlockId())) { - try { - abortBlock(sessionId, block.getBlockId()); - } catch (Exception ee) { - LOG.warn(format("fail to abort temp block %s after failing to load block", - block.getBlockId()), ee); - } - } - } - - private CompletableFuture timeoutAfter(long timeout, TimeUnit unit) { - CompletableFuture result = new CompletableFuture<>(); - mDelayer.schedule(() -> result.completeExceptionally(new DeadlineExceededRuntimeException( - format("time out after waiting for %s %s", timeout, unit))), timeout, unit); - return result; - } - - @Override - public void close() throws IOException { - mLocalBlockStore.close(); - mUnderFileSystemBlockStore.close(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/PinListSync.java b/core/server/worker/src/main/java/alluxio/worker/block/PinListSync.java deleted file mode 100644 index a85a50092a3c..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/PinListSync.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.worker.file.FileSystemMasterClient; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Set; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * PinListSync periodically syncs the set of pinned inodes from master, and saves the new pinned - * inodes to the {@link BlockWorker}. - * - */ -@NotThreadSafe -public final class PinListSync implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(PinListSync.class); - - /** Block worker handle responsible for interacting with Alluxio and UFS storage. */ - private final BlockWorker mBlockWorker; - - /** Client for all master communication. */ - private final FileSystemMasterClient mMasterClient; - - /** - * Creates a new instance of {@link PinListSync}. - * - * @param blockWorker the block worker handle - * @param masterClient the Alluxio master client - */ - public PinListSync(BlockWorker blockWorker, FileSystemMasterClient masterClient) { - mBlockWorker = blockWorker; - mMasterClient = masterClient; - } - - @Override - public void heartbeat() { - // Send the sync - try { - Set pinList = mMasterClient.getPinList(); - mBlockWorker.updatePinList(pinList); - } catch (Exception e) { - // An error occurred, retry after 1 second or error if sync timeout is reached - LOG.warn("Failed to receive pinlist: {}", e.toString()); - LOG.debug("Exception: ", e); - // TODO(gene): Add this method to AbstractMasterClient. - // mMasterClient.resetConnection(); - } - } - - @Override - public void close() { - // Nothing to clean up - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/RemoteBlockReader.java b/core/server/worker/src/main/java/alluxio/worker/block/RemoteBlockReader.java deleted file mode 100644 index 42bdb1ed98f8..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/RemoteBlockReader.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.client.block.stream.BlockInStream; -import alluxio.client.file.FileSystemContext; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proto.dataserver.Protocol; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.io.BlockReader; - -import com.codahale.metrics.Counter; -import com.google.common.base.Preconditions; -import io.netty.buffer.ByteBuf; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; - -/** - * Reads a block from a remote worker node. This should only be used for reading entire blocks. - */ -public class RemoteBlockReader extends BlockReader { - private static final Counter BLOCKS_READ_REMOTE = - MetricsSystem.counter(MetricKey.WORKER_BLOCKS_READ_REMOTE.getName()); - - private final long mBlockId; - private final long mBlockSize; - private final InetSocketAddress mDataSource; - private final Protocol.OpenUfsBlockOptions mUfsOptions; - private final FileSystemContext mFsContext; - - private BlockInStream mInputStream; - private ReadableByteChannel mChannel; - private boolean mClosed; - - /** - * Constructs a remote block reader. It will read from a remote worker based on the data source. - * - * @param fsContext the filesystem context to use to create {@link BlockInStream} - * @param blockId the block to cache - * @param blockSize the size of the block - * @param dataSource the data source to cache from - * @param ufsOptions the options to read the block from ufs if necessary - */ - public RemoteBlockReader(FileSystemContext fsContext, long blockId, long blockSize, - InetSocketAddress dataSource, - Protocol.OpenUfsBlockOptions ufsOptions) { - mBlockId = blockId; - mBlockSize = blockSize; - mDataSource = dataSource; - mUfsOptions = ufsOptions; - mClosed = false; - mFsContext = fsContext; - } - - @Override - public ByteBuffer read(long offset, long length) throws IOException { - throw new UnsupportedOperationException("RemoteBlockReader#read is not supported"); - } - - @Override - public long getLength() { - return mUfsOptions.getBlockSize(); - } - - @Override - public ReadableByteChannel getChannel() { - Preconditions.checkState(!mClosed); - init(); - return mChannel; - } - - @Override - public int transferTo(ByteBuf buf) throws IOException { - Preconditions.checkState(!mClosed); - init(); - if (mInputStream == null || mInputStream.remaining() <= 0) { - return -1; - } - int bytesToRead = (int) Math.min(buf.writableBytes(), mInputStream.remaining()); - return buf.writeBytes(mInputStream, bytesToRead); - } - - @Override - public boolean isClosed() { - return mClosed; - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - super.close(); - if (mInputStream != null) { - mInputStream.close(); - mChannel.close(); - } - mClosed = true; - BLOCKS_READ_REMOTE.inc(); - } - - @Override - public String getLocation() { - return mDataSource.toString(); - } - - private void init() { - if (mInputStream != null) { - return; - } - WorkerNetAddress address = new WorkerNetAddress().setHost(mDataSource.getHostName()) - .setDataPort(mDataSource.getPort()); - mInputStream = BlockInStream.createRemoteBlockInStream(mFsContext, - mBlockId, - address, BlockInStream.BlockInStreamSource.REMOTE, mBlockSize, mUfsOptions); - mChannel = Channels.newChannel(mInputStream); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/TieredBlockStore.java b/core/server/worker/src/main/java/alluxio/worker/block/TieredBlockStore.java deleted file mode 100644 index b9806903f125..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/TieredBlockStore.java +++ /dev/null @@ -1,966 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static com.google.common.base.Preconditions.checkState; -import static java.lang.String.format; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.exception.status.DeadlineExceededException; -import alluxio.master.block.BlockId; -import alluxio.resource.LockResource; -import alluxio.util.io.FileUtils; -import alluxio.worker.block.allocator.Allocator; -import alluxio.worker.block.annotator.BlockIterator; -import alluxio.worker.block.annotator.BlockOrder; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.io.DelegatingBlockReader; -import alluxio.worker.block.io.StoreBlockReader; -import alluxio.worker.block.io.StoreBlockWriter; -import alluxio.worker.block.management.DefaultStoreLoadTracker; -import alluxio.worker.block.management.ManagementTaskCoordinator; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTier; -import alluxio.worker.block.meta.TempBlockMeta; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.text.MessageFormat; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import javax.annotation.concurrent.ThreadSafe; - -/** - * This class represents an object store that manages all the blocks in the local tiered storage. - * This store exposes simple public APIs to operate blocks. Inside this store, it creates an - * Allocator to decide where to put a new block, an Evictor to decide where to evict a stale block, - * a BlockMetadataManager to maintain the status of the tiered storage, and a LockManager to - * coordinate read/write on the same block. - *

- * This class is thread-safe, using the following lock hierarchy to ensure thread-safety: - *

    - *
  • Any block-level operation (e.g., read, move or remove) on an existing block must acquire a - * block lock for this block via {@link TieredBlockStore#mLockManager}. This block lock is a - * read/write lock, guarding both the metadata operations and the following I/O on this block. It - * coordinates different threads (clients) when accessing the same block concurrently.
    • - *
  • Any metadata operation (read or write) must go through {@link TieredBlockStore#mMetaManager} - * and guarded by {@link TieredBlockStore#mMetadataLock}. This is also a read/write lock and - * coordinates different threads (clients) when accessing the shared data structure for metadata. - *
    • - *
  • Method {@link #createBlock} does not acquire the block lock, because it only creates a - * temp block which is only visible to its writer before committed (thus no concurrent access).
    • - *
  • Method {@link #abortBlock(long, long)} does not acquire the block lock, because only - * temporary blocks can be aborted, and they are only visible to their writers (thus no concurrent - * access). - *
  • Eviction is done in {@link #freeSpace} and it is on the basis of best effort. For - * operations that may trigger this eviction (e.g., move, create, requestSpace), retry is used
    • - *
- */ -@ThreadSafe -public class TieredBlockStore implements LocalBlockStore { - private static final Logger LOG = LoggerFactory.getLogger(TieredBlockStore.class); - private static final Long REMOVE_BLOCK_TIMEOUT_MS = 60_000L; - private static final long FREE_AHEAD_BYTETS = - Configuration.getBytes(PropertyKey.WORKER_TIERED_STORE_FREE_AHEAD_BYTES); - private final BlockMetadataManager mMetaManager; - private final BlockLockManager mLockManager; - private final Allocator mAllocator; - - private final List mBlockStoreEventListeners = - new CopyOnWriteArrayList<>(); - - /** A set of pinned inodes fetched from the master. */ - private final Set mPinnedInodes = new HashSet<>(); - - /** Lock to guard metadata operations. */ - private final ReentrantReadWriteLock mMetadataLock = new ReentrantReadWriteLock(); - - /** ReadLock provided by {@link #mMetadataLock} to guard metadata read operations. */ - private final Lock mMetadataReadLock = mMetadataLock.readLock(); - - /** WriteLock provided by {@link #mMetadataLock} to guard metadata write operations. */ - private final Lock mMetadataWriteLock = mMetadataLock.writeLock(); - - /** Management task coordinator. */ - private final ManagementTaskCoordinator mTaskCoordinator; - - /** - * Creates a new instance of {@link TieredBlockStore}. - */ - public TieredBlockStore() { - this(BlockMetadataManager.createBlockMetadataManager(), new BlockLockManager()); - } - - /** - * Creates a new instance of {@link TieredBlockStore}. - * - * @param metaManager the block metadata manager - * @param lockManager the lock manager - */ - @VisibleForTesting - public TieredBlockStore(BlockMetadataManager metaManager, - BlockLockManager lockManager) { - mMetaManager = metaManager; - mLockManager = lockManager; - - BlockIterator blockIterator = mMetaManager.getBlockIterator(); - // Register listeners required by the block iterator. - for (BlockStoreEventListener listener : blockIterator.getListeners()) { - registerBlockStoreEventListener(listener); - } - - BlockMetadataEvictorView initManagerView = new BlockMetadataEvictorView(mMetaManager, - Collections.emptySet(), Collections.emptySet()); - mAllocator = Allocator.Factory.create(initManagerView); - if (mAllocator instanceof BlockStoreEventListener) { - registerBlockStoreEventListener((BlockStoreEventListener) mAllocator); - } - - // Initialize and start coordinator. - mTaskCoordinator = new ManagementTaskCoordinator(this, mMetaManager, - new DefaultStoreLoadTracker(), this::getUpdatedView); - mTaskCoordinator.start(); - } - - @Override - public Optional pinBlock(long sessionId, long blockId) { - LOG.debug("pinBlock: sessionId={}, blockId={}", sessionId, blockId); - BlockLock lock = mLockManager.acquireBlockLock(sessionId, blockId, BlockLockType.READ); - if (hasBlockMeta(blockId)) { - return Optional.of(lock); - } - lock.close(); - return Optional.empty(); - } - - @Override - public BlockWriter createBlockWriter(long sessionId, long blockId) { - LOG.debug("getBlockWriter: sessionId={}, blockId={}", sessionId, blockId); - // NOTE: a temp block is supposed to only be visible by its own writer, unnecessary to acquire - // block lock here since no sharing - // TODO(bin): Handle the case where multiple writers compete for the same block. - checkBlockDoesNotExist(blockId); - return new StoreBlockWriter(checkAndGetTempBlockMeta(sessionId, blockId)); - } - - @Override - public BlockReader createBlockReader(long sessionId, long blockId, long offset) - throws IOException { - LOG.debug("createBlockReader: sessionId={}, blockId={}, offset={}", - sessionId, blockId, offset); - Closeable blockLock = mLockManager.acquireBlockLock(sessionId, blockId, BlockLockType.READ); - Optional blockMeta; - try (LockResource r = new LockResource(mMetadataReadLock)) { - blockMeta = mMetaManager.getBlockMeta(blockId); - } - if (!blockMeta.isPresent()) { - blockLock.close(); - throw new BlockDoesNotExistRuntimeException(blockId); - } - try { - BlockReader reader = new StoreBlockReader(sessionId, blockMeta.get()); - ((FileChannel) reader.getChannel()).position(offset); - accessBlock(sessionId, blockId); - return new DelegatingBlockReader(reader, blockLock); - } catch (Exception e) { - blockLock.close(); - throw new IOException(format("Failed to get local block reader, sessionId=%d, " - + "blockId=%d, offset=%d", sessionId, blockId, offset), e); - } - } - - @Override - public TempBlockMeta createBlock(long sessionId, long blockId, AllocateOptions options) { - LOG.debug("createBlock: sessionId={}, blockId={}, options={}", sessionId, blockId, options); - TempBlockMeta tempBlockMeta = createBlockMetaInternal(sessionId, blockId, true, options); - createBlockFile(tempBlockMeta.getPath()); - return tempBlockMeta; - } - - // TODO(bin): Make this method to return a snapshot. - @Override - public Optional getVolatileBlockMeta(long blockId) { - LOG.debug("getVolatileBlockMeta: blockId={}", blockId); - try (LockResource r = new LockResource(mMetadataReadLock)) { - return mMetaManager.getBlockMeta(blockId); - } - } - - @Override - public Optional getTempBlockMeta(long blockId) { - LOG.debug("getTempBlockMeta: blockId={}", blockId); - try (LockResource r = new LockResource(mMetadataReadLock)) { - return mMetaManager.getTempBlockMeta(blockId); - } - } - - @Override - public void commitBlock(long sessionId, long blockId, boolean pinOnCreate) { - LOG.debug("commitBlock: sessionId={}, blockId={}, pinOnCreate={}", - sessionId, blockId, pinOnCreate); - try (BlockLock lock = mLockManager.acquireBlockLock(sessionId, blockId, BlockLockType.WRITE)) { - BlockStoreLocation loc = commitBlockInternal(sessionId, blockId, pinOnCreate); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onCommitBlock(blockId, loc); - } - } - } - } - - @Override - public BlockLock commitBlockLocked(long sessionId, long blockId, boolean pinOnCreate) { - LOG.debug("commitBlock: sessionId={}, blockId={}, pinOnCreate={}", - sessionId, blockId, pinOnCreate); - BlockLock lock = mLockManager.acquireBlockLock(sessionId, blockId, BlockLockType.WRITE); - BlockStoreLocation loc; - try { - loc = commitBlockInternal(sessionId, blockId, pinOnCreate); - } catch (RuntimeException e) { - lock.close(); - throw e; - } - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onCommitBlock(blockId, loc); - } - } - return lock; - } - - @Override - public void abortBlock(long sessionId, long blockId) { - LOG.debug("abortBlock: sessionId={}, blockId={}", sessionId, blockId); - abortBlockInternal(sessionId, blockId); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onAbortBlock(blockId); - } - } - } - - @Override - public void requestSpace(long sessionId, long blockId, long additionalBytes) { - LOG.debug("requestSpace: sessionId={}, blockId={}, additionalBytes={}", sessionId, blockId, - additionalBytes); - if (additionalBytes <= 0) { - return; - } - // NOTE: a temp block is only visible to its own writer, unnecessary to acquire - // block lock here since no sharing - try (LockResource r = new LockResource(mMetadataWriteLock)) { - TempBlockMeta tempBlockMeta = checkAndGetTempBlockMeta(sessionId, blockId); - BlockStoreLocation location = tempBlockMeta.getBlockLocation(); - StorageDirView allocationDir = allocateSpace(sessionId, - AllocateOptions.forRequestSpace(additionalBytes, location)); - checkState(allocationDir.toBlockStoreLocation().equals(location), - format("Allocation error: location enforcement failed for location: %s", - allocationDir.toBlockStoreLocation())); - // Increase the size of this temp block - mMetaManager.resizeTempBlockMeta(tempBlockMeta, - tempBlockMeta.getBlockSize() + additionalBytes); - } - } - - @Override - public void moveBlock(long sessionId, long blockId, AllocateOptions moveOptions) - throws IOException { - LOG.debug("moveBlock: sessionId={}, blockId={}, options={}", sessionId, - blockId, moveOptions); - BlockMeta meta = getVolatileBlockMeta(blockId).orElseThrow( - () -> new IllegalStateException(ExceptionMessage.BLOCK_META_NOT_FOUND.getMessage(blockId))); - if (meta.getBlockLocation().belongsTo(moveOptions.getLocation())) { - return; - } - // Execute the block move if necessary - MoveBlockResult result = moveBlockInternal(sessionId, blockId, moveOptions); - if (result.getSuccess()) { - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onMoveBlockByClient(blockId, result.getSrcLocation(), - result.getDstLocation()); - } - } - return; - } - throw new ResourceExhaustedRuntimeException( - ExceptionMessage.NO_SPACE_FOR_BLOCK_MOVE.getMessage(moveOptions.getLocation(), blockId), - false); - } - - @Override - public void removeBlock(long sessionId, long blockId) throws IOException { - LOG.debug("removeBlock: sessionId={}, blockId={}", sessionId, blockId); - Optional blockMeta = removeBlockInternal( - sessionId, blockId, REMOVE_BLOCK_TIMEOUT_MS); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onRemoveBlockByClient(blockId); - blockMeta.ifPresent(meta -> listener.onRemoveBlock( - blockId, meta.getBlockLocation())); - } - } - } - - @VisibleForTesting - Optional removeBlockInternal(long sessionId, long blockId, long timeoutMs) - throws IOException { - Optional optionalLock = - mLockManager.tryAcquireBlockLock(sessionId, blockId, BlockLockType.WRITE, - timeoutMs, TimeUnit.MILLISECONDS); - if (!optionalLock.isPresent()) { - throw new DeadlineExceededException( - format("Can not acquire lock to remove block %d for session %d after %d ms", - blockId, sessionId, REMOVE_BLOCK_TIMEOUT_MS)); - } - - try (BlockLock lock = optionalLock.get(); - LockResource r = new LockResource(mMetadataWriteLock)) { - if (mMetaManager.hasTempBlockMeta(blockId)) { - throw new IllegalStateException( - ExceptionMessage.REMOVE_UNCOMMITTED_BLOCK.getMessage(blockId)); - } - Optional blockMeta = mMetaManager.getBlockMeta(blockId); - if (blockMeta.isPresent()) { - removeBlockFileAndMeta(blockMeta.get()); - } - return blockMeta; - } - } - - @Override - public void accessBlock(long sessionId, long blockId) { - LOG.debug("accessBlock: sessionId={}, blockId={}", sessionId, blockId); - Optional blockMeta; - try (LockResource r = new LockResource(mMetadataReadLock)) { - blockMeta = mMetaManager.getBlockMeta(blockId); - } - if (blockMeta.isPresent()) { - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onAccessBlock(blockId); - listener.onAccessBlock(blockId, blockMeta.get().getBlockLocation()); - } - } - } - } - - @Override - public void cleanupSession(long sessionId) { - LOG.debug("cleanupSession: sessionId={}", sessionId); - // Release all locks the session is holding. - mLockManager.cleanupSession(sessionId); - - // Collect a list of temp blocks the given session owns and abort all of them with best effort - List tempBlocksToRemove; - try (LockResource r = new LockResource(mMetadataReadLock)) { - tempBlocksToRemove = mMetaManager.getSessionTempBlocks(sessionId); - } - for (TempBlockMeta tempBlockMeta : tempBlocksToRemove) { - try { - LOG.warn("Clean up expired temporary block {} from session {}.", tempBlockMeta.getBlockId(), - sessionId); - abortBlockInternal(sessionId, tempBlockMeta.getBlockId()); - } catch (Exception e) { - LOG.error("Failed to cleanup tempBlock {}", tempBlockMeta.getBlockId(), e); - } - } - } - - @Override - public boolean hasBlockMeta(long blockId) { - LOG.debug("hasBlockMeta: blockId={}", blockId); - try (LockResource r = new LockResource(mMetadataReadLock)) { - return mMetaManager.hasBlockMeta(blockId); - } - } - - @Override - public boolean hasTempBlockMeta(long blockId) { - LOG.debug("hasBlockMeta: blockId={}", blockId); - try (LockResource r = new LockResource(mMetadataReadLock)) { - return mMetaManager.hasTempBlockMeta(blockId); - } - } - - @Override - public BlockStoreMeta getBlockStoreMeta() { - // Removed DEBUG logging because this is very noisy - // LOG.debug("getBlockStoreMeta:"); - BlockStoreMeta storeMeta; - try (LockResource r = new LockResource(mMetadataReadLock)) { - storeMeta = mMetaManager.getBlockStoreMeta(); - } - return storeMeta; - } - - @Override - public BlockStoreMeta getBlockStoreMetaFull() { - // Removed DEBUG logging because this is very noisy - // LOG.debug("getBlockStoreMetaFull:"); - BlockStoreMeta storeMeta; - try (LockResource r = new LockResource(mMetadataReadLock)) { - storeMeta = mMetaManager.getBlockStoreMetaFull(); - } - return storeMeta; - } - - @Override - public void registerBlockStoreEventListener(BlockStoreEventListener listener) { - LOG.debug("registerBlockStoreEventListener: listener={}", listener); - mBlockStoreEventListeners.add(listener); - } - - /** - * Checks if block id is a temporary block and owned by session id. This method must be enclosed - * by {@link #mMetadataLock}. - * - * @param sessionId the id of session - * @param blockId the id of block - */ - private TempBlockMeta checkAndGetTempBlockMeta(long sessionId, long blockId) { - Optional tempBlockMeta; - try (LockResource r = new LockResource(mMetadataReadLock)) { - tempBlockMeta = mMetaManager.getTempBlockMeta(blockId); - } - checkState(tempBlockMeta.isPresent(), - ExceptionMessage.TEMP_BLOCK_META_NOT_FOUND.getMessage(blockId)); - checkState(tempBlockMeta.get().getSessionId() == sessionId, - ExceptionMessage.BLOCK_ID_FOR_DIFFERENT_SESSION.getMessage(blockId, - tempBlockMeta.get().getSessionId(), sessionId)); - return tempBlockMeta.get(); - } - - private void checkBlockDoesNotExist(long blockId) { - checkState(!hasBlockMeta(blockId), - ExceptionMessage.TEMP_BLOCK_ID_COMMITTED.getMessage(blockId)); - } - - private void checkTempBlockDoesNotExist(long blockId) { - checkState(!hasTempBlockMeta(blockId), MessageFormat - .format("Temp blockId {0,number,#} is not available, because it already exists", blockId)); - } - - /** - * Aborts a temp block. - * - * @param sessionId the id of session - * @param blockId the id of block - */ - private void abortBlockInternal(long sessionId, long blockId) { - checkBlockDoesNotExist(blockId); - TempBlockMeta tempBlockMeta = checkAndGetTempBlockMeta(sessionId, blockId); - - // The metadata lock is released during heavy IO. The temp block is private to one session, so - // we do not lock it. - FileUtils.delete(tempBlockMeta.getPath()); - try (LockResource r = new LockResource(mMetadataWriteLock)) { - mMetaManager.abortTempBlockMeta(tempBlockMeta); - } - } - - /** - * Commits a temp block. - * - * @param sessionId the id of session - * @param blockId the id of block - * @param pinOnCreate is block pinned on create - * @return destination location to move the block - */ - private BlockStoreLocation commitBlockInternal(long sessionId, long blockId, - boolean pinOnCreate) { - if (mMetaManager.hasBlockMeta(blockId)) { - LOG.debug("Block {} has been in block store, this could be a retry due to master-side RPC " - + "failure", blockId); - return mMetaManager.getBlockMeta(blockId).get().getBlockLocation(); - } - // When committing TempBlockMeta, the final BlockMeta calculates the block size according to - // the actual file size of this TempBlockMeta. Therefore, commitTempBlockMeta must happen - // after moving actual block file to its committed path. - TempBlockMeta tempBlockMeta = checkAndGetTempBlockMeta(sessionId, blockId); - String srcPath = tempBlockMeta.getPath(); - String dstPath = tempBlockMeta.getCommitPath(); - BlockStoreLocation loc = tempBlockMeta.getBlockLocation(); - - // Heavy IO is guarded by block lock but not metadata lock. This may throw IOException. - try { - FileUtils.move(srcPath, dstPath); - } catch (IOException e) { - // TODO(jianjian) move IOException handling once we figure out how MoveResult works - throw AlluxioRuntimeException.from(e); - } - - try (LockResource r = new LockResource(mMetadataWriteLock)) { - mMetaManager.commitTempBlockMeta(tempBlockMeta); - } - - // Check if block is pinned on commit - if (pinOnCreate) { - addToPinnedInodes(BlockId.getFileId(blockId)); - } - - return loc; - } - - private StorageDirView allocateSpace(long sessionId, AllocateOptions options) { - StorageDirView dirView; - BlockMetadataView allocatorView = - new BlockMetadataAllocatorView(mMetaManager, options.canUseReservedSpace()); - // Convenient way to break on failure cases, no intention to loop - while (true) { - if (options.isForceLocation()) { - // Try allocating from given location. Skip the review because the location is forced. - dirView = mAllocator.allocateBlockWithView(options.getSize(), - options.getLocation(), allocatorView, true); - if (dirView != null) { - return dirView; - } - if (options.isEvictionAllowed()) { - LOG.debug("Free space for block expansion: freeing {} bytes on {}. ", - options.getSize(), options.getLocation()); - freeSpace(sessionId, options.getSize(), options.getSize(), options.getLocation()); - // Block expansion are forcing the location. We do not want the review's opinion. - dirView = mAllocator.allocateBlockWithView(options.getSize(), - options.getLocation(), allocatorView.refreshView(), true); - if (LOG.isDebugEnabled()) { - LOG.debug("Allocation after freeing space for block expansion, {}", dirView == null - ? "no available dir." : "available bytes in dir: " + dirView.getAvailableBytes()); - } - if (dirView == null) { - LOG.error("Target tier: {} has no evictable space to store {} bytes for session: {}", - options.getLocation(), options.getSize(), sessionId); - break; - } - } else { - // We are not evicting in the target tier so having no available space just - // means the tier is currently full. - LOG.warn("Target tier: {} has no available space to store {} bytes for session: {}", - options.getLocation(), options.getSize(), sessionId); - break; - } - } else { - // Try allocating from given location. This may be rejected by the review logic. - dirView = mAllocator.allocateBlockWithView(options.getSize(), - options.getLocation(), allocatorView, false); - if (dirView != null) { - return dirView; - } - LOG.debug("Allocate to anyTier for {} bytes on {}", options.getSize(), - options.getLocation()); - dirView = mAllocator.allocateBlockWithView(options.getSize(), - BlockStoreLocation.anyTier(), allocatorView, false); - if (dirView != null) { - return dirView; - } - - if (options.isEvictionAllowed()) { - // There is no space left on worker. - // Free more than requested by configured free-ahead size. - long toFreeBytes = options.getSize() + FREE_AHEAD_BYTETS; - LOG.debug("Allocation on anyTier failed. Free space for {} bytes on anyTier", - toFreeBytes); - freeSpace(sessionId, options.getSize(), toFreeBytes, - BlockStoreLocation.anyTier()); - // Skip the review as we want the allocation to be in the place we just freed - dirView = mAllocator.allocateBlockWithView(options.getSize(), - BlockStoreLocation.anyTier(), allocatorView.refreshView(), true); - if (LOG.isDebugEnabled()) { - LOG.debug("Allocation after freeing space for block creation, {} ", dirView == null - ? "no available dir." : "available bytes in dir: " + dirView.getAvailableBytes()); - } - } - } - if (dirView == null) { - break; - } - return dirView; - } - throw new ResourceExhaustedRuntimeException( - format("Allocation failure. Options: %s. Error:", options), false); - } - - /** - * Creates a temp block meta only if allocator finds available space. This method will not trigger - * any eviction. - * - * @param sessionId session id - * @param blockId block id - * @param newBlock true if this temp block is created for a new block - * @param options block allocation options - * @return a temp block created if successful - */ - private TempBlockMeta createBlockMetaInternal(long sessionId, long blockId, boolean newBlock, - AllocateOptions options) { - if (newBlock) { - checkBlockDoesNotExist(blockId); - checkTempBlockDoesNotExist(blockId); - } - try (LockResource r = new LockResource(mMetadataWriteLock)) { - // NOTE: a temp block is supposed to be visible for its own writer, - // unnecessary to acquire block lock here since no sharing. - // Allocate space. - StorageDirView dirView = allocateSpace(sessionId, options); - - // TODO(carson): Add tempBlock to corresponding storageDir and remove the use of - // StorageDirView.createTempBlockMeta. - TempBlockMeta tempBlock = dirView.createTempBlockMeta(sessionId, blockId, options.getSize()); - // Add allocated temp block to metadata manager. This should never fail if allocator - // correctly assigns a StorageDir. - mMetaManager.addTempBlockMeta(tempBlock); - return tempBlock; - } - } - - /** - * Free space is the entry for immediate block deletion in order to open up space for - * new or ongoing blocks. - * - * - New blocks creations will not try to free space until all tiers are out of space. - * - Ongoing blocks could end up freeing space oftenly, when the file's origin location is - * low on space. - * - * This method is synchronized in order to prevent race in its only client, allocations. - * If not synchronized, new allocations could steal space reserved by ongoing ones. - * Removing synchronized requires implementing retries to this call along with an optimal - * locking strategy for fairness. - * - * TODO(ggezer): Remove synchronized. - * - * @param sessionId the session id - * @param minContiguousBytes the minimum amount of contigious free space in bytes - * @param minAvailableBytes the minimum amount of free space in bytes - * @param location the location to free space - */ - @VisibleForTesting - public synchronized void freeSpace(long sessionId, long minContiguousBytes, - long minAvailableBytes, BlockStoreLocation location) { - LOG.debug("freeSpace: sessionId={}, minContiguousBytes={}, minAvailableBytes={}, location={}", - sessionId, minAvailableBytes, minAvailableBytes, location); - // TODO(ggezer): Too much memory pressure when pinned-inodes list is large. - BlockMetadataEvictorView evictorView = getUpdatedView(); - boolean contiguousSpaceFound = false; - boolean availableBytesFound = false; - - int blocksIterated = 0; - int blocksRemoved = 0; - int spaceFreed = 0; - - // List of all dirs that belong to the given location. - List dirViews = evictorView.getDirs(location); - - Iterator evictionCandidates = mMetaManager.getBlockIterator() - .getIterator(location, BlockOrder.NATURAL); - while (true) { - // Check if minContiguousBytes is satisfied. - if (!contiguousSpaceFound) { - for (StorageDirView dirView : dirViews) { - if (dirView.getAvailableBytes() >= minContiguousBytes) { - contiguousSpaceFound = true; - break; - } - } - } - - // Check minAvailableBytes is satisfied. - if (!availableBytesFound) { - if (evictorView.getAvailableBytes(location) >= minAvailableBytes) { - availableBytesFound = true; - } - } - - if (contiguousSpaceFound && availableBytesFound) { - break; - } - - if (!evictionCandidates.hasNext()) { - break; - } - - long blockToDelete = evictionCandidates.next(); - blocksIterated++; - if (evictorView.isBlockEvictable(blockToDelete)) { - Optional optionalBlockMeta = mMetaManager.getBlockMeta(blockToDelete); - if (!optionalBlockMeta.isPresent()) { - LOG.warn("Failed to evict blockId {}, it could be already deleted", blockToDelete); - continue; - } - BlockMeta blockMeta = optionalBlockMeta.get(); - removeBlockFileAndMeta(blockMeta); - blocksRemoved++; - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onRemoveBlockByWorker(blockMeta.getBlockId()); - listener.onRemoveBlock(blockMeta.getBlockId(), - blockMeta.getBlockLocation()); - } - } - spaceFreed += blockMeta.getBlockSize(); - } - } - - if (!contiguousSpaceFound || !availableBytesFound) { - throw new ResourceExhaustedRuntimeException( - format("Failed to free %d bytes space at location %s. " - + "Min contiguous requested: %d, Min available requested: %d, " - + "Blocks iterated: %d, Blocks removed: %d, Space freed: %d", - minAvailableBytes, location.tierAlias(), minContiguousBytes, minAvailableBytes, - blocksIterated, blocksRemoved, spaceFreed), false); - } - } - - /** - * Gets the most updated view with most recent information on pinned inodes, and currently locked - * blocks. - * - * @return {@link BlockMetadataEvictorView}, an updated view with most recent information - */ - private BlockMetadataEvictorView getUpdatedView() { - // TODO(calvin): Update the view object instead of creating new one every time. - synchronized (mPinnedInodes) { - return new BlockMetadataEvictorView(mMetaManager, mPinnedInodes, - mLockManager.getLockedBlocks()); - } - } - - /** - * Moves a block to new location only if allocator finds available space in newLocation. This - * method will not trigger any eviction. Returns {@link MoveBlockResult}. - * - * @param sessionId session id - * @param blockId block id - * @param moveOptions the allocate options for the move - * @return the resulting information about the move operation - */ - private MoveBlockResult moveBlockInternal(long sessionId, long blockId, - AllocateOptions moveOptions) throws IOException { - try (BlockLock lock = mLockManager.acquireBlockLock(sessionId, blockId, BlockLockType.WRITE)) { - checkTempBlockDoesNotExist(blockId); - BlockMeta srcBlockMeta; - try (LockResource r = new LockResource(mMetadataReadLock)) { - srcBlockMeta = mMetaManager.getBlockMeta(blockId).orElseThrow(() -> - new IllegalStateException(ExceptionMessage.BLOCK_META_NOT_FOUND.getMessage(blockId))); - } - - BlockStoreLocation srcLocation = srcBlockMeta.getBlockLocation(); - String srcFilePath = srcBlockMeta.getPath(); - long blockSize = srcBlockMeta.getBlockSize(); - // Update moveOptions with the block size. - moveOptions.setSize(blockSize); - if (srcLocation.belongsTo(moveOptions.getLocation())) { - return new MoveBlockResult(true, blockSize, srcLocation, srcLocation); - } - - TempBlockMeta dstTempBlock; - try { - dstTempBlock = createBlockMetaInternal(sessionId, blockId, false, moveOptions); - } catch (Exception e) { - return new MoveBlockResult(false, blockSize, null, null); - } - - // When `newLocation` is some specific location, the `newLocation` and the `dstLocation` are - // just the same; while for `newLocation` with a wildcard significance, the `dstLocation` - // is a specific one with specific tier and dir which belongs to newLocation. - BlockStoreLocation dstLocation = dstTempBlock.getBlockLocation(); - - // When the dstLocation belongs to srcLocation, simply abort the tempBlockMeta just created - // internally from the newLocation and return success with specific block location. - if (dstLocation.belongsTo(srcLocation)) { - mMetaManager.abortTempBlockMeta(dstTempBlock); - return new MoveBlockResult(true, blockSize, srcLocation, dstLocation); - } - String dstFilePath = dstTempBlock.getCommitPath(); - - // Heavy IO is guarded by block lock but not metadata lock. This may throw IOException. - FileUtils.move(srcFilePath, dstFilePath); - - try (LockResource r = new LockResource(mMetadataWriteLock)) { - // If this metadata update fails, we panic for now. - // TODO(bin): Implement rollback scheme to recover from IO failures. - mMetaManager.moveBlockMeta(srcBlockMeta, dstTempBlock); - } - return new MoveBlockResult(true, blockSize, srcLocation, dstLocation); - } - } - - /** - * Removes a block physically and from metadata. - * - * @param blockMeta block metadata - */ - private void removeBlockFileAndMeta(BlockMeta blockMeta) { - FileUtils.delete(blockMeta.getPath()); - mMetaManager.removeBlockMeta(blockMeta); - } - - /** - * Creates a file to represent a block denoted by the given block path. This file will be owned - * by the Alluxio worker but have 777 permissions so processes under users different from the - * user that launched the Alluxio worker can read and write to the file. The tiered storage - * directory has the sticky bit so only the worker user can delete or rename files it creates. - * - * @param blockPath the block path to create - */ - // TODO(peis): Consider using domain socket to avoid setting the permission to 777. - private static void createBlockFile(String blockPath) { - FileUtils.createBlockPath(blockPath, - Configuration.getString(PropertyKey.WORKER_DATA_FOLDER_PERMISSIONS)); - FileUtils.createFile(blockPath); - FileUtils.changeLocalFileToFullPermission(blockPath); - LOG.debug("Created new file block, block path: {}", blockPath); - } - - /** - * Updates the pinned blocks. - * - * @param inodes a set of ids inodes that are pinned - */ - @Override - public void updatePinnedInodes(Set inodes) { - LOG.debug("updatePinnedInodes: inodes={}", inodes); - synchronized (mPinnedInodes) { - mPinnedInodes.clear(); - mPinnedInodes.addAll(Preconditions.checkNotNull(inodes)); - } - } - - /** - * Add a single inode to set of pinned ids. - * - * @param inode an inode that is pinned - */ - private void addToPinnedInodes(Long inode) { - LOG.debug("addToPinnedInodes: inode={}", inode); - synchronized (mPinnedInodes) { - mPinnedInodes.add(Preconditions.checkNotNull(inode)); - } - } - - @Override - public void removeInaccessibleStorage() { - try (LockResource r = new LockResource(mMetadataWriteLock)) { - for (StorageTier tier : mMetaManager.getTiers()) { - for (StorageDir dir : tier.getStorageDirs()) { - String path = dir.getDirPath(); - if (!FileUtils.isStorageDirAccessible(path)) { - LOG.error("Storage check failed for path {}. The directory will be excluded.", path); - removeDir(dir); - } - } - } - } - } - - /** - * Removes a storage directory. - * - * @param dir storage directory to be removed - */ - public void removeDir(StorageDir dir) { - // TODO(feng): Add a command for manually removing directory - try (LockResource r = new LockResource(mMetadataWriteLock)) { - String tierAlias = dir.getParentTier().getTierAlias(); - dir.getParentTier().removeStorageDir(dir); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - dir.getBlockIds().forEach(listener::onBlockLost); - listener.onStorageLost(tierAlias, dir.getDirPath()); - listener.onStorageLost(dir.toBlockStoreLocation()); - } - } - } - } - - @Override - public void close() throws IOException { - mTaskCoordinator.close(); - } - - /** - * A wrapper on necessary info after a move block operation. - */ - private static class MoveBlockResult { - /** Whether this move operation succeeds. */ - private final boolean mSuccess; - /** Size of this block in bytes. */ - private final long mBlockSize; - /** Source location of this block to move. */ - private final BlockStoreLocation mSrcLocation; - /** Destination location of this block to move. */ - private final BlockStoreLocation mDstLocation; - - /** - * Creates a new instance of {@link MoveBlockResult}. - * - * @param success success indication - * @param blockSize block size - * @param srcLocation source location - * @param dstLocation destination location - */ - MoveBlockResult(boolean success, long blockSize, BlockStoreLocation srcLocation, - BlockStoreLocation dstLocation) { - mSuccess = success; - mBlockSize = blockSize; - mSrcLocation = srcLocation; - mDstLocation = dstLocation; - } - - /** - * @return the success indicator - */ - boolean getSuccess() { - return mSuccess; - } - - /** - * @return the block size - */ - long getBlockSize() { - return mBlockSize; - } - - /** - * @return the source location - */ - BlockStoreLocation getSrcLocation() { - return mSrcLocation; - } - - /** - * @return the destination location - */ - BlockStoreLocation getDstLocation() { - return mDstLocation; - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/UnderFileSystemBlockReader.java b/core/server/worker/src/main/java/alluxio/worker/block/UnderFileSystemBlockReader.java deleted file mode 100644 index cc3f66356618..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/UnderFileSystemBlockReader.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static alluxio.worker.block.BlockMetadataManager.WORKER_STORAGE_TIER_ASSOC; - -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.resource.CloseableResource; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.options.OpenOptions; -import alluxio.util.IdUtils; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.meta.UnderFileSystemBlockMeta; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; -import com.google.common.base.Preconditions; -import io.netty.buffer.ByteBuf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class implements a {@link BlockReader} to read a block directly from UFS, and - * optionally cache the block to the Alluxio worker if the whole block it is read. - */ -@NotThreadSafe -public final class UnderFileSystemBlockReader extends BlockReader { - private static final Logger LOG = LoggerFactory.getLogger(UnderFileSystemBlockReader.class); - - /** Metrics. */ - private static final Counter BLOCKS_READ_UFS = - MetricsSystem.counter(MetricKey.WORKER_BLOCKS_READ_UFS.getName()); - - private final Counter mUfsBytesRead; - private final Meter mUfsBytesReadThroughput; - - /** The initial size of the block allocated in Alluxio storage when the block is cached. */ - private final long mInitialBlockSize; - /** The block metadata for the UFS block. */ - private final UnderFileSystemBlockMeta mBlockMeta; - /** The Local block store. It is used to interact with Alluxio. */ - private final LocalBlockStore mLocalBlockStore; - /** The cache for all ufs instream. */ - private final UfsInputStreamCache mUfsInstreamCache; - /** The ufs client resource. */ - private final CloseableResource mUfsResource; - private final boolean mIsPositionShort; - - /** The input stream to read from UFS. */ - private InputStream mUnderFileSystemInputStream; - /** The block writer to write the block to Alluxio. */ - private BlockWriter mBlockWriter; - /** If set, the reader is closed and should not be used afterwards. */ - private boolean mClosed; - - /** - * The position of mUnderFileSystemInputStream (if not null) is blockStart + mInStreamPos. - * When mUnderFileSystemInputStream is not set, this is set to -1 (an invalid state) when - * mUnderFileSystemInputStream is null. Check mUnderFileSystemInputStream directly to see whether - * that is valid instead of relying on this invalid state of the position to be safe. - */ - private long mInStreamPos; - - /** - * Creates an instance of {@link UnderFileSystemBlockReader} and initializes it with a reading - * offset. - * - * @param blockMeta the block meta - * @param offset the position within the block to start the read - * @param localBlockStore the Local block store - * @param ufsClient the manager of ufs - * @param positionShort whether the client op is a positioned read to a small buffer - * @param ufsInStreamCache the UFS in stream cache - * @param ufsBytesRead counter metric to track ufs bytes read - * @param ufsBytesReadThroughput meter metric to track bytes read throughput - * @return the block reader - */ - public static UnderFileSystemBlockReader create(UnderFileSystemBlockMeta blockMeta, long offset, - boolean positionShort, LocalBlockStore localBlockStore, UfsManager.UfsClient ufsClient, - UfsInputStreamCache ufsInStreamCache, Counter ufsBytesRead, Meter ufsBytesReadThroughput) - throws IOException { - UnderFileSystemBlockReader ufsBlockReader = - new UnderFileSystemBlockReader(blockMeta, positionShort, localBlockStore, ufsClient, - ufsInStreamCache, ufsBytesRead, ufsBytesReadThroughput); - ufsBlockReader.init(offset); - return ufsBlockReader; - } - - /** - * Creates an instance of {@link UnderFileSystemBlockReader}. - * - * @param blockMeta the block meta - * @param localBlockStore the Local block store - * @param ufsClient the ufs client - * @param positionShort whether the client op is a positioned read to a small buffer - * @param ufsInStreamCache the UFS in stream cache - * @param ufsBytesRead counter metric to track ufs bytes read - * @param ufsBytesReadThroughput meter metric to track bytes read throughput - */ - private UnderFileSystemBlockReader(UnderFileSystemBlockMeta blockMeta, boolean positionShort, - LocalBlockStore localBlockStore, UfsManager.UfsClient ufsClient, - UfsInputStreamCache ufsInStreamCache, Counter ufsBytesRead, Meter ufsBytesReadThroughput) { - mInitialBlockSize = blockMeta.getBlockSize(); - mBlockMeta = blockMeta; - mLocalBlockStore = localBlockStore; - mInStreamPos = -1; - mUfsInstreamCache = ufsInStreamCache; - mUfsResource = ufsClient.acquireUfsResource(); - mIsPositionShort = positionShort; - mUfsBytesRead = ufsBytesRead; - mUfsBytesReadThroughput = ufsBytesReadThroughput; - } - - /** - * Initializes the reader. This is only called in the factory method. - * - * @param offset the position within the block to start the read - */ - private void init(long offset) throws IOException { - updateUnderFileSystemInputStream(offset); - updateBlockWriter(offset); - } - - @Override - public ReadableByteChannel getChannel() { - throw new UnsupportedOperationException("UFSFileBlockReader#getChannel is not supported"); - } - - @Override - public long getLength() { - return mBlockMeta.getBlockSize(); - } - - @Override - public ByteBuffer read(long offset, long length) throws IOException { - Preconditions.checkState(!mClosed); - updateUnderFileSystemInputStream(offset); - updateBlockWriter(offset); - - long bytesToRead = Math.min(length, mBlockMeta.getBlockSize() - offset); - if (bytesToRead <= 0) { - return ByteBuffer.allocate(0); - } - byte[] data = new byte[(int) bytesToRead]; - int bytesRead = 0; - Preconditions.checkNotNull(mUnderFileSystemInputStream, "mUnderFileSystemInputStream"); - while (bytesRead < bytesToRead) { - int read; - try { - read = mUnderFileSystemInputStream.read(data, bytesRead, (int) (bytesToRead - bytesRead)); - } catch (IOException e) { - throw AlluxioStatusException.fromIOException(e); - } - if (read == -1) { - break; - } - bytesRead += read; - } - mInStreamPos += bytesRead; - - // We should always read the number of bytes as expected since the UFS file length (hence block - // size) should be always accurate. - Preconditions.checkState(bytesRead == bytesToRead, - "Not enough bytes have been read [bytesRead: %s, bytesToRead: %s] from the UFS file: %s.", - bytesRead, bytesToRead, mBlockMeta.getUnderFileSystemPath()); - if (mBlockWriter != null && mBlockWriter.getPosition() < mInStreamPos) { - try { - Preconditions.checkState(mBlockWriter.getPosition() >= offset); - mLocalBlockStore.requestSpace(mBlockMeta.getSessionId(), mBlockMeta.getBlockId(), - mInStreamPos - mBlockWriter.getPosition()); - ByteBuffer buffer = ByteBuffer.wrap(data, (int) (mBlockWriter.getPosition() - offset), - (int) (mInStreamPos - mBlockWriter.getPosition())); - mBlockWriter.append(buffer.duplicate()); - } catch (Exception e) { - LOG.warn("Failed to cache data read from UFS (on read()): {}", e.toString()); - try { - cancelBlockWriter(); - } catch (IOException ee) { - LOG.error("Failed to cancel block writer:", ee); - } - } - } - mUfsBytesRead.inc(bytesRead); - mUfsBytesReadThroughput.mark(bytesRead); - return ByteBuffer.wrap(data, 0, bytesRead); - } - - /** - * This interface is supposed to be used for sequence block reads. - * - * @param buf the byte buffer - * @return the number of bytes read, -1 if it reaches EOF and none was read - */ - @Override - public int transferTo(ByteBuf buf) throws IOException { - Preconditions.checkState(!mClosed); - if (mUnderFileSystemInputStream == null) { - return -1; - } - if (mBlockMeta.getBlockSize() <= mInStreamPos) { - return -1; - } - // Make a copy of the state to keep track of what we have read in this transferTo call. - ByteBuf bufCopy = null; - if (mBlockWriter != null) { - bufCopy = buf.duplicate(); - bufCopy.readerIndex(bufCopy.writerIndex()); - } - int bytesToRead = - (int) Math.min(buf.writableBytes(), mBlockMeta.getBlockSize() - mInStreamPos); - int bytesRead = buf.writeBytes(mUnderFileSystemInputStream, bytesToRead); - if (bytesRead <= 0) { - return bytesRead; - } - - mInStreamPos += bytesRead; - - if (mBlockWriter != null && bufCopy != null) { - try { - bufCopy.writerIndex(buf.writerIndex()); - while (bufCopy.readableBytes() > 0) { - mLocalBlockStore.requestSpace(mBlockMeta.getSessionId(), mBlockMeta.getBlockId(), - mInStreamPos - mBlockWriter.getPosition()); - mBlockWriter.append(bufCopy); - } - } catch (Exception e) { - LOG.warn("Failed to cache data read from UFS (on transferTo()): {}", e.toString()); - cancelBlockWriter(); - } - } - mUfsBytesRead.inc(bytesRead); - mUfsBytesReadThroughput.mark(bytesRead); - return bytesRead; - } - - /** - * Closes the block reader. After this, this block reader should not be used anymore. - * This is recommended to be called after the client finishes reading the block. It is usually - * triggered when the client unlocks the block. - */ - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - - super.close(); - try { - // This aborts the block if the block is not fully read. - updateBlockWriter(mBlockMeta.getBlockSize()); - - if (mUnderFileSystemInputStream != null) { - mUfsInstreamCache.release(mUnderFileSystemInputStream); - mUnderFileSystemInputStream = null; - } - - if (mBlockWriter != null) { - mBlockWriter.close(); - } - - mUfsResource.close(); - } finally { - mClosed = true; - BLOCKS_READ_UFS.inc(); - } - } - - @Override - public boolean isClosed() { - return mClosed; - } - - @Override - public String getLocation() { - return mBlockMeta.getUnderFileSystemPath(); - } - - /** - * Updates the UFS input stream given an offset to read. - * - * @param offset the read offset within the block - */ - private void updateUnderFileSystemInputStream(long offset) throws IOException { - if ((mUnderFileSystemInputStream != null) && offset != mInStreamPos) { - mUfsInstreamCache.release(mUnderFileSystemInputStream); - mUnderFileSystemInputStream = null; - mInStreamPos = -1; - } - - if (mUnderFileSystemInputStream == null && offset < mBlockMeta.getBlockSize()) { - UnderFileSystem ufs = mUfsResource.get(); - mUnderFileSystemInputStream = mUfsInstreamCache - .acquire(ufs, mBlockMeta.getUnderFileSystemPath(), - IdUtils.fileIdFromBlockId(mBlockMeta.getBlockId()), - OpenOptions.defaults().setOffset(mBlockMeta.getOffset() + offset) - .setPositionShort(mIsPositionShort)); - mInStreamPos = offset; - } - } - - /** - * Closes the current block writer, cleans up its temp block and sets it to null. - */ - private void cancelBlockWriter() throws IOException { - if (mBlockWriter == null) { - return; - } - try { - mBlockWriter.close(); - mBlockWriter = null; - mLocalBlockStore.abortBlock(mBlockMeta.getSessionId(), mBlockMeta.getBlockId()); - } catch (Exception e) { - // We cannot skip the exception here because we need to make sure that the user of this - // reader does not commit the block if it fails to abort the block. - throw AlluxioStatusException.fromThrowable(e); - } - } - - /** - * Updates the block writer given an offset to read. If the offset is beyond the current - * position of the block writer, the block writer will be aborted. - * - * @param offset the read offset - */ - private void updateBlockWriter(long offset) throws IOException { - if (mBlockWriter != null && offset > mBlockWriter.getPosition()) { - cancelBlockWriter(); - } - try { - if (mBlockWriter == null && offset == 0 && !mBlockMeta.isNoCache()) { - BlockStoreLocation loc = BlockStoreLocation.anyDirInTier( - WORKER_STORAGE_TIER_ASSOC.getAlias(0)); - mLocalBlockStore.createBlock(mBlockMeta.getSessionId(), mBlockMeta.getBlockId(), - AllocateOptions.forCreate(mInitialBlockSize, loc)); - mBlockWriter = mLocalBlockStore.createBlockWriter( - mBlockMeta.getSessionId(), mBlockMeta.getBlockId()); - } - } catch (AlluxioRuntimeException e) { - LOG.warn( - "Failed to update block writer for UFS block [blockId: {}, ufsPath: {}, offset: {}]: {}", - mBlockMeta.getBlockId(), mBlockMeta.getUnderFileSystemPath(), offset, e.toString()); - mBlockWriter = null; - } catch (IllegalStateException e) { - // This can happen when there are concurrent UFS readers who are all trying to cache to block. - LOG.debug( - "Failed to update block writer for UFS block [blockId: {}, ufsPath: {}, offset: {}]." - + "Concurrent UFS readers may be caching the same block.", - mBlockMeta.getBlockId(), mBlockMeta.getUnderFileSystemPath(), offset, e); - mBlockWriter = null; - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/UnderFileSystemBlockStore.java b/core/server/worker/src/main/java/alluxio/worker/block/UnderFileSystemBlockStore.java deleted file mode 100644 index 07df037ddc23..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/UnderFileSystemBlockStore.java +++ /dev/null @@ -1,473 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static java.lang.String.format; - -import alluxio.AlluxioURI; -import alluxio.exception.BlockAlreadyExistsException; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; -import alluxio.exception.runtime.NotFoundRuntimeException; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.metrics.MetricInfo; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proto.dataserver.Protocol; -import alluxio.resource.LockResource; -import alluxio.underfs.UfsManager; -import alluxio.worker.SessionCleanable; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.meta.UnderFileSystemBlockMeta; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; -import javax.annotation.concurrent.GuardedBy; - -/** - * This class manages the virtual blocks in the UFS for delegated UFS reads/writes. - * - * The usage pattern: - * acquireAccess(sessionId, blockId, options) - * closeReaderOrWriter(sessionId, blockId) - * releaseAccess(sessionId, blockId) - * - * If the client is lost before releasing or cleaning up the session, the session cleaner will - * clean the data. - */ -public final class UnderFileSystemBlockStore implements SessionCleanable, Closeable { - private static final Logger LOG = LoggerFactory.getLogger(UnderFileSystemBlockStore.class); - - /** - * This lock protects mBlocks, mSessionIdToBlockIds. For any read/write - * operations to these maps, the lock needs to be acquired. But once you get the block - * information from the map (e.g. mBlocks), the lock does not need to be acquired. For example, - * the block reader/writer within the BlockInfo can be updated without acquiring this lock. - * This is based on the assumption that one session won't open multiple readers/writers on the - * same block. If the client do that, the client can see failures but the worker won't crash. - */ - private final ReentrantLock mLock = new ReentrantLock(); - /** Maps from the {@link Key} to the {@link BlockInfo}. */ - @GuardedBy("mLock") - private final Map mBlocks = new HashMap<>(); - /** Maps from the session ID to the block IDs. */ - @GuardedBy("mLock") - private final Map> mSessionIdToBlockIds = new HashMap<>(); - - private final ConcurrentMap mUfsBytesReadMetrics = - new ConcurrentHashMap<>(); - - private final ConcurrentMap mUfsBytesReadThroughputMetrics = - new ConcurrentHashMap<>(); - - /** The Local block store. */ - private final LocalBlockStore mLocalBlockStore; - - /** The manager for all ufs. */ - private final UfsManager mUfsManager; - - /** The manager for all ufs. */ - private final ConcurrentMap mUfsIOManager = new ConcurrentHashMap<>(); - - /** The cache for all ufs instream. */ - private final UfsInputStreamCache mUfsInstreamCache; - - /** - * Creates an instance of {@link UnderFileSystemBlockStore}. - * - * @param localBlockStore the local block store - * @param ufsManager the file manager - */ - public UnderFileSystemBlockStore(LocalBlockStore localBlockStore, UfsManager ufsManager) { - mLocalBlockStore = localBlockStore; - mUfsManager = ufsManager; - mUfsInstreamCache = new UfsInputStreamCache(); - } - - /** - * Acquires access for a UFS block given a {@link UnderFileSystemBlockMeta} and the limit on - * the maximum concurrency on the block. If the number of concurrent readers on this UFS block - * exceeds a threshold, the token is not granted and this method returns false. - * - * @param sessionId the session ID - * @param blockId maximum concurrency - * @param options the options - * @return whether an access token is acquired - * @throws BlockAlreadyExistsException if the block already exists for a session ID - */ - @VisibleForTesting - public boolean acquireAccess(long sessionId, long blockId, Protocol.OpenUfsBlockOptions options) - throws BlockAlreadyExistsException { - UnderFileSystemBlockMeta blockMeta = new UnderFileSystemBlockMeta(sessionId, blockId, options); - try (LockResource lr = new LockResource(mLock)) { - Key key = new Key(sessionId, blockId); - if (mBlocks.containsKey(key)) { - throw new BlockAlreadyExistsException(MessageFormat.format( - "UFS block {0,number,#} from UFS file {1} exists for session {2,number,#}", blockId, - blockMeta.getUnderFileSystemPath(), sessionId)); - } - mBlocks.put(key, new BlockInfo(blockMeta)); - Set blockIds = mSessionIdToBlockIds.computeIfAbsent(sessionId, k -> new HashSet<>()); - blockIds.add(blockId); - } - return true; - } - - /** - * Closes the block reader or writer and checks whether it is necessary to commit the block - * to Local block store. - * - * During UFS block read, this is triggered when the block is unlocked. - * During UFS block write, this is triggered when the UFS block is committed. - * - * @param sessionId the session ID - * @param blockId the block ID - */ - public void closeBlock(long sessionId, long blockId) throws IOException { - BlockInfo blockInfo; - try (LockResource lr = new LockResource(mLock)) { - blockInfo = mBlocks.get(new Key(sessionId, blockId)); - if (blockInfo == null) { - LOG.warn("Key (block ID: {}, session ID {}) is not found when cleaning up the UFS block.", - blockId, sessionId); - return; - } - } - blockInfo.close(); - } - - /** - * Releases the access token of this block by removing this (sessionId, blockId) pair from the - * store. - * - * @param sessionId the session ID - * @param blockId the block ID - */ - public void releaseAccess(long sessionId, long blockId) { - try (LockResource lr = new LockResource(mLock)) { - Key key = new Key(sessionId, blockId); - if (!mBlocks.containsKey(key)) { - LOG.warn("Key (block ID: {}, session ID {}) is not found when releasing the UFS block.", - blockId, sessionId); - } - mBlocks.remove(key); - Set blockIds = mSessionIdToBlockIds.get(sessionId); - if (blockIds != null) { - blockIds.remove(blockId); - if (blockIds.isEmpty()) { - mSessionIdToBlockIds.remove(sessionId); - } - } - } - } - - /** - * Cleans up all the block information(e.g. block reader/writer) that belongs to this session. - * - * @param sessionId the session ID - */ - @Override - public void cleanupSession(long sessionId) { - Set blockIds; - try (LockResource lr = new LockResource(mLock)) { - blockIds = mSessionIdToBlockIds.get(sessionId); - if (blockIds == null) { - return; - } - } - // Note that, there can be a race condition that blockIds can be stale when we release the - // access. The race condition only has a minimal negative consequence (printing extra logging - // message), and is expected very rare to trigger. - for (Long blockId : blockIds) { - try { - // Note that we don't need to explicitly call abortBlock to cleanup the temp block - // in Local block store because they will be cleanup by the session cleaner in the - // Local block store. - closeBlock(sessionId, blockId); - releaseAccess(sessionId, blockId); - } catch (Exception e) { - LOG.warn("Failed to cleanup UFS block {}, session {}.", blockId, sessionId); - } - } - } - - @Override - public void close() throws IOException { - mUfsIOManager.forEach((key, value) -> value.close()); - } - - /** - * Creates a block reader that reads from UFS and optionally caches the block to the Alluxio - * block store. - * - * @param sessionId the client session ID that requested this read - * @param blockId the ID of the block to read - * @param offset the read offset within the block (NOT the file) - * @param positionShort whether the client op is a positioned read to a small buffer - * @param options the open ufs options - * @return the block reader instance - * {@link UnderFileSystemBlockStore} - */ - public BlockReader createBlockReader(final long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException, BlockAlreadyExistsException { - if (!options.hasUfsPath() && options.getBlockInUfsTier()) { - // This is a fallback UFS block read. Reset the UFS block path according to the UfsBlock - // flag.mUnderFileSystemBlockStore - UfsManager.UfsClient ufsClient = mUfsManager.get(options.getMountId()); - options = options.toBuilder() - .setUfsPath(alluxio.worker.BlockUtils.getUfsBlockPath(ufsClient, blockId)).build(); - } - acquireAccess(sessionId, blockId, options); - final BlockInfo blockInfo; - try (LockResource lr = new LockResource(mLock)) { - blockInfo = getBlockInfo(sessionId, blockId); - BlockReader blockReader = blockInfo.getBlockReader(); - if (blockReader != null) { - return blockReader; - } - } - UfsManager.UfsClient ufsClient = mUfsManager.get(blockInfo.getMeta().getMountId()); - Counter ufsBytesRead = mUfsBytesReadMetrics.computeIfAbsent( - new BytesReadMetricKey(ufsClient.getUfsMountPointUri(), options.getUser()), - key -> key.mUser == null - ? MetricsSystem.counterWithTags( - MetricKey.WORKER_BYTES_READ_UFS.getName(), - MetricKey.WORKER_BYTES_READ_UFS.isClusterAggregated(), - MetricInfo.TAG_UFS, MetricsSystem.escape(key.mUri)) - : MetricsSystem.counterWithTags( - MetricKey.WORKER_BYTES_READ_UFS.getName(), - MetricKey.WORKER_BYTES_READ_UFS.isClusterAggregated(), - MetricInfo.TAG_UFS, MetricsSystem.escape(key.mUri), - MetricInfo.TAG_USER, key.mUser)); - Meter ufsBytesReadThroughput = mUfsBytesReadThroughputMetrics.computeIfAbsent( - ufsClient.getUfsMountPointUri(), - uri -> MetricsSystem.meterWithTags( - MetricKey.WORKER_BYTES_READ_UFS_THROUGHPUT.getName(), - MetricKey.WORKER_BYTES_READ_UFS_THROUGHPUT.isClusterAggregated(), - MetricInfo.TAG_UFS, - MetricsSystem.escape(uri))); - BlockReader reader = - UnderFileSystemBlockReader.create(blockInfo.getMeta(), offset, positionShort, - mLocalBlockStore, ufsClient, mUfsInstreamCache, ufsBytesRead, ufsBytesReadThroughput); - blockInfo.setBlockReader(reader); - return reader; - } - - /** - * Get ufsIOManager for the mount or add if absent. - * @param mountId mount identifier - * @return ufsIOManager for the mount - */ - public UfsIOManager getOrAddUfsIOManager(long mountId) { - return mUfsIOManager.computeIfAbsent(mountId, id -> { - try { - UfsIOManager manager = new UfsIOManager(mUfsManager.get(mountId)); - manager.start(); - return manager; - } catch (AlluxioStatusException e) { - throw AlluxioRuntimeException.from(e); - } - }); - } - - /** - * @param sessionId the session ID - * @param blockId the block ID - * @return true if mNoCache is set - */ - public boolean isNoCache(long sessionId, long blockId) { - final BlockInfo blockInfo; - try (LockResource lr = new LockResource(mLock)) { - blockInfo = getBlockInfo(sessionId, blockId); - } - return blockInfo.getMeta().isNoCache(); - } - - /** - * Gets the {@link UnderFileSystemBlockMeta} for a session ID and block ID pair. - * The caller must have acquired the lock before calling this method. - * - * @param sessionId the session ID - * @param blockId the block ID - * @return the {@link UnderFileSystemBlockMeta} instance - * @throws BlockDoesNotExistRuntimeException if the UFS block does not exist in the - * {@link UnderFileSystemBlockStore} - */ - @GuardedBy("mLock") - private BlockInfo getBlockInfo(long sessionId, long blockId) { - Key key = new Key(sessionId, blockId); - BlockInfo blockInfo = mBlocks.get(key); - if (blockInfo == null) { - throw new NotFoundRuntimeException(format( - "UFS block %s does not exist for session %s", blockId, sessionId)); - } - return blockInfo; - } - - /** - * This class is to wrap session ID amd block ID. - */ - private static class Key { - private final long mSessionId; - private final long mBlockId; - - /** - * Creates an instance of the Key class. - * - * @param sessionId the session ID - * @param blockId the block ID - */ - public Key(long sessionId, long blockId) { - mSessionId = sessionId; - mBlockId = blockId; - } - - /** - * @return the block ID - */ - public long getBlockId() { - return mBlockId; - } - - /** - * @return the session ID - */ - public long getSessionId() { - return mSessionId; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof Key)) { - return false; - } - - Key that = (Key) o; - return Objects.equal(mBlockId, that.mBlockId) && Objects.equal(mSessionId, that.mSessionId); - } - - @Override - public int hashCode() { - return Objects.hashCode(mBlockId, mSessionId); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("blockId", mBlockId).add("sessionId", mSessionId) - .toString(); - } - } - - private static class BytesReadMetricKey { - private final AlluxioURI mUri; - private final String mUser; - - BytesReadMetricKey(AlluxioURI uri, String user) { - mUri = uri; - mUser = user; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - BytesReadMetricKey that = (BytesReadMetricKey) o; - return mUri.equals(that.mUri) && mUser.equals(that.mUser); - } - - @Override - public int hashCode() { - return Objects.hashCode(mUri, mUser); - } - } - - /** - * This class is to wrap block reader/writer and the block meta into one class. The block - * reader/writer is not part of the {@link UnderFileSystemBlockMeta} because - * 1. UnderFileSystemBlockMeta only keeps immutable information. - * 2. We do not want a cyclic dependency between {@link UnderFileSystemBlockReader} and - * {@link UnderFileSystemBlockMeta}. - */ - private static class BlockInfo { - private final UnderFileSystemBlockMeta mMeta; - - private BlockReader mBlockReader; - - /** - * Creates an instance of {@link BlockInfo}. - * - * @param meta the UFS block meta - */ - public BlockInfo(UnderFileSystemBlockMeta meta) { - mMeta = meta; - } - - /** - * @return the UFS block meta - */ - public UnderFileSystemBlockMeta getMeta() { - return mMeta; - } - - /** - * @return the cached the block reader if it is not closed - */ - public synchronized BlockReader getBlockReader() { - if (mBlockReader != null && mBlockReader.isClosed()) { - mBlockReader = null; - } - return mBlockReader; - } - - /** - * @param blockReader the block reader to be set - */ - public synchronized void setBlockReader(BlockReader blockReader) { - mBlockReader = blockReader; - } - - /** - * Closes the block reader or writer. - */ - public synchronized void close() throws IOException { - if (mBlockReader != null) { - mBlockReader.close(); - mBlockReader = null; - } - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/allocator/Allocator.java b/core/server/worker/src/main/java/alluxio/worker/block/allocator/Allocator.java deleted file mode 100644 index 9af9440c0149..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/allocator/Allocator.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import alluxio.annotation.PublicApi; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.worker.block.BlockMetadataView; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.reviewer.Reviewer; - -import com.google.common.base.Preconditions; - -/** - * Interface for the allocation policy of Alluxio managed data. - */ -@PublicApi -public interface Allocator { - - /** - * Factory for {@link Allocator}. - */ - class Factory { - - private Factory() {} // prevent instantiation - - /** - * Factory for {@link Allocator}. - * - * @param view {@link BlockMetadataView} to pass to {@link Allocator} - * @return the generated {@link Allocator}, it will be a {@link MaxFreeAllocator} by default - */ - public static Allocator create(BlockMetadataView view) { - BlockMetadataView metadataView = Preconditions.checkNotNull(view, "view"); - return CommonUtils.createNewClassInstance( - Configuration.getClass(PropertyKey.WORKER_ALLOCATOR_CLASS), - new Class[] {BlockMetadataView.class}, new Object[] {metadataView}); - } - } - - /** - * Allocates a block from the given block store location under a given view. The location can be a - * specific location, or {@link BlockStoreLocation#anyTier()} or - * {@link BlockStoreLocation#anyDirInTier(String)}. - * - * TODO(jiacheng): Refactor Allocator interface to decouple Reviewer logic from Allocator. - * - * The proposed allocation will be reviewed by a {@link Reviewer}. - * The reviewer will check certain aspects of the allocation and may reject the allocation if - * it does not meet certain criteria. - * The parameter skipReview specifies whether the review should be skipped. - * The review should be skipped when we want the behavior of the allocator to be deterministic - * and do not want it to be affected by the reviewer. - * E.g. We just freed up some space in Alluxio and want the allocation to use the freed space. - * - * @param blockSize the size of block in bytes - * @param location the location in block store - * @param view of the block metadata - * @param skipReview whether the review should be skipped - * @return a {@link StorageDirView} in which to create the temp block meta if success, null - * otherwise - */ - StorageDirView allocateBlockWithView(long blockSize, BlockStoreLocation location, - BlockMetadataView view, boolean skipReview); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/allocator/GreedyAllocator.java b/core/server/worker/src/main/java/alluxio/worker/block/allocator/GreedyAllocator.java deleted file mode 100644 index f3ca53c2d070..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/allocator/GreedyAllocator.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import alluxio.worker.block.BlockMetadataView; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTierView; -import alluxio.worker.block.reviewer.Reviewer; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A greedy allocator that returns the first Storage dir fitting the size of block to allocate. This - * class serves as an example how to implement an allocator. - */ -@NotThreadSafe -public final class GreedyAllocator implements Allocator { - private static final Logger LOG = LoggerFactory.getLogger(GreedyAllocator.class); - - private BlockMetadataView mMetadataView; - private final Reviewer mReviewer; - - /** - * Creates a new instance of {@link GreedyAllocator}. - * - * @param view {@link BlockMetadataView} to pass to the allocator - */ - public GreedyAllocator(BlockMetadataView view) { - mMetadataView = Preconditions.checkNotNull(view, "view"); - mReviewer = Reviewer.Factory.create(); - } - - @Override - public StorageDirView allocateBlockWithView(long blockSize, - BlockStoreLocation location, BlockMetadataView metadataView, boolean skipReview) { - mMetadataView = Preconditions.checkNotNull(metadataView, "view"); - return allocateBlock(blockSize, location, skipReview); - } - - /** - * Allocates a block from the given block store location. The location can be a specific location, - * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}. - * - * @param blockSize the size of block in bytes - * @param location the location in block store - * @return a {@link StorageDirView} in which to create the temp block meta if success, - * null otherwise - */ - @Nullable - private StorageDirView allocateBlock(long blockSize, - BlockStoreLocation location, boolean skipReview) { - Preconditions.checkNotNull(location, "location"); - if (location.isAnyTier() && location.isAnyDir()) { - // When any tier is ok, loop over all tier views and dir views, - // and return a temp block meta from the first available dirview. - for (StorageTierView tierView : mMetadataView.getTierViews()) { - for (StorageDirView dirView : tierView.getDirViews()) { - if ((location.isAnyMedium() - || dirView.getMediumType().equals(location.mediumType())) - && dirView.getAvailableBytes() >= blockSize) { - if (skipReview || mReviewer.acceptAllocation(dirView)) { - return dirView; - } else { - // The allocation is rejected. Try the next dir. - LOG.debug("Allocation rejected for anyTier: {}", dirView.toBlockStoreLocation()); - } - } - } - } - return null; - } - - String tierAlias = location.tierAlias(); - StorageTierView tierView = mMetadataView.getTierView(tierAlias); - if (location.isAnyDirWithTier()) { - // Loop over all dir views in the given tier - for (StorageDirView dirView : tierView.getDirViews()) { - if (dirView.getAvailableBytes() >= blockSize) { - if (skipReview || mReviewer.acceptAllocation(dirView)) { - return dirView; - } else { - // Try the next dir - LOG.debug("Allocation rejected for anyDirInTier: {}", - dirView.toBlockStoreLocation()); - } - } - } - return null; - } - - // For allocation in a specific directory, we are not checking the reviewer, - // because we do not want the reviewer to reject it. - int dirIndex = location.dir(); - StorageDirView dirView = tierView.getDirView(dirIndex); - if (dirView != null && dirView.getAvailableBytes() >= blockSize) { - return dirView; - } - return null; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/allocator/MaxFreeAllocator.java b/core/server/worker/src/main/java/alluxio/worker/block/allocator/MaxFreeAllocator.java deleted file mode 100644 index ac8d29ac22d7..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/allocator/MaxFreeAllocator.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import alluxio.worker.block.BlockMetadataView; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTierView; -import alluxio.worker.block.reviewer.Reviewer; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.concurrent.NotThreadSafe; - -/** - * An allocator that allocates a block in the storage dir with most free space. It always allocates - * to the highest tier if the requested block store location is any tier. - */ -@NotThreadSafe -public final class MaxFreeAllocator implements Allocator { - private static final Logger LOG = LoggerFactory.getLogger(MaxFreeAllocator.class); - - private BlockMetadataView mMetadataView; - private final Reviewer mReviewer; - - /** - * Creates a new instance of {@link MaxFreeAllocator}. - * - * @param view {@link BlockMetadataView} to pass to the allocator - */ - public MaxFreeAllocator(BlockMetadataView view) { - mMetadataView = Preconditions.checkNotNull(view, "view"); - mReviewer = Reviewer.Factory.create(); - } - - @Override - public StorageDirView allocateBlockWithView(long blockSize, - BlockStoreLocation location, BlockMetadataView metadataView, boolean skipReview) { - mMetadataView = Preconditions.checkNotNull(metadataView, "view"); - return allocateBlock(blockSize, location, skipReview); - } - - /** - * Allocates a block from the given block store location. The location can be a specific location, - * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}. - * - * @param blockSize the size of block in bytes - * @param location the location in block store - * @param skipReview whether the review should be skipped - * @return a {@link StorageDirView} in which to create the temp block meta if success, - * null otherwise - */ - private StorageDirView allocateBlock(long blockSize, - BlockStoreLocation location, boolean skipReview) { - Preconditions.checkNotNull(location, "location"); - StorageDirView candidateDirView = null; - - if (location.isAnyTier() && location.isAnyDir()) { - for (StorageTierView tierView : mMetadataView.getTierViews()) { - candidateDirView = getCandidateDirInTier(tierView, blockSize, location); - if (candidateDirView != null) { - if (skipReview || mReviewer.acceptAllocation(candidateDirView)) { - break; - } - // We tried the dir on this tier with max free bytes but that is not good enough. - // So we move on to the lower tier. - LOG.debug("Allocation rejected for anyTier: {}", - candidateDirView.toBlockStoreLocation()); - candidateDirView = null; - } - } - } else if (location.isAnyDirWithTier()) { - StorageTierView tierView = mMetadataView.getTierView(location.tierAlias()); - candidateDirView = getCandidateDirInTier(tierView, blockSize, location); - if (candidateDirView != null) { - // The allocation is not good enough. Revert it. - if (!skipReview && !mReviewer.acceptAllocation(candidateDirView)) { - LOG.debug("Allocation rejected for anyDirInTier: {}", - candidateDirView.toBlockStoreLocation()); - candidateDirView = null; - } - } - } else { - // For allocation in a specific directory, we are not checking the reviewer, - // because we do not want the reviewer to reject it. - StorageTierView tierView = mMetadataView.getTierView(location.tierAlias()); - StorageDirView dirView = tierView.getDirView(location.dir()); - if (dirView != null && dirView.getAvailableBytes() >= blockSize) { - candidateDirView = dirView; - } - } - - return candidateDirView; - } - - /** - * Finds a directory view in a tier view that has max free space and is able to store the block. - * - * @param tierView the storage tier view - * @param blockSize the size of block in bytes - * @param location the block store location - * @return the storage directory view if found, null otherwise - */ - private StorageDirView getCandidateDirInTier(StorageTierView tierView, - long blockSize, BlockStoreLocation location) { - StorageDirView candidateDirView = null; - long maxFreeBytes = blockSize - 1; - for (StorageDirView dirView : tierView.getDirViews()) { - if ((location.isAnyMedium() - || dirView.getMediumType().equals(location.mediumType())) - && dirView.getAvailableBytes() > maxFreeBytes) { - maxFreeBytes = dirView.getAvailableBytes(); - candidateDirView = dirView; - } - } - return candidateDirView; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/allocator/RoundRobinAllocator.java b/core/server/worker/src/main/java/alluxio/worker/block/allocator/RoundRobinAllocator.java deleted file mode 100644 index add4438f03f4..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/allocator/RoundRobinAllocator.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import alluxio.worker.block.BlockMetadataView; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTierView; -import alluxio.worker.block.reviewer.Reviewer; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A round-robin allocator that allocates a block in the storage dir. It will allocate the block in - * the highest tier possible: It always starts from the highest tier in a RR manner and goes to the - * next tier when there is not enough space. The allocator only considers non-specific writes in its - * RR policy (the location is either AnyTier or AnyDirInTier). - */ -@NotThreadSafe -public final class RoundRobinAllocator implements Allocator { - private static final Logger LOG = LoggerFactory.getLogger(RoundRobinAllocator.class); - - private BlockMetadataView mMetadataView; - private final Reviewer mReviewer; - - // We need to remember the last dir index for every storage tier - private final Map> mTierAliasToDirIteratorMap = new HashMap<>(); - - /** - * Creates a new instance of {@link RoundRobinAllocator}. - * - * @param view {@link BlockMetadataView} to pass to the allocator - */ - public RoundRobinAllocator(BlockMetadataView view) { - mMetadataView = Preconditions.checkNotNull(view, "view"); - for (StorageTierView tierView : mMetadataView.getTierViews()) { - mTierAliasToDirIteratorMap.put( - tierView.getTierViewAlias(), tierView.getDirViews().iterator()); - } - mReviewer = Reviewer.Factory.create(); - } - - @Override - public StorageDirView allocateBlockWithView(long blockSize, - BlockStoreLocation location, BlockMetadataView metadataView, boolean skipReview) { - mMetadataView = Preconditions.checkNotNull(metadataView, "view"); - return allocateBlock(blockSize, location, skipReview); - } - - /** - * Allocates a block from the given block store location. The location can be a specific location, - * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}. - * - * @param blockSize the size of block in bytes - * @param location the location in block store - * @return a {@link StorageDirView} in which to create the temp block meta if success, - * null otherwise - * @throws IllegalArgumentException if block location is invalid - */ - @Nullable - private StorageDirView allocateBlock(long blockSize, - BlockStoreLocation location, boolean skipReview) { - Preconditions.checkNotNull(location, "location"); - if (location.isAnyDirWithTier()) { - StorageTierView tierView = mMetadataView.getTierView(location.tierAlias()); - // The review logic is handled in getNextAvailDirInTier - return getNextAvailDirInTier(tierView, blockSize, location, skipReview); - } else if (location.isAnyTier() && location.isAnyDir()) { - for (StorageTierView tierView : mMetadataView.getTierViews()) { - // The review logic is handled in getNextAvailDirInTier - StorageDirView dir = getNextAvailDirInTier(tierView, blockSize, location, skipReview); - if (dir != null) { - return dir; - } - } - } else { - // For allocation in a specific directory, we are not checking the reviewer, - // because we do not want the reviewer to reject it. - StorageTierView tierView = mMetadataView.getTierView(location.tierAlias()); - StorageDirView dirView = tierView.getDirView(location.dir()); - if (dirView != null && dirView.getAvailableBytes() >= blockSize) { - return dirView; - } - } - - return null; - } - - /** - * Finds an available dir in a given tier for a block with blockSize. - * - * @param tierView the tier to find a dir - * @param blockSize the requested block size - * @param location the block store location - * @return the index of the dir if non-negative; -1 if fail to find a dir - */ - private StorageDirView getNextAvailDirInTier(StorageTierView tierView, long blockSize, - BlockStoreLocation location, boolean skipReview) { - Iterator iterator = mTierAliasToDirIteratorMap.get(tierView.getTierViewAlias()); - int processed = 0; - while (processed < tierView.getDirViews().size()) { - if (!iterator.hasNext()) { - iterator = tierView.getDirViews().iterator(); - mTierAliasToDirIteratorMap.put(tierView.getTierViewAlias(), iterator); - } - StorageDirView dir = iterator.next(); - if ((location.isAnyMedium() - || dir.getMediumType().equals(location.mediumType())) - && dir.getAvailableBytes() >= blockSize) { - if (skipReview || mReviewer.acceptAllocation(dir)) { - return dir; - } - // The allocation is rejected. Try the next dir. - LOG.debug("Allocation to dir {} rejected: {}", dir, - dir.toBlockStoreLocation()); - } - processed++; - } - return null; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockAnnotator.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockAnnotator.java deleted file mode 100644 index 710cdc17505d..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockAnnotator.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.annotation.PublicApi; -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; - -import java.util.List; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Interface for providers that annotates blocks for sorting. - * - * @param sorted-field type for the annotator - */ -@PublicApi -public interface BlockAnnotator { - - /** - * Factory for {@link BlockAnnotator}. - */ - @ThreadSafe - class Factory { - private Factory() {} // prevent instantiation - - /** - * Creates {@link BlockAnnotator} implementation based - * on Alluxio configuration. - * - * @return the generated {@link BlockAnnotator} instance - */ - public static BlockAnnotator create() { - return CommonUtils.createNewClassInstance( - Configuration.getClass(PropertyKey.WORKER_BLOCK_ANNOTATOR_CLASS), null, - null); - } - } - - /** - * Used to get a new sorted-field for the given block - * at the current logical time. - * - * @param blockId block Id - * @param oldValue old sorted-field value - * @return the new sorted-field value - */ - BlockSortedField updateSortedField(long blockId, T oldValue); - - /** - * Updates sorted-field values for all {block-id, sorted-field} pairs - * at the same logical time. - * Note: Currently not required for online schemes, so not called. - * - * @param blockList list of {block-id, sorted-field} pairs - */ - void updateSortedFields(List> blockList); - - /** - * Used to report whether the block annotator is an online sorter. - * - * For offline sorters, {@link #updateSortedFields(List)} will be called before - * acquiring an iterator for a particular location. - * - * @return {@code true} if an online sorter - */ - boolean isOnlineSorter(); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockIterator.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockIterator.java deleted file mode 100644 index 8b4ff68691cb..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockIterator.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.collections.Pair; -import alluxio.worker.block.BlockStoreEventListener; -import alluxio.worker.block.BlockStoreLocation; - -import java.util.Iterator; -import java.util.List; -import java.util.function.Function; - -/** - * An interface to provide various block iteration utilities for eviction - * and various management tasks. - */ -public interface BlockIterator { - /** - * Gets an iterator of block-Ids. - * - * @param location location to iterate for blocks - * @param order order of blocks - * @return an iterator - */ - Iterator getIterator(BlockStoreLocation location, BlockOrder order); - - /** - * Used to get blocks within an intersection of locations in sorted order. - * - * It receives a filter using which certain blocks could be excluded from intersection iterator. - * - * @param srcLocation source location - * @param srcOrder order for the source location - * @param dstLocation destination location - * @param dstOrder order for the destination location - * @param intersectionWidth width of intersection - * @param intersectionOrder the order of intersected blocks - * @param blockFilterFunc a filter for blocks - * @return list of blocks in intersection - */ - List getIntersectionList(BlockStoreLocation srcLocation, BlockOrder srcOrder, - BlockStoreLocation dstLocation, BlockOrder dstOrder, int intersectionWidth, - BlockOrder intersectionOrder, Function blockFilterFunc); - - /** - * Used to list of blocks, that if swapped, could eliminate overlap between two tiers. - * - * It receives a filter using which certain blocks could be excluded from intersection iterator. - * - * @param srcLocation source location - * @param srcOrder order for the source location - * @param dstLocation destination location - * @param dstOrder order for the destination location - * @param swapRange per-tier range of swap - * @param intersectionOrder the order of intersected blocks - * @param blockFilterFunc a filter for blocks - * @return 2 list of blocks to swap - */ - Pair, List> getSwaps(BlockStoreLocation srcLocation, BlockOrder srcOrder, - BlockStoreLocation dstLocation, BlockOrder dstOrder, int swapRange, - BlockOrder intersectionOrder, Function blockFilterFunc); - - /** - * Used to detect presence of an overlap between two locations. - * Overlap will be decided based on configured eviction order policy. - * - * It receives a filter using which certain blocks could be excluded from overlap decision. - * - * @param srcLocation source location - * @param dstLocation destination location - * @param order order of comparison between locations - * @param blockFilterFunc a filter for blocks - * @return {@code true} if tiers are aligned - */ - boolean aligned(BlockStoreLocation srcLocation, BlockStoreLocation dstLocation, BlockOrder order, - Function blockFilterFunc); - - /** - * Used to acquire and register listeners that are used by this iterator. - * - * @return list of event listeners - */ - List getListeners(); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockOrder.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockOrder.java deleted file mode 100644 index 6bdd1cdf0202..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockOrder.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import java.util.Comparator; - -/** - * Enumeration class to specify natural & reverse orders - * along with some utilities. - */ -public enum BlockOrder { - NATURAL, REVERSE; - - /** - * @return the opposite of the order - */ - public BlockOrder reversed() { - switch (this) { - case NATURAL: - return BlockOrder.REVERSE; - case REVERSE: - return BlockOrder.NATURAL; - default: - throw new IllegalArgumentException(String.format("Undefined block order:%s", this)); - } - } - - /** - * @return the comparator for the order - */ - public Comparator comparator() { - switch (this) { - case NATURAL: - return Comparator.naturalOrder(); - case REVERSE: - return Comparator.reverseOrder(); - default: - throw new IllegalArgumentException(String.format("Undefined block order:%s", this)); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockSortedField.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockSortedField.java deleted file mode 100644 index 20525747b990..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/BlockSortedField.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.annotation.PublicApi; - -/** - * Used by {@link BlockAnnotator} implementations to embed sorted-fields per blocks. - */ -@PublicApi -public interface BlockSortedField extends Comparable { -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/DefaultBlockIterator.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/DefaultBlockIterator.java deleted file mode 100644 index 00f521fbc660..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/DefaultBlockIterator.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import static java.util.Objects.requireNonNull; - -import alluxio.collections.ConcurrentHashSet; -import alluxio.collections.Pair; -import alluxio.worker.block.AbstractBlockStoreEventListener; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreEventListener; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageTier; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterators; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * The default {@link BlockIterator} implementation that integrates with the - * {@link BlockMetadataManager}. - */ -public class DefaultBlockIterator implements BlockIterator { - private static final Logger LOG = LoggerFactory.getLogger(DefaultBlockIterator.class); - - /** Map of sorted block set collections per directory. */ - private final Map> mPerDirOrderedSets - = new ConcurrentHashMap<>(); - - /** Used to update total order for offline sorting schemes. */ - private final Set mUnorderedLocations = new ConcurrentHashSet<>(); - - /** Configured block annotator class. */ - private final BlockAnnotator mBlockAnnotator; - - /** Underlying meta manager. */ - private final BlockMetadataManager mMetaManager; - - /** Block store delegate listener. */ - private final BlockStoreEventListener mListener = new Listener(); - - /** - * Creates default block iterator instance. - * - * @param metaManager meta manager - * @param blockAnnotator block annotator - */ - public DefaultBlockIterator(BlockMetadataManager metaManager, BlockAnnotator blockAnnotator) { - mMetaManager = requireNonNull(metaManager, "metaManager is null"); - mBlockAnnotator = requireNonNull(blockAnnotator, "blockAnnotator is null"); - - initialize(); - } - - /** - * Initializes with the existing blocks. - */ - private void initialize() { - // Initialize sets per location. - for (StorageTier tier : mMetaManager.getTiers()) { - for (StorageDir dir : tier.getStorageDirs()) { - mPerDirOrderedSets.put(dir.toBlockStoreLocation(), new SortedBlockSet()); - mUnorderedLocations.add(dir.toBlockStoreLocation()); - } - } - - // Initialize with existing items. - for (StorageTier tier : mMetaManager.getTiers()) { - for (StorageDir dir : tier.getStorageDirs()) { - BlockStoreLocation dirLocation = dir.toBlockStoreLocation(); - for (long blockId : dir.getBlockIds()) { - blockUpdated(blockId, dirLocation); - } - } - } - } - - /** - * Called by block-store event callbacks to update a block. - */ - private void blockUpdated(long blockId, BlockStoreLocation location) { - // Acquire the sorted-set for the target location. - SortedBlockSet sortedSet = mPerDirOrderedSets.get(location); - // Get new sort-field for the block. - BlockSortedField sortedField = - mBlockAnnotator.updateSortedField(blockId, sortedSet.getSortField(blockId)); - // Update the sorted-set. - sortedSet.put(blockId, sortedField); - - // Mark the location for offline sort providers. - if (!mBlockAnnotator.isOnlineSorter()) { - mUnorderedLocations.add(location); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Block:{} updated at {} with {} blocks.", blockId, location, sortedSet.size()); - } - } - - /** - * Called by block-store event callbacks to remove a block. - */ - private void blockRemoved(long blockId, BlockStoreLocation location) { - // Acquire the sorted-set for the target location. - SortedBlockSet sortedSet = mPerDirOrderedSets.get(location); - // Remove from the sorted-set. - sortedSet.remove(blockId); - - if (LOG.isDebugEnabled()) { - LOG.debug("Block:{} removed from {} with {} blocks.", blockId, location, sortedSet.size()); - } - } - - /** - * Called by block-store event callbacks to move a block. - */ - private void blockMoved(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) { - // TODO(ggezer): Fix callback logic to not called for the same locations. - if (!oldLocation.equals(newLocation)) { - // Acquire the sorted-set for the block's current location. - SortedBlockSet oldSortedSet = mPerDirOrderedSets.get(oldLocation); - // Extract the old sorted-field and remove from source location's sorted-set. - BlockSortedField oldSortField = oldSortedSet.getSortField(blockId); - oldSortedSet.remove(blockId); - // Update the destination sorted-set with the old sorted-field. - SortedBlockSet newSortedSet = mPerDirOrderedSets.get(newLocation); - newSortedSet.put(blockId, oldSortField); - - // Mark the location for offline annotators. - if (!mBlockAnnotator.isOnlineSorter()) { - mUnorderedLocations.add(newLocation); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Block: {} moved from {} with {} blocks to {} with {} blocks.", - blockId, oldLocation, oldSortedSet.size(), newLocation, newSortedSet.size()); - } - } - } - - @Override - public List getListeners() { - return ImmutableList.of(mListener); - } - - @Override - public Iterator getIterator(BlockStoreLocation location, BlockOrder order) { - return Iterators.transform(getIteratorInternal(location, order), (pair) -> pair.getFirst()); - } - - @Override - public List getIntersectionList(BlockStoreLocation srcLocation, BlockOrder srcOrder, - BlockStoreLocation dstLocation, BlockOrder dstOrder, int intersectionWidth, - BlockOrder intersectionOrder, Function blockFilterFunc) { - - // Acquire source iterator based on given order. - Iterator> srcIterator = getIteratorInternal(srcLocation, srcOrder); - // Acquire destination iterator based on given order. - Iterator> dstIterator = getIteratorInternal(dstLocation, dstOrder); - - // Build the list of entries with the entries that belongs to intersection. - List> intersectionList = new ArrayList<>(intersectionWidth); - while (srcIterator.hasNext() || dstIterator.hasNext()) { - if (intersectionList.size() >= intersectionWidth) { - break; - } - while (srcIterator.hasNext()) { - Pair currPair = srcIterator.next(); - if (!blockFilterFunc.apply(currPair.getFirst())) { - intersectionList.add(currPair); - break; - } - } - while (dstIterator.hasNext()) { - Pair currPair = dstIterator.next(); - if (!blockFilterFunc.apply(currPair.getFirst())) { - intersectionList.add(currPair); - break; - } - } - } - - // Sort the intersection list. - intersectionList.sort((o1, o2) -> intersectionOrder.comparator().compare( - o1.getSecond(), o2.getSecond())); - // Return iterator off of sorted intersection list. - return intersectionList.stream().map((kv) -> kv.getFirst()).collect(Collectors.toList()); - } - - @Override - public Pair, List> getSwaps(BlockStoreLocation srcLocation, BlockOrder srcOrder, - BlockStoreLocation dstLocation, BlockOrder dstOrder, int swapRange, - BlockOrder intersectionOrder, Function blockFilterFunc) { - // Acquire source iterator based on given order. - Iterator> srcIterator = getIteratorInternal(srcLocation, srcOrder); - // Acquire destination iterator based on given order. - Iterator> dstIterator = getIteratorInternal(dstLocation, dstOrder); - - // These lists are used to emulate block swapping. - List> srcList = new ArrayList<>(swapRange); - List> dstList = new ArrayList<>(swapRange); - - while (srcIterator.hasNext() && srcList.size() < swapRange) { - Pair currPair = srcIterator.next(); - if (!blockFilterFunc.apply(currPair.getFirst())) { - srcList.add(currPair); - } - } - while (dstIterator.hasNext() && dstList.size() < swapRange) { - Pair currPair = dstIterator.next(); - if (!blockFilterFunc.apply(currPair.getFirst())) { - dstList.add(currPair); - } - } - - // Simulate swapping until both ends of the list are aligned. - int swapLimit = Math.min(srcList.size(), dstList.size()); - int swapCount = 0; - while (swapCount < swapLimit) { - Pair srcItem = srcList.get(swapCount); - Pair dstItem = dstList.get(swapCount); - - if (intersectionOrder.comparator().compare(srcItem.getSecond(), dstItem.getSecond()) <= 0) { - break; - } - - swapCount++; - } - - return new Pair<>( - srcList.subList(0, swapCount).stream().map((kv) -> kv.getFirst()) - .collect(Collectors.toList()), - dstList.subList(0, swapCount).stream().map((kv) -> kv.getFirst()) - .collect(Collectors.toList())); - } - - @Override - public boolean aligned(BlockStoreLocation srcLocation, BlockStoreLocation dstLocation, - BlockOrder order, Function blockFilterFunc) { - // Get source iterator with given source order. - Iterator> srcIterator = - getIteratorInternal(srcLocation, order); - // Get destination iterator with the reverse of given source order. - Iterator> dstIterator = - getIteratorInternal(dstLocation, order.reversed()); - - /** - * Gets the first valid block entry from both locations based on given order. - * Comparison of these entries will reveal if there is an overlap between locations. - */ - Pair srcItem = null; - while (srcIterator.hasNext()) { - Pair currPair = srcIterator.next(); - if (!blockFilterFunc.apply(currPair.getFirst())) { - srcItem = currPair; - break; - } - } - Pair dstItem = null; - while (dstIterator.hasNext()) { - Pair currPair = dstIterator.next(); - if (!blockFilterFunc.apply(currPair.getFirst())) { - dstItem = currPair; - break; - } - } - - return srcItem == null || dstItem == null - || order.comparator().compare(srcItem.getSecond(), dstItem.getSecond()) >= 0; - } - - /** - * Internal utility to get sorted block iterator for a given location and order. - */ - private Iterator> getIteratorInternal(BlockStoreLocation location, - BlockOrder order) { - // Gather each directory location that is under requested location. - List locations = mPerDirOrderedSets.keySet().stream() - .filter((dirLocation) -> dirLocation.belongsTo(location)).collect(Collectors.toList()); - - // For offline order providers, update total order for each dirty location. - if (!mBlockAnnotator.isOnlineSorter()) { - if (mUnorderedLocations.stream() - .anyMatch((dirtyLocation) -> dirtyLocation.belongsTo(location))) { - LOG.debug("Updating total order for directories that belong to {}", location); - updateTotalOrder(locations); - } - } - - // Gather iterators per each directory based on given order. - List>> iteratorList = - new ArrayList<>(mPerDirOrderedSets.size()); - for (BlockStoreLocation dirLocation : locations) { - switch (order) { - case NATURAL: - iteratorList.add(mPerDirOrderedSets.get(dirLocation).getAscendingIterator()); - break; - case REVERSE: - iteratorList.add(mPerDirOrderedSets.get(dirLocation).getDescendingIterator()); - break; - default: - throw new IllegalArgumentException( - String.format("Unsupported sort order: %s", order.name())); - } - } - - // Return a merge-sorted iterator for gathered iterators. - return Iterators.mergeSorted(iteratorList, - Comparator.comparing(Pair::getSecond, order.comparator())); - } - - /** - * Offline order providers require full sweep of its range in order to provide - * total order among all elements. (For instance, LRFU..) - * - * This will invoke order provider with the full list in order to satisfy this requirement. - * - * TODO(ggezer): Consider adding a new {@link BlockAnnotator} API to extract logical time. - */ - private synchronized void updateTotalOrder(List locations) { - // No need if there is no unordered locations. - if (mUnorderedLocations.isEmpty()) { - return; - } - - // Used to store and update old sorted fields for each location. - List> updatedEntries = new ArrayList<>(); - - for (BlockStoreLocation location : locations) { - // Acquire sorted-block-set for the location. - SortedBlockSet sortedSet = mPerDirOrderedSets.get(location); - // Populate list for existing blocks on location. - updatedEntries.clear(); - Iterator> locationIter = sortedSet.getAscendingIterator(); - while (locationIter.hasNext()) { - updatedEntries.add(locationIter.next()); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Updating total order for {} with {} blocks.", - location, updatedEntries.size()); - } - - // Invoke order provider to update fields all together. - mBlockAnnotator.updateSortedFields(updatedEntries); - - // Reinsert updated values back to sorted-set. - for (Pair updatedEntry : updatedEntries) { - sortedSet.put(updatedEntry.getFirst(), updatedEntry.getSecond()); - } - - // Location is ordered now. - mUnorderedLocations.remove(location); - } - } - - /** - * Internal class used to forward block store events to this iterator implementation. - */ - class Listener extends AbstractBlockStoreEventListener { - @Override - public void onAccessBlock(long blockId, BlockStoreLocation location) { - blockUpdated(blockId, location); - } - - @Override - public void onCommitBlock(long blockId, BlockStoreLocation location) { - blockUpdated(blockId, location); - } - - @Override - public void onRemoveBlock(long blockId, BlockStoreLocation location) { - blockRemoved(blockId, location); - } - - @Override - public void onStorageLost(BlockStoreLocation dirLocation) { - mPerDirOrderedSets.remove(dirLocation); - } - - @Override - public void onMoveBlockByClient(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) { - blockMoved(blockId, oldLocation, newLocation); - } - - @Override - public void onMoveBlockByWorker(long blockId, BlockStoreLocation oldLocation, - BlockStoreLocation newLocation) { - blockMoved(blockId, oldLocation, newLocation); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/EmulatingBlockIterator.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/EmulatingBlockIterator.java deleted file mode 100644 index f1f5e7995ba1..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/EmulatingBlockIterator.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import static alluxio.worker.block.BlockMetadataManager.WORKER_STORAGE_TIER_ASSOC; - -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreEventListener; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.evictor.EvictionPlan; -import alluxio.worker.block.evictor.Evictor; -import alluxio.worker.block.meta.StorageTier; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * This is used to support deprecated {@link Evictor} implementations. - * It works by assuming an order in the eviction plan generated by the evictor. - */ -public class EmulatingBlockIterator implements BlockIterator { - /** Block metadata manager. */ - private final BlockMetadataManager mMetadataManager; - /** Evictor that is to be emulated. */ - private final Evictor mEvictor; - /** Mapping from tier alias to space in bytes to be reserved on the tier. */ - private final Map mReservedSpaces = new HashMap<>(); - - /** - * Creates a block iterator that emulates an {@link Evictor}. - * - * @param metadataManager the metadata manager - * @param evictor the evictor - */ - public EmulatingBlockIterator(BlockMetadataManager metadataManager, Evictor evictor) { - mMetadataManager = metadataManager; - mEvictor = evictor; - - initEvictorConfiguration(); - } - - /** - * Used to calculate configuration that evictors need to run. - */ - private void initEvictorConfiguration() { - // Calculate tier capacities. - Map tierCapacities = new HashMap<>(); - for (StorageTier tier : mMetadataManager.getTiers()) { - tierCapacities.put(tier.getTierAlias(), tier.getCapacityBytes()); - } - - long lastTierReservedBytes = 0; - for (int ordinal = 0; ordinal < WORKER_STORAGE_TIER_ASSOC.size(); ordinal++) { - String tierAlias = WORKER_STORAGE_TIER_ASSOC.getAlias(ordinal); - long tierCapacity = tierCapacities.get(tierAlias); - // High watermark defines when to start the space reserving process. - // It's only validated in this emulator, it doesn't trigger any background task. - PropertyKey tierHighWatermarkProp = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO.format(ordinal); - double tierHighWatermarkConf = Configuration.getDouble(tierHighWatermarkProp); - Preconditions.checkArgument(tierHighWatermarkConf > 0, - "The high watermark of tier %s should be positive, but is %s", Integer.toString(ordinal), - tierHighWatermarkConf); - Preconditions.checkArgument(tierHighWatermarkConf < 1, - "The high watermark of tier %s should be less than 1.0, but is %s", - Integer.toString(ordinal), tierHighWatermarkConf); - - // Low watermark defines when to stop the space reserving process if started - PropertyKey tierLowWatermarkProp = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_LOW_WATERMARK_RATIO.format(ordinal); - double tierLowWatermarkConf = Configuration.getDouble(tierLowWatermarkProp); - Preconditions.checkArgument(tierLowWatermarkConf >= 0, - "The low watermark of tier %s should not be negative, but is %s", - Integer.toString(ordinal), tierLowWatermarkConf); - Preconditions.checkArgument(tierLowWatermarkConf < tierHighWatermarkConf, - "The low watermark (%s) of tier %d should not be smaller than the high watermark (%s)", - tierLowWatermarkConf, ordinal, tierHighWatermarkConf); - long reservedSpace = (long) (tierCapacity - tierCapacity * tierLowWatermarkConf); - lastTierReservedBytes += reservedSpace; - // On each tier, we reserve no more than its capacity - lastTierReservedBytes = - (lastTierReservedBytes <= tierCapacity) ? lastTierReservedBytes : tierCapacity; - mReservedSpaces.put(tierAlias, lastTierReservedBytes); - // Update special ANY_TIER to have total reserved capacity. - if (mReservedSpaces.containsKey(BlockStoreLocation.ANY_TIER)) { - mReservedSpaces.put(BlockStoreLocation.ANY_TIER, - mReservedSpaces.get(BlockStoreLocation.ANY_TIER) + lastTierReservedBytes); - } else { - mReservedSpaces.put(BlockStoreLocation.ANY_TIER, lastTierReservedBytes); - } - } - } - - @Override - public Iterator getIterator(BlockStoreLocation location, BlockOrder order) { - /** - * Invoke the evictor for the location with configured free space threshold. - * From the generated plan, extract the order imposed by cascading eviction logic. - * - * This emulation assumes that toEvict() and toMove() lists of the plan - * is populated based on an internal order maintained by the evictor implementation. - */ - - // Generate the plan as every block being evictable. - EvictionPlan evictionPlan = - mEvictor.freeSpaceWithView(mReservedSpaces.get(location.tierAlias()), location, - new BlockMetadataEvictorView(mMetadataManager, Collections.emptySet(), - Collections.emptySet()), - Evictor.Mode.BEST_EFFORT); - - // Extract evicted blocks in order. - List toEvict = Collections.emptyList(); - if (evictionPlan.toEvict() != null) { - toEvict = - evictionPlan.toEvict().stream().map((kvp) -> kvp.getFirst()).collect(Collectors.toList()); - } - - // Extract moved blocks in order. - List toMove = Collections.emptyList(); - if (evictionPlan.toMove() != null) { - toMove = evictionPlan.toMove().stream().map((kvp) -> kvp.getSrcBlockId()) - .collect(Collectors.toList()); - } - - // Select which list to feed as iterator. - // For the lowest tier blocks to evict will be considered. - // For the other cases blocks to move will be considered. - List iteratorList = toEvict; - if (!toMove.isEmpty()) { - iteratorList = toMove; - } - - // Reverse the list if requested. - if (order == BlockOrder.REVERSE) { - Collections.reverse(iteratorList); - } - - // Return an iterator from combined block id list. - return iteratorList.iterator(); - } - - @Override - public List getIntersectionList(BlockStoreLocation srcLocation, BlockOrder srcOrder, - BlockStoreLocation dstLocation, BlockOrder dstOrder, int intersectionWidth, - BlockOrder intersectionOrder, Function blockFilterFunc) { - // Intersection calculation not possible. - return Collections.emptyList(); - } - - @Override - public Pair, List> getSwaps(BlockStoreLocation srcLocation, BlockOrder srcOrder, - BlockStoreLocation dstLocation, BlockOrder dstOrder, int swapRange, - BlockOrder intersectionOrder, Function blockFilterFunc) { - // Swap calculation not possible. - return new Pair(Collections.emptyList(), Collections.emptyList()); - } - - @Override - public boolean aligned(BlockStoreLocation srcLocation, BlockStoreLocation dstLocation, - BlockOrder order, Function blockFilterFunc) { - // Overlap report not possible. - return false; - } - - @Override - public List getListeners() { - if (mEvictor instanceof BlockStoreEventListener) { - return ImmutableList.of((BlockStoreEventListener) mEvictor); - } - return ImmutableList.of(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/LRFUAnnotator.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/LRFUAnnotator.java deleted file mode 100644 index 456ff16a85f8..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/LRFUAnnotator.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicLong; - -/** - * {@link BlockAnnotator} implementation of LRFU scheme. - */ -public class LRFUAnnotator implements BlockAnnotator { - private static final Logger LOG = LoggerFactory.getLogger(LRFUAnnotator.class); - - /** LRU logical clock. */ - private final AtomicLong mLRUClock = new AtomicLong(); - - /** In the range of [0, 1]. Closer to 0, LRFU closer to LFU. Closer to 1, LRFU closer to LRU. */ - private static final double STEP_FACTOR; - /** The attenuation factor is in the range of [2, INF]. */ - private static final double ATTENUATION_FACTOR; - - static { - STEP_FACTOR = Configuration.getDouble( - PropertyKey.WORKER_BLOCK_ANNOTATOR_LRFU_STEP_FACTOR); - ATTENUATION_FACTOR = Configuration.getDouble( - PropertyKey.WORKER_BLOCK_ANNOTATOR_LRFU_ATTENUATION_FACTOR); - } - - @Override - public BlockSortedField updateSortedField(long blockId, LRFUSortedField oldValue) { - return getNewSortedField(blockId, oldValue, mLRUClock.incrementAndGet()); - } - - @Override - public void updateSortedFields(List> blockList) { - // Grab the current logical clock, for updating the given entries under. - long clockValue = mLRUClock.get(); - for (Pair blockField : blockList) { - blockField.setSecond(getNewSortedField( - blockField.getFirst(), blockField.getSecond(), clockValue)); - } - } - - /** - * LRFU is an offline scheme. - * - * @return {@code false} - */ - @Override - public boolean isOnlineSorter() { - return false; - } - - private LRFUSortedField getNewSortedField(long blockId, LRFUSortedField oldValue, long clock) { - double crfValue = (oldValue != null) ? oldValue.mCrfValue : 1.0; - if (oldValue != null && clock != oldValue.mClockValue) { - // CRF(currentLogicTime)=CRF(lastUpdateTime)*F(currentLogicTime-lastUpdateTime)+F(0) - crfValue = oldValue.mCrfValue * calculateAccessWeight(clock - oldValue.mClockValue) + 1.0; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("LRFU update for Block: {}. Clock:{}, CRF: {}", blockId, clock, crfValue); - } - - return new LRFUSortedField(clock, crfValue); - } - - private double calculateAccessWeight(long logicTimeInterval) { - return Math.pow(1.0 / ATTENUATION_FACTOR, logicTimeInterval * STEP_FACTOR); - } - - /** - * Sorted-field for LRFU. - */ - protected static class LRFUSortedField implements BlockSortedField { - private final long mClockValue; - private final double mCrfValue; - - private LRFUSortedField(long clockValue, double crfValue) { - mClockValue = clockValue; - mCrfValue = crfValue; - } - - @Override - public int compareTo(BlockSortedField o) { - Preconditions.checkState(o instanceof LRFUSortedField); - return Double.compare(mCrfValue, ((LRFUSortedField) o).mCrfValue); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof LRFUSortedField)) { - return false; - } - return Double.compare(mCrfValue, ((LRFUSortedField) o).mCrfValue) == 0; - } - - @Override - public int hashCode() { - return Objects.hash(mCrfValue); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("Clock", mClockValue) - .add("CRF", mCrfValue) - .toString(); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/LRUAnnotator.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/LRUAnnotator.java deleted file mode 100644 index 64376c16eda3..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/LRUAnnotator.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.collections.Pair; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -/** - * {@link BlockAnnotator} implementation of LRU scheme. - */ -public class LRUAnnotator implements BlockAnnotator { - private static final Logger LOG = LoggerFactory.getLogger(LRUAnnotator.class); - - private final AtomicLong mLRUClock; - - /** - * Creates a new LRU annotator. - */ - public LRUAnnotator() { - mLRUClock = new AtomicLong(0); - } - - @Override - public BlockSortedField updateSortedField(long blockId, LRUSortedField oldValue) { - long clockValue = mLRUClock.incrementAndGet(); - if (LOG.isDebugEnabled()) { - LOG.debug("LRU update for Block: {}. Clock: {}", blockId, clockValue); - } - return new LRUSortedField(clockValue); - } - - @Override - public void updateSortedFields(List> blocks) { - long currentClock = mLRUClock.get(); - for (Pair blockField : blocks) { - blockField.setSecond(new LRUSortedField(currentClock)); - } - } - - /** - * LRU is an online scheme. - * - * @return {@code true} - */ - @Override - public boolean isOnlineSorter() { - return true; - } - - /** - * Sorted-field for LRU. - */ - protected class LRUSortedField implements BlockSortedField { - private final Long mClockValue; - - private LRUSortedField(long clockValue) { - mClockValue = clockValue; - } - - @Override - public int compareTo(BlockSortedField o) { - Preconditions.checkState(o instanceof LRUSortedField); - return mClockValue.compareTo(((LRUSortedField) o).mClockValue); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof LRUSortedField)) { - return false; - } - return Long.compare(mClockValue, ((LRUSortedField) o).mClockValue) == 0; - } - - @Override - public int hashCode() { - return mClockValue.hashCode(); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("Clock", mClockValue) - .toString(); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/annotator/SortedBlockSet.java b/core/server/worker/src/main/java/alluxio/worker/block/annotator/SortedBlockSet.java deleted file mode 100644 index efa9cb37b163..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/annotator/SortedBlockSet.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.collections.Pair; - -import com.google.common.base.MoreObjects; -import com.google.common.collect.Iterators; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Concurrent set implementation for storing block Ids in sorted form. - * - * @param Field type used for sorting block Ids - */ -public class SortedBlockSet { - private static final Logger LOG = LoggerFactory.getLogger(SortedBlockSet.class); - - /** Used for ordering among block Ids with equal sort values. */ - private final AtomicLong mChangeIndex = new AtomicLong(0); - - /** Underlying concurrent set. */ - private final ConcurrentSkipListSet> mSortedSet; - /** Used to overcome equality side effect of java's sorted sets. */ - private final ConcurrentHashMap> mLastSortId; - - /** - * Creates a new sorted block set. - */ - public SortedBlockSet() { - mSortedSet = new ConcurrentSkipListSet<>(); - mLastSortId = new ConcurrentHashMap<>(); - } - - /** - * Used to get the current sort field for a block Id. - * - * @param blockId block Id - * @return the current sort field - */ - public T getSortField(long blockId) { - Pair sortId = mLastSortId.compute(blockId, (k, v) -> v); - return (sortId != null) ? sortId.getSecond() : null; - } - - /** - * Updates or inserts the collection with new block Id. - * - * @param blockId block id - * @param sortedField sorted field for the block id - */ - public void put(long blockId, T sortedField) { - mLastSortId.compute(blockId, (k, v) -> { - if (v != null) { - SortedBlockSetEntry oldEntry = - new SortedBlockSetEntry<>(blockId, v.getFirst(), v.getSecond()); - boolean wasPresent = mSortedSet.remove(oldEntry); - if (LOG.isDebugEnabled()) { - LOG.debug("#put(): Removed the old entry: {}. WasPresent: {}", oldEntry, wasPresent); - } - } - - Pair newSortId = new Pair<>(mChangeIndex.incrementAndGet(), sortedField); - SortedBlockSetEntry newEntry = - new SortedBlockSetEntry<>(blockId, newSortId.getFirst(), newSortId.getSecond()); - boolean wasNew = mSortedSet.add(newEntry); - if (LOG.isDebugEnabled()) { - LOG.debug("#put(): Added a new entry: {}. WasNew: {}", newEntry, wasNew); - } - return newSortId; - }); - } - - /** - * Removes a block id from the collection. - * - * @param blockId block id - */ - public void remove(long blockId) { - mLastSortId.compute(blockId, (k, v) -> { - if (v != null) { - SortedBlockSetEntry oldEntry = - new SortedBlockSetEntry<>(blockId, v.getFirst(), v.getSecond()); - boolean wasPresent = mSortedSet.remove(oldEntry); - if (LOG.isDebugEnabled()) { - LOG.debug("#remove(): Removed the old entry: {}. WasPresent: {}", oldEntry, wasPresent); - } - } else { - LOG.warn("#remove(): No old entry found for blockId:{}", blockId); - } - return null; - }); - } - - /** - * @return the size of the collection - */ - public int size() { - return mSortedSet.size(); - } - - /** - * @return an ascending iterator of "<BlockId,SortedField>" pairs - */ - public Iterator> getAscendingIterator() { - return Iterators.transform(mSortedSet.iterator(), - (e) -> new Pair<>(e.mBlockId, e.mSortedField)); - } - - /** - * @return a descending iterator of "<BlockId,SortedField>" pairs - */ - public Iterator> getDescendingIterator() { - return Iterators.transform(mSortedSet.descendingIterator(), - (e) -> new Pair<>(e.mBlockId, e.mSortedField)); - } - - /** - * An entry that is stored in an internal set that enforces sorting/equality - * based on the given sorted-field. - * - * @param type of the sorted field - */ - class SortedBlockSetEntry implements Comparable { - private final long mBlockId; - private final T mSortedField; - private final long mChangeIndex; - - public SortedBlockSetEntry(long blockId, long changeIndex, T sortId) { - mBlockId = blockId; - mChangeIndex = changeIndex; - mSortedField = sortId; - } - - @Override - public int compareTo(SortedBlockSetEntry o) { - /* - * Comparison should be consistent with {@link Object#equals} for {@link SortedSet} to - * maintain set semantics. - * - * This means {@link Comparable#compareTo} should never return 0 for non-equal objects. - * This is achieved by using a sequentially increasing change-index field which helps to - * differentiate between unique fields with identical sort-field. - */ - int sortRes = mSortedField.compareTo(o.mSortedField); - if (sortRes == 0) { - return Long.compare(mChangeIndex, o.mChangeIndex); - } else { - return sortRes; - } - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof SortedBlockSetEntry)) { - return false; - } - SortedBlockSetEntry other = (SortedBlockSetEntry) o; - return Objects.equals(mBlockId, other.mBlockId) - && Objects.equals(mSortedField, other.mSortedField) - && Objects.equals(mChangeIndex, other.mChangeIndex); - } - - @Override - public int hashCode() { - return Objects.hash(mBlockId, mSortedField, mChangeIndex); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("BlockId", mBlockId) - .add("SortedField", mSortedField) - .add("ChangeIndex", mChangeIndex) - .toString(); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/evictor/AbstractEvictor.java b/core/server/worker/src/main/java/alluxio/worker/block/evictor/AbstractEvictor.java deleted file mode 100644 index 666d266261fc..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/evictor/AbstractEvictor.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.evictor; - -import alluxio.collections.Pair; -import alluxio.worker.block.AbstractBlockStoreEventListener; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.allocator.Allocator; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.StorageDirEvictorView; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTierView; - -import com.google.common.base.Preconditions; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Provides the basic implementation for every evictor. - * - * @deprecated use block annotator instead - */ -@NotThreadSafe -@Deprecated -public abstract class AbstractEvictor extends AbstractBlockStoreEventListener implements Evictor { - protected final Allocator mAllocator; - protected BlockMetadataEvictorView mMetadataView; - - /** - * Creates a new instance of {@link AbstractEvictor}. - * - * @param view a view of block metadata information - * @param allocator an allocation policy - */ - public AbstractEvictor(BlockMetadataEvictorView view, Allocator allocator) { - mMetadataView = Preconditions.checkNotNull(view, "view"); - mAllocator = Preconditions.checkNotNull(allocator, "allocator"); - } - - /** - * A recursive implementation of cascading eviction. - * - * This method uses a specific eviction strategy to find blocks to evict in the requested - * location. After eviction, one {@link alluxio.worker.block.meta.StorageDir} in the location has - * the specific amount of free space. It then uses an allocation strategy to allocate space in the - * next tier to move each evicted blocks. If the next tier fails to allocate space for the evicted - * blocks, the next tier will continue to evict its blocks to free space. - * - * This method is only used in - * {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataEvictorView)}. - * - * @param bytesToBeAvailable bytes to be available after eviction - * @param location target location to evict blocks from - * @param plan the plan to be recursively updated, is empty when first called in - * {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataEvictorView)} - * @param mode the eviction mode - * @return the first {@link StorageDirEvictorView} in the range of location - * to evict/move bytes from, or null if there is no plan - */ - protected StorageDirEvictorView cascadingEvict(long bytesToBeAvailable, - BlockStoreLocation location, EvictionPlan plan, Mode mode) { - // 1. If bytesToBeAvailable can already be satisfied without eviction, return the eligible - // StorageDirView - StorageDirEvictorView candidateDirView = selectDirWithRequestedSpace( - bytesToBeAvailable, location, mMetadataView); - if (candidateDirView != null) { - return candidateDirView; - } - - // 2. Iterate over blocks in order until we find a StorageDirEvictorView that is - // in the range of location and can satisfy bytesToBeAvailable - // after evicting its blocks iterated so far - EvictionDirCandidates dirCandidates = new EvictionDirCandidates(); - Iterator it = getBlockIterator(); - while (it.hasNext() && dirCandidates.candidateSize() < bytesToBeAvailable) { - long blockId = it.next(); - Optional optionalBlockMeta = mMetadataView.getBlockMeta(blockId); - if (!optionalBlockMeta.isPresent()) { - it.remove(); - onRemoveBlockFromIterator(blockId); - continue; - } - BlockMeta block = optionalBlockMeta.get(); - // might not present in this view - if (block.getBlockLocation().belongsTo(location)) { - String tierAlias = block.getParentDir().getParentTier().getTierAlias(); - int dirIndex = block.getParentDir().getDirIndex(); - StorageDirView dirView = mMetadataView.getTierView(tierAlias).getDirView(dirIndex); - if (dirView != null) { - dirCandidates.add((StorageDirEvictorView) dirView, blockId, block.getBlockSize()); - } - } - } - - // 3. If there is no eligible StorageDirEvictorView, return null - if (mode == Mode.GUARANTEED && dirCandidates.candidateSize() < bytesToBeAvailable) { - return null; - } - - // 4. cascading eviction: try to allocate space in the next tier to move candidate blocks - // there. If allocation fails, the next tier will continue to evict its blocks to free space. - // Blocks are only evicted from the last tier or it can not be moved to the next tier. - candidateDirView = dirCandidates.candidateDir(); - if (candidateDirView == null) { - return null; - } - List candidateBlocks = dirCandidates.candidateBlocks(); - StorageTierView nextTierView - = mMetadataView.getNextTier(candidateDirView.getParentTierView()); - if (nextTierView == null) { - // This is the last tier, evict all the blocks. - for (Long blockId : candidateBlocks) { - Optional block = mMetadataView.getBlockMeta(blockId); - if (block.isPresent()) { - candidateDirView.markBlockMoveOut(blockId, block.get().getBlockSize()); - plan.toEvict().add(new Pair<>(blockId, candidateDirView.toBlockStoreLocation())); - } - } - } else { - for (Long blockId : candidateBlocks) { - Optional optionalBlock = mMetadataView.getBlockMeta(blockId); - if (!optionalBlock.isPresent()) { - continue; - } - BlockMeta block = optionalBlock.get(); - StorageDirEvictorView nextDirView = - (StorageDirEvictorView) mAllocator.allocateBlockWithView( - block.getBlockSize(), - BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), - mMetadataView, true); - if (nextDirView == null) { - nextDirView = cascadingEvict(block.getBlockSize(), - BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), plan, mode); - } - if (nextDirView == null) { - // If we failed to find a dir in the next tier to move this block, evict it and - // continue. Normally this should not happen. - plan.toEvict().add(new Pair<>(blockId, block.getBlockLocation())); - candidateDirView.markBlockMoveOut(blockId, block.getBlockSize()); - continue; - } - plan.toMove().add(BlockTransferInfo.createMove(block.getBlockLocation(), blockId, - nextDirView.toBlockStoreLocation())); - candidateDirView.markBlockMoveOut(blockId, block.getBlockSize()); - nextDirView.markBlockMoveIn(blockId, block.getBlockSize()); - } - } - - return candidateDirView; - } - - @Override - public EvictionPlan freeSpaceWithView(long bytesToBeAvailable, BlockStoreLocation location, - BlockMetadataEvictorView view) { - return freeSpaceWithView(bytesToBeAvailable, location, view, Mode.GUARANTEED); - } - - @Override - public EvictionPlan freeSpaceWithView(long bytesToBeAvailable, BlockStoreLocation location, - BlockMetadataEvictorView view, Mode mode) { - mMetadataView = view; - - List toMove = new ArrayList<>(); - List> toEvict = new ArrayList<>(); - EvictionPlan plan = new EvictionPlan(toMove, toEvict); - StorageDirEvictorView candidateDir = cascadingEvict(bytesToBeAvailable, location, plan, mode); - - mMetadataView.clearBlockMarks(); - if (candidateDir == null) { - return null; - } - - return plan; - } - - /** - * Returns an iterator for evictor cache blocks. The evictor is responsible for specifying the - * iteration order using its own strategy. For example, {@link LRUEvictor} returns an iterator - * that iterates through the block ids in LRU order. - * - * @return an iterator over the ids of the blocks in the evictor cache - */ - protected abstract Iterator getBlockIterator(); - - /** - * Performs additional cleanup when a block is removed from the iterator returned by - * {@link #getBlockIterator()}. - */ - protected void onRemoveBlockFromIterator(long blockId) {} - - /** - * Finds a directory in the given location range with capacity upwards of the given bound. - * - * @param bytesToBeAvailable the capacity bound - * @param location the location range - * @param mManagerView the storage manager view - * @return a {@link StorageDirEvictorView} in the range of location that already - * has availableBytes larger than bytesToBeAvailable, otherwise null - */ - @Nullable - private static StorageDirEvictorView selectDirWithRequestedSpace(long bytesToBeAvailable, - BlockStoreLocation location, BlockMetadataEvictorView mManagerView) { - if (location.hasNoRestriction()) { - for (StorageTierView tierView : mManagerView.getTierViews()) { - for (StorageDirView dirView : tierView.getDirViews()) { - if (dirView.getAvailableBytes() >= bytesToBeAvailable) { - return (StorageDirEvictorView) dirView; - } - } - } - return null; - } - - String tierAlias = location.tierAlias(); - StorageTierView tierView = mManagerView.getTierView(tierAlias); - if (location.isAnyDir() && location.isAnyMedium()) { - for (StorageDirView dirView : tierView.getDirViews()) { - if (dirView.getAvailableBytes() >= bytesToBeAvailable) { - return (StorageDirEvictorView) dirView; - } - } - return null; - } - - StorageDirView dirView = tierView.getDirView(location.dir()); - return (dirView != null && dirView.getAvailableBytes() >= bytesToBeAvailable) - ? (StorageDirEvictorView) dirView : null; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/evictor/BlockTransferInfo.java b/core/server/worker/src/main/java/alluxio/worker/block/evictor/BlockTransferInfo.java deleted file mode 100644 index 64842879afe3..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/evictor/BlockTransferInfo.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.evictor; - -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.base.MoreObjects; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Provides information about the transfer of a block. - */ -@ThreadSafe -public class BlockTransferInfo { - private static final int INVALID = -1; - - private final long mSrcBlockId; - private final long mDstBlockId; - private final BlockStoreLocation mSrcLocation; - private final BlockStoreLocation mDstLocation; - - private BlockTransferInfo(BlockStoreLocation srcLocation, long srcBlockId, - BlockStoreLocation dstLocation, long dstBlockId) { - mSrcLocation = srcLocation; - mSrcBlockId = srcBlockId; - mDstLocation = dstLocation; - mDstBlockId = dstBlockId; - } - - /** - * Creates a new instance of {@link BlockTransferInfo} for moving a block. - * - * @param srcLocation the source {@link BlockStoreLocation} - * @param srcBlockId the source block id - * @param dstLocation the destination {@link BlockStoreLocation} - * @return the transfer info object - */ - public static BlockTransferInfo createMove(BlockStoreLocation srcLocation, long srcBlockId, - BlockStoreLocation dstLocation) { - return new BlockTransferInfo(srcLocation, srcBlockId, dstLocation, INVALID); - } - - /** - * Creates a new instance of {@link BlockTransferInfo} for swapping two blocks. - * - * @param srcLocation the source {@link BlockStoreLocation} - * @param srcBlockId the source block id - * @param dstLocation the destination {@link BlockStoreLocation} - * @param dstBlockId the destination block id - * @return the transfer info object - */ - public static BlockTransferInfo createSwap(BlockStoreLocation srcLocation, long srcBlockId, - BlockStoreLocation dstLocation, long dstBlockId) { - return new BlockTransferInfo(srcLocation, srcBlockId, dstLocation, dstBlockId); - } - - /** - * @return the source block id - */ - public long getSrcBlockId() { - return mSrcBlockId; - } - - /** - * @return the destination block id - */ - public long getDstBlockId() { - return mDstBlockId; - } - - /** - * @return {@code true} if this tranfer info is for a swap operation - */ - public boolean isSwap() { - return mDstBlockId != INVALID; - } - - /** - * @return the source {@link BlockStoreLocation} - */ - public BlockStoreLocation getSrcLocation() { - return mSrcLocation; - } - - /** - * @return the destination {@link BlockStoreLocation} - */ - public BlockStoreLocation getDstLocation() { - return mDstLocation; - } - - @Override - public String toString() { - MoreObjects.ToStringHelper strHelper = MoreObjects.toStringHelper(this); - if (isSwap()) { - strHelper.add("TransferType", "SWAP"); - strHelper.add("SrcBlockId", mSrcBlockId); - strHelper.add("DstBlockId", mDstBlockId); - } else { - strHelper.add("TransferType", "MOVE"); - strHelper.add("BlockId", mSrcBlockId); - } - - strHelper.add("SrcLocation", mSrcLocation); - strHelper.add("DstLocation", mDstLocation); - return strHelper.toString(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/evictor/EvictionDirCandidates.java b/core/server/worker/src/main/java/alluxio/worker/block/evictor/EvictionDirCandidates.java deleted file mode 100644 index c291a3e3edad..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/evictor/EvictionDirCandidates.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.evictor; - -import alluxio.collections.Pair; -import alluxio.worker.block.meta.StorageDirEvictorView; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A collection of candidate blocks for eviction organized by directory. - * - * This class lets you add blocks with the {@link alluxio.worker.block.meta.StorageDir}s they reside - * in, the blockIds and the sizes in bytes, then it tells you which StorageDir added so far has the - * maximum sum of available bytes and total bytes of added blocks. Assume metadata of StorageDir - * will not be changed during adding blocks. - * - * Example usage can be found in {@link LRUEvictor#freeSpaceWithView}. - */ -@NotThreadSafe -class EvictionDirCandidates { - /** - * Map from {@link StorageDirEvictorView} to pair of list of candidate blockIds - * and their total size in bytes. - */ - private final Map, Long>> mDirCandidates = new HashMap<>(); - /** Maximum sum of available bytes in a StorageDir and all its added blocks. */ - private long mMaxBytes = 0; - private StorageDirEvictorView mDirWithMaxBytes = null; - - /** - * Constructs a new {@link EvictionDirCandidates}. - */ - public EvictionDirCandidates() {} - - /** - * Adds the block in the directory to this collection. - * - * @param dir the dir where the block resides - * @param blockId blockId of the block - * @param blockSizeBytes block size in bytes - */ - public void add(StorageDirEvictorView dir, long blockId, long blockSizeBytes) { - Pair, Long> candidate; - if (mDirCandidates.containsKey(dir)) { - candidate = mDirCandidates.get(dir); - } else { - candidate = new Pair<>(new ArrayList<>(), 0L); - mDirCandidates.put(dir, candidate); - } - - candidate.getFirst().add(blockId); - long blockBytes = candidate.getSecond() + blockSizeBytes; - candidate.setSecond(blockBytes); - - long sum = blockBytes + dir.getAvailableBytes(); - if (mMaxBytes < sum) { - mMaxBytes = sum; - mDirWithMaxBytes = dir; - } - } - - /** - * The maximum sum of available bytes and total bytes of added blocks in a directory. - * - * @return maximum bytes, if no directory has been added, return 0 - */ - public long candidateSize() { - return mMaxBytes; - } - - /** - * @return list of blockIds in the directory that has the maximum {@link #candidateSize()}, - * otherwise an empty list if no directory has been added - */ - public List candidateBlocks() { - Pair, Long> evict = mDirCandidates.get(mDirWithMaxBytes); - if (evict == null) { - return new ArrayList<>(); - } - return evict.getFirst(); - } - - /** - * @return the {@link alluxio.worker.block.meta.StorageDir} that has the maximum - * {@link #candidateSize()}, otherwise null if no directory has been added - */ - public StorageDirEvictorView candidateDir() { - return mDirWithMaxBytes; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/evictor/EvictionPlan.java b/core/server/worker/src/main/java/alluxio/worker/block/evictor/EvictionPlan.java deleted file mode 100644 index 130a28b0fe64..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/evictor/EvictionPlan.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.evictor; - -import alluxio.collections.Pair; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; - -import java.util.List; -import javax.annotation.concurrent.ThreadSafe; - -/** - * This class provides information about the blocks that need to be moved when evicting. - */ -@ThreadSafe -public final class EvictionPlan { - /** A list of block transfer information, with block id, source and destination location. */ - private final List mToMove; - /** A list of pairs of block id to remove and its location. */ - private final List> mToEvict; - - /** - * Creates a new instance of {@link EvictionPlan}. - * - * @param toTransfer a list of block transfer information - * @param toEvict a list of blocks to be evicted - */ - public EvictionPlan(List toTransfer, - List> toEvict) { - mToMove = Preconditions.checkNotNull(toTransfer, "toTransfer"); - mToEvict = Preconditions.checkNotNull(toEvict, "toEvict"); - } - - /** - * @return a list of block transfer information, with block id, source and destination location - */ - public List toMove() { - return mToMove; - } - - /** - * @return a list of pairs of block id to remove and its location - */ - public List> toEvict() { - return mToEvict; - } - - /** - * Whether the plan is empty, an empty plan means both toMove and toEvict are empty, also, an - * empty plan indicates no action (move or evict) needs to be taken to meet the requirement. - * - * @return true if empty otherwise false - */ - public boolean isEmpty() { - return mToEvict.isEmpty() && mToMove.isEmpty(); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("toMove", mToMove) - .add("toEvict", mToEvict).toString(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/evictor/Evictor.java b/core/server/worker/src/main/java/alluxio/worker/block/evictor/Evictor.java deleted file mode 100644 index ab05724ad589..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/evictor/Evictor.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.evictor; - -import alluxio.annotation.PublicApi; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.LocalBlockStore; -import alluxio.worker.block.allocator.Allocator; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Interface for the eviction policy in Alluxio. - */ -@PublicApi -public interface Evictor { - - /** - * The eviction mode. - */ - enum Mode { - BEST_EFFORT, GUARANTEED - } - - /** - * Factory for {@link Evictor}. - */ - @ThreadSafe - class Factory { - - private Factory() {} // prevent instantiation - - /** - * Factory for {@link Evictor}. - * - * @param view {@link BlockMetadataEvictorView} to pass to {@link Evictor} - * @param allocator an allocation policy - * @return the generated {@link Evictor} - */ - public static Evictor create(BlockMetadataEvictorView view, Allocator allocator) { - return CommonUtils.createNewClassInstance( - Configuration.getClass(PropertyKey.WORKER_EVICTOR_CLASS), - new Class[] {BlockMetadataEvictorView.class, Allocator.class}, - new Object[] {view, allocator}); - } - } - - /** - * Frees space with the guaranteed mode. - * - * @param availableBytes the amount of free space in bytes to be ensured after eviction - * @param location the location in block store - * @param view generated and passed by block store - * @return an {@link EvictionPlan} (possibly with empty fields) to get the free space, or null - * if no plan is feasible - */ - EvictionPlan freeSpaceWithView(long availableBytes, BlockStoreLocation location, - BlockMetadataEvictorView view); - - /** - * Frees space in the given block store location and with the given view. - * - * With the GUARANTEED mode, after eviction at least one - * {@link alluxio.worker.block.meta.StorageDir} in the location has the specific amount of free - * space after eviction. The location can be a specific - * {@link alluxio.worker.block.meta.StorageDir}, or {@link BlockStoreLocation#anyTier()} or - * {@link BlockStoreLocation#anyDirInTier(String)}. The view is generated and passed by the - * calling {@link LocalBlockStore}. This method returns null if {@link Evictor} - * fails to propose a feasible plan to meet the requirement. - *

- * With the BEST_EFFORT mode, the evictor always returns an eviction plan with toMove and toEvict - * fields to indicate how to free space. Even if the tier does not have the amount of free space, - * the evictor returns the plan to free the max space. - *

- * If both toMove and toEvict of the plan are empty, it indicates that {@link Evictor} has no - * actions to take and the requirement is already met. - *

- * Throws an {@link IllegalArgumentException} if the given block location is invalid. - * - * @param availableBytes the amount of free space in bytes to be ensured after eviction - * @param location the location in block store - * @param view generated and passed by block store - * @param mode the eviction mode - * @return an {@link EvictionPlan} (possibly with empty fields) to get the free space, or null - * if no plan is feasible - */ - EvictionPlan freeSpaceWithView(long availableBytes, BlockStoreLocation location, - BlockMetadataEvictorView view, Mode mode); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/evictor/LRUEvictor.java b/core/server/worker/src/main/java/alluxio/worker/block/evictor/LRUEvictor.java deleted file mode 100644 index 6074e4358f4f..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/evictor/LRUEvictor.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.evictor; - -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.allocator.Allocator; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.StorageDirEvictorView; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTierView; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Implementation of an evictor which follows the least recently used algorithm. It discards the - * least recently used item based on its access. - * - * @deprecated use block annotator instead - */ -@NotThreadSafe -@Deprecated -public class LRUEvictor extends AbstractEvictor { - private static final int LINKED_HASH_MAP_INIT_CAPACITY = 200; - private static final float LINKED_HASH_MAP_INIT_LOAD_FACTOR = 0.75f; - private static final boolean LINKED_HASH_MAP_ACCESS_ORDERED = true; - private static final boolean UNUSED_MAP_VALUE = true; - - /** - * Access-ordered {@link java.util.LinkedHashMap} from blockId to {@link #UNUSED_MAP_VALUE}(just a - * placeholder to occupy the value), acts as a LRU double linked list where most recently accessed - * element is put at the tail while least recently accessed element is put at the head. - */ - protected Map mLRUCache = - Collections.synchronizedMap(new LinkedHashMap(LINKED_HASH_MAP_INIT_CAPACITY, - LINKED_HASH_MAP_INIT_LOAD_FACTOR, LINKED_HASH_MAP_ACCESS_ORDERED)); - - /** - * Creates a new instance of {@link LRUEvictor}. - * - * @param view a view of block metadata information - * @param allocator an allocation policy - */ - public LRUEvictor(BlockMetadataEvictorView view, Allocator allocator) { - super(view, allocator); - - // preload existing blocks loaded by StorageDir to Evictor - for (StorageTierView tierView : mMetadataView.getTierViews()) { - for (StorageDirView dirView : tierView.getDirViews()) { - for (BlockMeta blockMeta : ((StorageDirEvictorView) dirView) - .getEvictableBlocks()) { // all blocks with initial view - mLRUCache.put(blockMeta.getBlockId(), UNUSED_MAP_VALUE); - } - } - } - } - - @Override - protected Iterator getBlockIterator() { - List blocks = new ArrayList<>(mLRUCache.keySet()); - return blocks.iterator(); - } - - @Override - public void onAccessBlock(long blockId) { - mLRUCache.put(blockId, UNUSED_MAP_VALUE); - } - - @Override - public void onCommitBlock(long blockId, BlockStoreLocation location) { - // Since the temp block has been committed, update Evictor about the new added blocks - mLRUCache.put(blockId, UNUSED_MAP_VALUE); - } - - @Override - public void onRemoveBlockByClient(long blockId) { - mLRUCache.remove(blockId); - } - - @Override - public void onRemoveBlockByWorker(long blockId) { - mLRUCache.remove(blockId); - } - - @Override - public void onBlockLost(long blockId) { - mLRUCache.remove(blockId); - } - - @Override - protected void onRemoveBlockFromIterator(long blockId) { - mLRUCache.remove(blockId); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/evictor/package-info.java b/core/server/worker/src/main/java/alluxio/worker/block/evictor/package-info.java deleted file mode 100644 index 84fe4f8a5399..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/evictor/package-info.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -/** - * Set of evictors for evicting or moving blocks to other locations. - * - * The main entry point is {@link alluxio.worker.block.evictor.Evictor} which returns an evictor by - * calling alluxio.worker.block.Evictor.Factory#create. - * - *

Evictor

- * - * All evictors implement {@link alluxio.worker.block.evictor.Evictor} interface. In order to evict - * blocks by different policies, each evictor only need to implement - * {@link alluxio.worker.block.evictor.Evictor#freeSpaceWithView(long, - * alluxio.worker.block.BlockStoreLocation, alluxio.worker.block.BlockMetadataEvictorView)} - * method. When this method is called, blocks will be evicted or moved if no enough space left. - * - *

Cascading Eviction

Cascading Eviction means evict blocks recursively and is only called - * in {@link alluxio.worker.block.evictor.Evictor#freeSpaceWithView(long, - * alluxio.worker.block.BlockStoreLocation, alluxio.worker.block.BlockMetadataEvictorView)} - * method. Cascading Eviction will try to free space in next tier view where blocks need to be - * transferred to, if the next tier view does not have enough free space to hold the blocks, the - * next next tier view will be tried and so on until the bottom tier is reached, if blocks can not - * even be transferred to the bottom tier, they will be evicted, otherwise, only blocks to be freed - * in the bottom tier will be evicted. - * - * For example, - * {@link alluxio.worker.block.evictor.LRUEvictor#cascadingEvict(long, - * alluxio.worker.block.BlockStoreLocation,EvictionPlan)} - * is an implementation of Cascading Eviction. - * - *

Eviction Plan

- * - * Eviction plan will be returned when - * {@link alluxio.worker.block.evictor.Evictor#freeSpaceWithView(long, - * alluxio.worker.block.BlockStoreLocation, alluxio.worker.block.BlockMetadataEvictorView)} - * is called. The eviction plan will return two block lists, one is the blocks to be removed - * directly and another is the blocks to be moved to lower tier views. - */ - -package alluxio.worker.block.evictor; diff --git a/core/server/worker/src/main/java/alluxio/worker/block/io/BlockStreamTracker.java b/core/server/worker/src/main/java/alluxio/worker/block/io/BlockStreamTracker.java deleted file mode 100644 index 0912e446eb0c..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/io/BlockStreamTracker.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.io; - -import alluxio.worker.block.BlockStoreLocation; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -/** - * Used to emit block reader/writer open/close events. - */ -public class BlockStreamTracker { - /** List of listeners for this tracker. */ - private static final List LISTENERS = new CopyOnWriteArrayList<>(); - - /** - * Register a new listener. - * - * @param listener listener - */ - public static void registerListener(BlockClientListener listener) { - LISTENERS.add(listener); - } - - /** - * Unregisters a listener. - * - * @param listener listener - */ - public static void unregisterListener(BlockClientListener listener) { - LISTENERS.remove(listener); - } - - /** - * Called with a new block reader. - * - * @param reader block reader - * @param location location of read - */ - public static void readerOpened(BlockReader reader, BlockStoreLocation location) { - for (BlockClientListener listener : LISTENERS) { - listener.clientOpened(reader, location); - } - } - - /** - * Called when an existing block reader is closed. - * - * @param reader block reader - * @param location location of read - */ - public static void readerClosed(BlockReader reader, BlockStoreLocation location) { - for (BlockClientListener listener : LISTENERS) { - listener.clientClosed(reader, location); - } - } - - /** - * Called with a new block writer. - * - * @param writer block writer - * @param location location of write - */ - public static void writerOpened(BlockWriter writer, BlockStoreLocation location) { - for (BlockClientListener listener : LISTENERS) { - listener.clientOpened(writer, location); - } - } - - /** - * Called when an existing block writer is closed. - * - * @param writer block writer - * @param location location of write - */ - public static void writerClosed(BlockWriter writer, BlockStoreLocation location) { - for (BlockClientListener listener : LISTENERS) { - listener.clientClosed(writer, location); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/io/StoreBlockReader.java b/core/server/worker/src/main/java/alluxio/worker/block/io/StoreBlockReader.java deleted file mode 100644 index fb36e9362367..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/io/StoreBlockReader.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.io; - -import alluxio.worker.block.meta.BlockMeta; - -import java.io.IOException; - -/** - * A local block reader used by block store. - * It provides integration with the {@link BlockStreamTracker}. - */ -public class StoreBlockReader extends LocalFileBlockReader { - /** Session Id for the reader. */ - private final long mSessionId; - /** Block meta for the reader. */ - private final BlockMeta mBlockMeta; - - /** - * Creates new block reader for block store. - * - * @param sessionId session id - * @param blockMeta block meta - * @throws IOException - */ - public StoreBlockReader(long sessionId, BlockMeta blockMeta) throws IOException { - super(blockMeta.getPath()); - mSessionId = sessionId; - mBlockMeta = blockMeta; - if (mSessionId > 0) { - BlockStreamTracker.readerOpened(this, mBlockMeta.getBlockLocation()); - } - } - - @Override - public void close() throws IOException { - if (mSessionId > 0) { - BlockStreamTracker.readerClosed(this, mBlockMeta.getBlockLocation()); - } - super.close(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/io/StoreBlockWriter.java b/core/server/worker/src/main/java/alluxio/worker/block/io/StoreBlockWriter.java deleted file mode 100644 index 068763466685..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/io/StoreBlockWriter.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.io; - -import alluxio.worker.block.meta.TempBlockMeta; - -import java.io.IOException; - -/** - * A local block writer used by block store. - * It provides integration with the {@link BlockStreamTracker}. - */ -public class StoreBlockWriter extends LocalFileBlockWriter { - /** Temp block meta for the writer. */ - private final TempBlockMeta mBlockMeta; - - /** - * Creates new block writer for block store. - * - * @param blockMeta temp block meta - * @throws IOException - */ - public StoreBlockWriter(TempBlockMeta blockMeta) { - super(blockMeta.getPath()); - mBlockMeta = blockMeta; - if (mBlockMeta.getSessionId() > 0) { - BlockStreamTracker.writerOpened(this, mBlockMeta.getBlockLocation()); - } - } - - @Override - public void close() throws IOException { - if (mBlockMeta.getSessionId() > 0) { - BlockStreamTracker.writerClosed(this, mBlockMeta.getBlockLocation()); - } - super.close(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/AbstractBlockManagementTask.java b/core/server/worker/src/main/java/alluxio/worker/block/management/AbstractBlockManagementTask.java deleted file mode 100644 index 5f473cf34433..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/AbstractBlockManagementTask.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.LocalBlockStore; - -import java.util.concurrent.ExecutorService; - -/** - * Abstract block management task implementation. - */ -public abstract class AbstractBlockManagementTask implements BlockManagementTask { - protected final LocalBlockStore mBlockStore; - protected final BlockMetadataManager mMetadataManager; - protected final BlockMetadataEvictorView mEvictorView; - protected final StoreLoadTracker mLoadTracker; - protected final ExecutorService mExecutor; - protected final BlockTransferExecutor mTransferExecutor; - - /** - * Creates abstract task implementation. - * - * @param blockStore the block store - * @param metadataManager the meta manager - * @param evictorView the evictor view - * @param loadTracker the load tracker - * @param executor the executor to use for task execution - */ - public AbstractBlockManagementTask(LocalBlockStore blockStore, - BlockMetadataManager metadataManager, BlockMetadataEvictorView evictorView, - StoreLoadTracker loadTracker, ExecutorService executor) { - mBlockStore = blockStore; - mMetadataManager = metadataManager; - mEvictorView = evictorView; - mLoadTracker = loadTracker; - mExecutor = executor; - mTransferExecutor = new BlockTransferExecutor(executor, blockStore, loadTracker, - Configuration.getInt(PropertyKey.WORKER_MANAGEMENT_BLOCK_TRANSFER_CONCURRENCY_LIMIT)); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockManagementTask.java b/core/server/worker/src/main/java/alluxio/worker/block/management/BlockManagementTask.java deleted file mode 100644 index 47617859ee8a..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockManagementTask.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -/** - * An interface for BlockStore management task. - */ -public interface BlockManagementTask { - - /** - * Run management task. - * - * @return the task result - */ - BlockManagementTaskResult run(); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockManagementTaskResult.java b/core/server/worker/src/main/java/alluxio/worker/block/management/BlockManagementTaskResult.java deleted file mode 100644 index e685bc9ac1f4..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockManagementTaskResult.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -import com.google.common.base.MoreObjects; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Holds results of a {@link BlockManagementTask}. - * - * It contains sub-results for each {@link BlockOperationType} that is issued - * by the magement task. - */ -@NotThreadSafe -public class BlockManagementTaskResult { - /** Map of block-operation results. */ - private final Map mBlockOpResults = new HashMap<>(); - - /** - * Creates a new empty task result. - */ - public BlockManagementTaskResult() {} - - /** - * Add results for a specific {@link BlockOperationType}. - * Results will be merged, if this task result contains results for the given operation. - * - * @param opType block operation type - * @param result block operation result - * @return the updated task result - */ - public BlockManagementTaskResult addOpResults(BlockOperationType opType, - BlockOperationResult result) { - if (!mBlockOpResults.containsKey(opType)) { - mBlockOpResults.put(opType, new BlockOperationResult()); - } - mBlockOpResults.get(opType).mergeWith(result); - return this; - } - - /** - * @param opType block operation type - * @return results for the given operation type (null if don't exist) - */ - public BlockOperationResult getOperationResult(BlockOperationType opType) { - return mBlockOpResults.get(opType); - } - - /** - * @return {@code true} if the task had no progress due to failures/back-offs - */ - public boolean noProgress() { - int opCount = 0; - int failCount = 0; - int backOffCount = 0; - for (BlockOperationResult result : mBlockOpResults.values()) { - opCount += result.opCount(); - failCount += result.failCount(); - backOffCount += result.backOffCount(); - } - return opCount == failCount + backOffCount; - } - - @Override - public String toString() { - MoreObjects.ToStringHelper strHelper = MoreObjects.toStringHelper(this); - for (Map.Entry entry : mBlockOpResults.entrySet()) { - strHelper.add(entry.getKey().name(), entry.getValue()); - } - return strHelper.toString(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockOperationResult.java b/core/server/worker/src/main/java/alluxio/worker/block/management/BlockOperationResult.java deleted file mode 100644 index 40d7fa9e6666..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockOperationResult.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -import com.google.common.base.MoreObjects; - -/** - * Result container for unique block management sub-task. - * Sub-tasks are defined by {@link BlockOperationType} enum. - */ -public class BlockOperationResult { - /** How many operations. */ - private int mOpCount; - /** How many failed. */ - private int mFailCount; - /** How many backed off. */ - private int mBackoffCount; - - /** - * Creates an empty operation result. - */ - public BlockOperationResult() { - this(0, 0, 0); - } - - /** - * Creates an operation result. - * - * @param opCount operation count - * @param failCount failure count - * @param backOffCount back-off count - */ - public BlockOperationResult(int opCount, int failCount, int backOffCount) { - mOpCount = opCount; - mFailCount = failCount; - mBackoffCount = backOffCount; - } - - /** - * Merges result counters of this result with the given result. - * - * @param otherResult the result to merge with - * @return the updated block operation result - */ - public BlockOperationResult mergeWith(BlockOperationResult otherResult) { - mOpCount += otherResult.mOpCount; - mFailCount += otherResult.mFailCount; - mBackoffCount += otherResult.mBackoffCount; - return this; - } - - /** - * @return the operation count - */ - public int opCount() { - return mOpCount; - } - - /** - * @return the failure count - */ - public int failCount() { - return mFailCount; - } - - /** - * @return the back-off count - */ - public int backOffCount() { - return mBackoffCount; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("OpCount", mOpCount) - .add("FailCount", mFailCount) - .add("BackOffCount", mBackoffCount) - .toString(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockOperationType.java b/core/server/worker/src/main/java/alluxio/worker/block/management/BlockOperationType.java deleted file mode 100644 index f44eec91dc34..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockOperationType.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -/** - * Used to specify several operation stages executed by {@link BlockManagementTask}s. - */ -public enum BlockOperationType { - ALIGN_SWAP, // {@link AlignTask} swap transfers. - PROMOTE_MOVE, // {@link PromoteTask} move transfers. - SWAP_RESTORE_REMOVE, // {@link SwapRestoreTask} removals. - SWAP_RESTORE_FLUSH, // {@link SwapRestoreTask} flush moves. - SWAP_RESTORE_BALANCE // {@link SwapRestoreTask} balance moves. -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockTransferExecutor.java b/core/server/worker/src/main/java/alluxio/worker/block/management/BlockTransferExecutor.java deleted file mode 100644 index d3f1c0904412..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockTransferExecutor.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -import alluxio.Sessions; -import alluxio.worker.block.AllocateOptions; -import alluxio.worker.block.LocalBlockStore; -import alluxio.worker.block.evictor.BlockTransferInfo; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; - -/** - * Used to execute list of {@link BlockTransferInfo} orders concurrently. - */ -public class BlockTransferExecutor { - private static final Logger LOG = LoggerFactory.getLogger(BlockTransferExecutor.class); - - private final ExecutorService mExecutor; - private final LocalBlockStore mBlockStore; - private final StoreLoadTracker mLoadTracker; - private final int mConcurrencyLimit; - private final BlockTransferPartitioner mPartitioner; - - /** - * Creates a new instance for executing block transfers. - * - * @param executor the executor to use - * @param blockStore the block store - * @param loadTracker the load tracker - * @param concurrencyLimit the max concurrent transfers - */ - public BlockTransferExecutor(ExecutorService executor, LocalBlockStore blockStore, - StoreLoadTracker loadTracker, int concurrencyLimit) { - mExecutor = executor; - mBlockStore = blockStore; - mLoadTracker = loadTracker; - mConcurrencyLimit = concurrencyLimit; - mPartitioner = new BlockTransferPartitioner(); - } - - /** - * Executes given list of {@link BlockTransferInfo}s. - * - * @param transferInfos the list of transfers - * @return the result of transfers - */ - public BlockOperationResult executeTransferList(List transferInfos) { - return executeTransferList(transferInfos, null); - } - - /** - * Executes given list of {@link BlockTransferInfo}s. - * - * @param transferInfos the list of transfers - * @param exceptionHandler exception handler for when a transfer fails - * @return the result of transfers - */ - public BlockOperationResult executeTransferList(List transferInfos, - Consumer exceptionHandler) { - LOG.debug("Executing transfer list of size: {}. Concurrency limit: {}", - transferInfos.size(), mConcurrencyLimit); - // Return immediately for an empty transfer list. - if (transferInfos.isEmpty()) { - return new BlockOperationResult(); - } - // Partition executions into sub-lists. - List> executionPartitions = - mPartitioner.partitionTransfers(transferInfos, mConcurrencyLimit); - // Counters for ops/failures/backoffs. - AtomicInteger opCount = new AtomicInteger(0); - AtomicInteger failCount = new AtomicInteger(0); - AtomicInteger backOffCount = new AtomicInteger(0); - // Execute to-be-transferred blocks from the plan. - Collection> executionTasks = new LinkedList<>(); - for (List executionPartition : executionPartitions) { - executionTasks.add(() -> { - // TODO(ggezer): Prevent collisions by locking on locations. - // Above to-do requires both source and destination locations to be allocated. - BlockOperationResult res = executeTransferPartition(executionPartition, exceptionHandler); - // Accumulate partition results. - opCount.addAndGet(res.opCount()); - failCount.addAndGet(res.failCount()); - backOffCount.addAndGet(res.backOffCount()); - return null; - }); - } - LOG.debug("Executing {} concurrent transfer partitions.", executionTasks.size()); - try { - mExecutor.invokeAll(executionTasks); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - return new BlockOperationResult(opCount.get(), failCount.get(), backOffCount.get()); - } - - /** - * Used as entry point for executing a single transfer partition. - */ - private BlockOperationResult executeTransferPartition(List transferInfos, - Consumer exceptionHandler) { - LOG.debug("Executing transfer partition of size {}", transferInfos.size()); - // Counters for failure and back-offs. - int failCount = 0; - int backOffCount = 0; - // Execute transfers in order. - for (BlockTransferInfo transferInfo : transferInfos) { - try { - if (mLoadTracker.loadDetected(transferInfo.getSrcLocation(), - transferInfo.getDstLocation())) { - LOG.debug("Skipping transfer-order: {} due to user activity.", transferInfo); - backOffCount++; - continue; - } - - boolean useReservedSpace = transferInfo.isSwap(); - - mBlockStore.moveBlock(Sessions.createInternalSessionId(), transferInfo.getSrcBlockId(), - AllocateOptions.forTierMove(transferInfo.getDstLocation()) - .setUseReservedSpace(useReservedSpace)); - if (transferInfo.isSwap()) { - // TODO(ggezer): Implement external allocations to guarantee a swap. - mBlockStore.moveBlock(Sessions.createInternalSessionId(), transferInfo.getDstBlockId(), - AllocateOptions.forTierMove(transferInfo.getSrcLocation()) - .setUseReservedSpace(useReservedSpace)); - } - } catch (Exception e) { - LOG.warn("Transfer-order: {} failed. {} ", transferInfo, e.toString()); - failCount++; - if (exceptionHandler != null) { - exceptionHandler.accept(e); - } - } - } - - return new BlockOperationResult(transferInfos.size(), failCount, backOffCount); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockTransferPartitioner.java b/core/server/worker/src/main/java/alluxio/worker/block/management/BlockTransferPartitioner.java deleted file mode 100644 index ce01daa14983..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/BlockTransferPartitioner.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.evictor.BlockTransferInfo; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Used to partition transfers for concurrent execution. - */ -public class BlockTransferPartitioner { - private static final Logger LOG = LoggerFactory.getLogger(BlockTransferPartitioner.class); - - /** - * It greedily partitions given transfers into sub-lists. - * - * @param transferInfos list of transfers to partition - * @param maxPartitionCount max partition count - * @return transfers partitioned into sub-lists - */ - public List> partitionTransfers(List transferInfos, - int maxPartitionCount) { - // Bucketing is possible if source or destination has exact location. - // Those allocated locations will be bucket key[s]. - TransferPartitionKey key = findTransferBucketKey(transferInfos); - // Can't bucketize transfers. - if (key == TransferPartitionKey.NONE) { - LOG.debug("Un-optimizable transfer list encountered."); - return new ArrayList>() { - { - add(transferInfos); - } - }; - } - - Map> transferBuckets = new HashMap<>(); - for (BlockTransferInfo transferInfo : transferInfos) { - BlockStoreLocation keyLoc; - switch (key) { - case SRC: - keyLoc = transferInfo.getSrcLocation(); - break; - case DST: - keyLoc = transferInfo.getDstLocation(); - break; - default: - throw new IllegalStateException( - String.format("Unsupported key type for bucketing transfer infos: %s", key.name())); - } - - if (!transferBuckets.containsKey(keyLoc)) { - transferBuckets.put(keyLoc, new LinkedList<>()); - } - - transferBuckets.get(keyLoc).add(transferInfo); - } - - List> balancedPartitions = balancePartitions( - transferBuckets.values().stream().collect(Collectors.toList()), maxPartitionCount); - - // Log partition details. - if (LOG.isDebugEnabled()) { - StringBuilder partitionDbgStr = new StringBuilder(); - partitionDbgStr - .append(String.format("Bucketed %d transfers into %d partitions using key:%s.%n", - transferInfos.size(), balancedPartitions.size(), key.name())); - // List each partition content. - for (int i = 0; i < balancedPartitions.size(); i++) { - partitionDbgStr.append(String.format("Partition-%d:%n ->%s%n", i, balancedPartitions.get(i) - .stream().map(Objects::toString).collect(Collectors.joining("\n ->")))); - } - LOG.debug("{}", partitionDbgStr); - } - return balancedPartitions; - } - - /** - * Used to balance partitions into given bucket count. It greedily tries to achieve each bucket - * having close count of tasks. - */ - private List> balancePartitions( - List> transferPartitions, int partitionLimit) { - // Return as is if less than requested bucket count. - if (transferPartitions.size() <= partitionLimit) { - return transferPartitions; - } - - // TODO(ggezer): Support partitioning that considers block sizes. - // Greedily build a balanced partitions by transfer count. - Collections.sort(transferPartitions, Comparator.comparingInt(List::size)); - - // Initialize balanced partitions. - List> balancedPartitions = new ArrayList<>(partitionLimit); - for (int i = 0; i < partitionLimit; i++) { - balancedPartitions.add(new LinkedList<>()); - } - // Greedily place transfer partitions into balanced partitions. - for (List transferPartition : transferPartitions) { - // Find the balanced partition with the least element size. - int selectedPartitionIdx = Integer.MAX_VALUE; - int selectedPartitionCount = Integer.MAX_VALUE; - for (int i = 0; i < partitionLimit; i++) { - if (balancedPartitions.get(i).size() < selectedPartitionCount) { - selectedPartitionIdx = i; - selectedPartitionCount = balancedPartitions.get(i).size(); - } - } - balancedPartitions.get(selectedPartitionIdx).addAll(transferPartition); - } - - return balancedPartitions; - } - - /** - * Used to determine right partitioning key by inspecting list of transfers. - */ - private TransferPartitionKey findTransferBucketKey(List transferInfos) { - // How many src/dst locations are fully identified. - int srcAllocatedCount = 0; - int dstAllocatedCount = 0; - // How many unique src/dst locations are seen. - Set srcLocations = new HashSet<>(); - Set dstLocations = new HashSet<>(); - // Iterate and process all transfers. - for (BlockTransferInfo transferInfo : transferInfos) { - if (!transferInfo.getSrcLocation().isAnyDir()) { - srcAllocatedCount++; - } - if (!transferInfo.getDstLocation().isAnyDir()) { - dstAllocatedCount++; - } - srcLocations.add(transferInfo.getSrcLocation()); - dstLocations.add(transferInfo.getDstLocation()); - } - - // The case for when optimization is not possible. - if (srcAllocatedCount == 0 && dstAllocatedCount == 0) { // No allocated location. - // Partitioning not possible. (This is not expected). - return TransferPartitionKey.NONE; - } - - // Choose the key by masses. - if (srcAllocatedCount > dstAllocatedCount) { - return TransferPartitionKey.SRC; - } else if (dstAllocatedCount > srcAllocatedCount) { - return TransferPartitionKey.DST; - } else { // It's a match. Choose the key by distinction. - // Partition based on the location that has more distinct sub-locations. - // This will later be capped by configured parallelism. - if (srcLocations.size() >= dstLocations.size()) { - return TransferPartitionKey.SRC; - } else { - return TransferPartitionKey.DST; - } - } - } - - /** - * Used to specify how transfers are grouped. - */ - private enum TransferPartitionKey { - SRC, DST, NONE - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/DefaultStoreLoadTracker.java b/core/server/worker/src/main/java/alluxio/worker/block/management/DefaultStoreLoadTracker.java deleted file mode 100644 index 03f667d39525..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/DefaultStoreLoadTracker.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -import alluxio.collections.ConcurrentHashSet; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.ThreadFactoryUtils; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.io.BlockClient; -import alluxio.worker.block.io.BlockClientListener; -import alluxio.worker.block.io.BlockStreamTracker; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * Default {@link StoreLoadTracker} that reports load based on open reader/writer streams to the - * local block store. - * - * TODO(ggezer): Add a safety net against close calls not being called. - */ -public class DefaultStoreLoadTracker implements StoreLoadTracker, BlockClientListener { - private static final Logger LOG = LoggerFactory.getLogger(DefaultStoreLoadTracker.class); - - /** Used to keep reference to stream readers/writers per location. */ - private final ConcurrentHashMap> mBlockClientsPerLocation; - /** Used for delayed removing of streams in order to emulate activity cool-down. */ - private final ScheduledExecutorService mScheduler; - /** For how long, an activity will remain active on load state. */ - private final long mLoadDetectionCoolDownMs; - - /** - * Creates the default load tracker instance. - */ - public DefaultStoreLoadTracker() { - mBlockClientsPerLocation = new ConcurrentHashMap<>(); - mScheduler = Executors - .newSingleThreadScheduledExecutor(ThreadFactoryUtils.build("load-tracker-thread-%d", true)); - mLoadDetectionCoolDownMs = - Configuration.getMs(PropertyKey.WORKER_MANAGEMENT_LOAD_DETECTION_COOL_DOWN_TIME); - - // BlockStreamTracker provides stream reader/writer events. - BlockStreamTracker.registerListener(this); - } - - @Override - public boolean loadDetected(BlockStoreLocation... locations) { - for (BlockStoreLocation location : locations) { - for (BlockStoreLocation trackedLocation : mBlockClientsPerLocation.keySet()) { - if (trackedLocation.belongsTo(location)) { - Set clientsPerLocation = mBlockClientsPerLocation.get(trackedLocation); - if (clientsPerLocation != null && clientsPerLocation.size() > 0) { - return true; - } - } - } - } - return false; - } - - @Override - public void clientOpened(BlockClient blockClient, BlockStoreLocation location) { - LOG.debug("BlockClient: {} opened at {}", blockClient, location); - Preconditions.checkState(locationValid(location)); - mBlockClientsPerLocation.compute(location, (k, streamSet) -> { - if (streamSet == null) { - streamSet = new ConcurrentHashSet<>(); - } - streamSet.add(blockClient); - return streamSet; - }); - } - - @Override - public void clientClosed(BlockClient blockClient, BlockStoreLocation location) { - LOG.debug("BlockClient: {} closed at {}", blockClient, location); - Preconditions.checkState(locationValid(location)); - mScheduler.schedule(() -> { - mBlockClientsPerLocation.compute(location, (k, streamSet) -> { - Preconditions.checkState(streamSet != null && !streamSet.isEmpty(), - "Unexpected load tracker state"); - streamSet.remove(blockClient); - return streamSet; - }); - }, mLoadDetectionCoolDownMs, TimeUnit.MILLISECONDS); - } - - /** - * Stream reader/writer locations are expected to be precise. - * - * @param location the location to check - * @return {@code true} if location is valid - */ - private static boolean locationValid(BlockStoreLocation location) { - return !location.isAnyTier() - && !location.isAnyMedium() - && !location.isAnyDir(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/ManagementTaskCoordinator.java b/core/server/worker/src/main/java/alluxio/worker/block/management/ManagementTaskCoordinator.java deleted file mode 100644 index d22903ea4e42..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/ManagementTaskCoordinator.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.ThreadFactoryUtils; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.LocalBlockStore; -import alluxio.worker.block.management.tier.TierManagementTaskProvider; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Supplier; - -/** - * Coordinator for instantiating and running various block management tasks. - */ -public class ManagementTaskCoordinator implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(ManagementTaskCoordinator.class); - /** Duration to sleep when a) load detected on worker. b) no work to do. */ - private static final long LOAD_DETECTION_COOL_DOWN_TIME = - Configuration.getMs(PropertyKey.WORKER_MANAGEMENT_LOAD_DETECTION_COOL_DOWN_TIME); - /** The back-off strategy. */ - private static final BackoffStrategy BACKOFF_STRATEGY = Configuration - .getEnum(PropertyKey.WORKER_MANAGEMENT_BACKOFF_STRATEGY, BackoffStrategy.class); - - /** Runner thread for launching management tasks. */ - private final Thread mRunnerThread; - /** Executor that will run the management tasks. */ - private final ExecutorService mTaskExecutor = Executors.newFixedThreadPool( - Configuration.getInt(PropertyKey.WORKER_MANAGEMENT_TASK_THREAD_COUNT), - ThreadFactoryUtils.build("block-management-task-%d", true)); - - private final LocalBlockStore mBlockStore; - private final BlockMetadataManager mMetadataManager; - private final StoreLoadTracker mLoadTracker; - - /** This coordinator requires calculating eviction view per each task. */ - private final Supplier mEvictionViewSupplier; - - /** List of management task providers. */ - private List mTaskProviders; - - /** - * Creates management coordinator. - * - * @param blockStore block store - * @param metadataManager meta manager - * @param loadTracker load tracker - * @param evictionViewSupplier eviction view supplier - */ - public ManagementTaskCoordinator(LocalBlockStore blockStore, BlockMetadataManager metadataManager, - StoreLoadTracker loadTracker, Supplier evictionViewSupplier) { - mBlockStore = blockStore; - mMetadataManager = metadataManager; - mLoadTracker = loadTracker; - mEvictionViewSupplier = evictionViewSupplier; - initializeTaskProviders(); - // Initialize runner thread. - mRunnerThread = new Thread(this::runManagement, "block-management-runner"); - mRunnerThread.setDaemon(true); - } - - /** - * Starts the coordinator. - */ - public void start() { - // Start runner thread. - mRunnerThread.start(); - } - - @Override - public void close() throws IOException { - try { - // Shutdown task executor. - mTaskExecutor.shutdownNow(); - // Interrupt and wait for runner thread. - mRunnerThread.interrupt(); - mRunnerThread.join(); - } catch (Exception e) { - throw new IOException("Failed to close management task coordinator", e); - } - } - - /** - * Register known task providers by priority order. - * - * TODO(ggezer): Re-implement async-cache as {@link BlockManagementTask}. - */ - private void initializeTaskProviders() { - mTaskProviders = new ArrayList<>(1); - if (Configuration.isSet(PropertyKey.WORKER_EVICTOR_CLASS)) { - LOG.warn("Tier management tasks will be disabled under eviction emulation mode."); - } else { - // TODO(ggezer): Improve on views per task type. - mTaskProviders.add(new TierManagementTaskProvider(mBlockStore, mMetadataManager, - mEvictionViewSupplier, mLoadTracker, mTaskExecutor)); - } - } - - /** - * @return the next management task to run, {@code null} if none pending - */ - private BlockManagementTask getNextTask() { - /* - * Order of providers in the registered list imposes an implicit priority of tasks. - * As long as a provider gives a task, providers next to it won't be consulted. - */ - for (ManagementTaskProvider taskProvider : mTaskProviders) { - BlockManagementTask task = taskProvider.getTask(); - if (task != null) { - return task; - } - } - // No task provided. - return null; - } - - /** - * Main management loop. - */ - private void runManagement() { - while (true) { - if (Thread.interrupted()) { - // Coordinator closed. - LOG.debug("Coordinator interrupted."); - break; - } - - BlockManagementTask currentTask; - try { - // Back off from worker if configured so. - if (BACKOFF_STRATEGY == BackoffStrategy.ANY - && mLoadTracker.loadDetected(BlockStoreLocation.anyTier())) { - LOG.debug("Load detected. Sleeping {}ms.", LOAD_DETECTION_COOL_DOWN_TIME); - Thread.sleep(LOAD_DETECTION_COOL_DOWN_TIME); - continue; - } - - final BlockManagementTask nextTask = getNextTask(); - if (nextTask == null) { - LOG.debug("No management task pending. Sleeping {}ms.", LOAD_DETECTION_COOL_DOWN_TIME); - Thread.sleep(LOAD_DETECTION_COOL_DOWN_TIME); - continue; - } - - // Submit and wait for the task. - currentTask = nextTask; - LOG.debug("Running task of type:{}", currentTask.getClass().getSimpleName()); - // Run the current task on coordinator thread. - try { - BlockManagementTaskResult result = currentTask.run(); - LOG.info("{} finished with result: {}", currentTask.getClass().getSimpleName(), result); - - if (result.noProgress()) { - LOG.debug("Task made no progress due to failures/back-offs. Sleeping {}ms", - LOAD_DETECTION_COOL_DOWN_TIME); - Thread.sleep(LOAD_DETECTION_COOL_DOWN_TIME); - } - } catch (Exception e) { - LOG.error("Management task failed: {}. Error: ", currentTask.getClass().getSimpleName(), - e); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } catch (Throwable t) { - LOG.error("Unexpected error during block management: ", t); - } - } - LOG.debug("Block management coordinator exited."); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/ManagementTaskProvider.java b/core/server/worker/src/main/java/alluxio/worker/block/management/ManagementTaskProvider.java deleted file mode 100644 index 11d93f16a488..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/ManagementTaskProvider.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -/** - * Used as factory interface of management tasks for particular management functions. - */ -public interface ManagementTaskProvider { - /** - * @return the next management task from this provider - */ - BlockManagementTask getTask(); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/StoreLoadTracker.java b/core/server/worker/src/main/java/alluxio/worker/block/management/StoreLoadTracker.java deleted file mode 100644 index d35b5655680d..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/StoreLoadTracker.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management; - -import alluxio.worker.block.BlockStoreLocation; - -/** - * Interface for detecting user activity on worker locations. - */ -public interface StoreLoadTracker { - /** - * Used to detect load on particular location. - * - * @param locations vararg of locations - * @return {@code true} if load detected on any given location - */ - boolean loadDetected(BlockStoreLocation... locations); -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/tier/AlignTask.java b/core/server/worker/src/main/java/alluxio/worker/block/management/tier/AlignTask.java deleted file mode 100644 index 5323f5f38852..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/tier/AlignTask.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.LocalBlockStore; -import alluxio.worker.block.annotator.BlockOrder; -import alluxio.worker.block.evictor.BlockTransferInfo; -import alluxio.worker.block.management.AbstractBlockManagementTask; -import alluxio.worker.block.management.BlockManagementTaskResult; -import alluxio.worker.block.management.BlockOperationResult; -import alluxio.worker.block.management.BlockOperationType; -import alluxio.worker.block.management.ManagementTaskCoordinator; -import alluxio.worker.block.management.StoreLoadTracker; -import alluxio.worker.block.meta.BlockMeta; - -import com.google.common.base.Function; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -/** - * A BlockStore management task that swaps blocks between tiers in order to align - * tiers based on block annotation policy. - * - * A single task may not be enough to completely align tiers, so - * {@link ManagementTaskCoordinator} will keep instantiating new tasks - * until no longer needed. - */ -public class AlignTask extends AbstractBlockManagementTask { - private static final Logger LOG = LoggerFactory.getLogger(AlignTask.class); - - /** - * Creates a new align task. - * - * @param blockStore the block store - * @param metadataManager the meta manager - * @param evictorView the evictor view - * @param loadTracker the load tracker - * @param executor the executor - */ - public AlignTask(LocalBlockStore blockStore, BlockMetadataManager metadataManager, - BlockMetadataEvictorView evictorView, StoreLoadTracker loadTracker, - ExecutorService executor) { - super(blockStore, metadataManager, evictorView, loadTracker, executor); - } - - @Override - public BlockManagementTaskResult run() { - LOG.debug("Running align task."); - // Acquire align range from the configuration. - // This will limit swap operations in a single run. - final int alignRange = - Configuration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_RANGE); - - BlockManagementTaskResult result = new BlockManagementTaskResult(); - // Align each tier intersection by swapping blocks. - for (Pair intersection : mMetadataManager - .getStorageTierAssoc().intersectionList()) { - BlockStoreLocation tierUpLoc = intersection.getFirst(); - BlockStoreLocation tierDownLoc = intersection.getSecond(); - - // Get list per tier that will be swapped for aligning the intersection. - Pair, List> swapLists = mMetadataManager.getBlockIterator().getSwaps( - tierUpLoc, BlockOrder.NATURAL, tierDownLoc, BlockOrder.REVERSE, alignRange, - BlockOrder.REVERSE, (blockId) -> !mEvictorView.isBlockEvictable(blockId)); - - Preconditions.checkArgument(swapLists.getFirst().size() == swapLists.getSecond().size()); - LOG.debug("Acquired {} block pairs to align tiers {} - {}", swapLists.getFirst().size(), - tierUpLoc.tierAlias(), tierDownLoc.tierAlias()); - - // Create exception handler to trigger swap-restore task when swap fails - // due to insufficient reserved space. - Consumer excHandler = (e) -> { - if (e instanceof ResourceExhaustedRuntimeException) { - LOG.warn("Insufficient space for worker swap space, swap restore task called."); - // Mark the need for running swap-space restoration task. - TierManagementTaskProvider.setSwapRestoreRequired(true); - } - }; - - // Execute swap transfers. - BlockOperationResult tierResult = - mTransferExecutor.executeTransferList(generateSwapTransferInfos(swapLists), excHandler); - result.addOpResults(BlockOperationType.ALIGN_SWAP, tierResult); - } - return result; - } - - private List generateSwapTransferInfos( - Pair, List> swapLists) { - if (LOG.isDebugEnabled()) { - LOG.debug( - "Generating transfer infos from swap lists.\n" - + "Source list of size:{} : {}\nDestination list of size:{} : {}", - swapLists.getFirst().size(), - swapLists.getFirst().stream().map(Object::toString).collect(Collectors.joining(",")), - swapLists.getSecond().size(), - swapLists.getSecond().stream().map(Object::toString).collect(Collectors.joining(","))); - } - - // Function that is used to map blockId to pair. - Function> blockToPairFunc = (blockId) -> { - Optional blockMeta = mEvictorView.getBlockMeta(blockId); - if (blockMeta.isPresent()) { - return new Pair(blockId, blockMeta.get().getBlockLocation()); - } else { - LOG.warn("Failed to find location of a block:{}.", blockId); - return new Pair(blockId, BlockStoreLocation.anyTier()); - } - }; - - // Generate an augmented block lists with locations. - List> blockLocPairListSrc = - swapLists.getFirst().stream().map(blockToPairFunc).collect(Collectors.toList()); - List> blockLocPairListDst = - swapLists.getSecond().stream().map(blockToPairFunc).collect(Collectors.toList()); - - // Sort augmented lists by location. - // This will help to generate the buckets by a linear sweep. - Comparator> comparator = (o1, o2) -> { - BlockStoreLocation loc1 = o1.getSecond(); - BlockStoreLocation loc2 = o2.getSecond(); - int tierComp = loc1.tierAlias().compareTo(loc2.tierAlias()); - if (tierComp != 0) { - return tierComp; - } else { - return loc1.dir() - loc2.dir(); - } - }; - Collections.sort(blockLocPairListSrc, comparator); - Collections.sort(blockLocPairListDst, comparator); - - if (LOG.isDebugEnabled()) { - LOG.debug( - "Generated and sorted augmented swap lists.\n" - + "Source list of size:{} :\n ->{}\nDestination list of size:{} :\n ->{}", - blockLocPairListSrc.size(), - blockLocPairListSrc.stream().map(Object::toString).collect(Collectors.joining("\n ->")), - blockLocPairListDst.size(), - blockLocPairListDst.stream().map(Object::toString).collect(Collectors.joining("\n ->"))); - } - - // Build transfer infos using the sorted locations. - List transferInfos = new ArrayList<>(blockLocPairListSrc.size()); - for (int i = 0; i < blockLocPairListSrc.size(); i++) { - transferInfos.add(BlockTransferInfo.createSwap(blockLocPairListSrc.get(i).getSecond(), - blockLocPairListSrc.get(i).getFirst(), blockLocPairListDst.get(i).getSecond(), - blockLocPairListDst.get(i).getFirst())); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Generated {} swap transfers: \n ->{}", transferInfos.size(), - transferInfos.stream().map(Object::toString).collect(Collectors.joining(",\n ->"))); - } - return transferInfos; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/tier/PromoteTask.java b/core/server/worker/src/main/java/alluxio/worker/block/management/tier/PromoteTask.java deleted file mode 100644 index 9b2923a23c34..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/tier/PromoteTask.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.LocalBlockStore; -import alluxio.worker.block.annotator.BlockOrder; -import alluxio.worker.block.evictor.BlockTransferInfo; -import alluxio.worker.block.management.AbstractBlockManagementTask; -import alluxio.worker.block.management.BlockManagementTaskResult; -import alluxio.worker.block.management.BlockOperationResult; -import alluxio.worker.block.management.BlockOperationType; -import alluxio.worker.block.management.ManagementTaskCoordinator; -import alluxio.worker.block.management.StoreLoadTracker; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.StorageTier; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.stream.Collectors; - -/** - * A BlockStore management task that is to move blocks to higher tiers. - * This is to ensure higher tiers are utilized to increase overall performance. - * - * A single task may not be enough to complete promotion, so {@link ManagementTaskCoordinator} - * will keep instantiating new promote tasks until no longer needed. - */ -public class PromoteTask extends AbstractBlockManagementTask { - private static final Logger LOG = LoggerFactory.getLogger(PromoteTask.class); - - /** - * Creates a new promote task. - * - * @param blockStore the block store - * @param metadataManager the meta manager - * @param evictorView the evictor view - * @param loadTracker the load tracker - * @param executor the executor - */ - public PromoteTask(LocalBlockStore blockStore, BlockMetadataManager metadataManager, - BlockMetadataEvictorView evictorView, StoreLoadTracker loadTracker, - ExecutorService executor) { - super(blockStore, metadataManager, evictorView, loadTracker, executor); - } - - @Override - public BlockManagementTaskResult run() { - LOG.debug("Running promote task."); - BlockManagementTaskResult result = new BlockManagementTaskResult(); - // Iterate each tier intersection and move to upper tier whenever required. - for (Pair intersection : mMetadataManager - .getStorageTierAssoc().intersectionList()) { - BlockStoreLocation tierUpLoc = intersection.getFirst(); - BlockStoreLocation tierDownLoc = intersection.getSecond(); - - // Acquire iterator for the tier below. - Iterator tierDownIterator = - mMetadataManager.getBlockIterator().getIterator(tierDownLoc, BlockOrder.REVERSE); - - // Acquire and execute promotion transfers. - BlockOperationResult tierResult = mTransferExecutor.executeTransferList( - getTransferInfos(tierDownIterator, tierUpLoc, tierDownLoc)); - - result.addOpResults(BlockOperationType.PROMOTE_MOVE, tierResult); - } - return result; - } - - /** - * @return list of block transfers - */ - private List getTransferInfos(Iterator iterator, - BlockStoreLocation tierUpLocation, BlockStoreLocation tierDownLocation) { - // Acquire promotion range from the configuration. - // This will limit promotions in single task run. - final int promoteRange = - Configuration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_RANGE); - - // Tier for where promotions are going to. - StorageTier tierUp = mMetadataManager.getTier(tierUpLocation.tierAlias()); - // Get quota for promotions. - int promotionQuota = - Configuration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_QUOTA_PERCENT); - Preconditions.checkArgument(promotionQuota >= 0 && promotionQuota <= 100, - "Invalid promotion quota percent"); - double quotaRatio = (double) promotionQuota / 100; - - // List to store transfer infos for selected blocks. - List transferInfos = new LinkedList<>(); - // Projected allocation for selected blocks. - long bytesToAllocate = 0; - // Gather blocks from iterator upto configured free space limit. - while (iterator.hasNext() && transferInfos.size() < promoteRange) { - // Stop moving if reached promotion quota on higher tier. - double projectedUsedRatio = 1.0 - - ((double) (tierUp.getAvailableBytes() - bytesToAllocate) / tierUp.getCapacityBytes()); - if (projectedUsedRatio >= quotaRatio) { - break; - } - - long blockId = iterator.next(); - // Read block info and store it. - Optional blockMeta = mEvictorView.getBlockMeta(blockId); - if (!blockMeta.isPresent()) { - LOG.debug("Block:{} exist but not available for promotion.", blockId); - continue; - } - bytesToAllocate += blockMeta.get().getBlockSize(); - transferInfos.add(BlockTransferInfo.createMove(blockMeta.get().getBlockLocation(), - blockId, tierUpLocation)); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Generated {} promotions from {} to {}.\n" + "Promotions transfers:\n ->{}", - transferInfos.size(), tierDownLocation.tierAlias(), tierUpLocation.tierAlias(), - transferInfos.stream().map(Objects::toString).collect(Collectors.joining("\n ->"))); - } - return transferInfos; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/tier/SwapRestoreTask.java b/core/server/worker/src/main/java/alluxio/worker/block/management/tier/SwapRestoreTask.java deleted file mode 100644 index 84dc701d3ea6..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/tier/SwapRestoreTask.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.Sessions; -import alluxio.StorageTierAssoc; -import alluxio.collections.Pair; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.LocalBlockStore; -import alluxio.worker.block.annotator.BlockOrder; -import alluxio.worker.block.evictor.BlockTransferInfo; -import alluxio.worker.block.management.AbstractBlockManagementTask; -import alluxio.worker.block.management.BlockManagementTaskResult; -import alluxio.worker.block.management.BlockOperationResult; -import alluxio.worker.block.management.BlockOperationType; -import alluxio.worker.block.management.StoreLoadTracker; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.StorageDirEvictorView; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTierView; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.stream.Collectors; - -/** - * Block management task that is used to free up space on reserved spaces of each directory. - * It's required to guarantee termination of {@link AlignTask}s under variable block size. - * - * It essentially does a full cascading evict followed by balancing of each directory. - */ -public class SwapRestoreTask extends AbstractBlockManagementTask { - private static final Logger LOG = LoggerFactory.getLogger(SwapRestoreTask.class); - - /** - * Creates a new swap-restore task. - * - * @param blockStore the block store - * @param metadataManager the meta manager - * @param evictorView the evictor view - * @param loadTracker the load tracker - * @param executor the executor - */ - public SwapRestoreTask(LocalBlockStore blockStore, BlockMetadataManager metadataManager, - BlockMetadataEvictorView evictorView, StoreLoadTracker loadTracker, - ExecutorService executor) { - super(blockStore, metadataManager, evictorView, loadTracker, executor); - } - - @Override - public BlockManagementTaskResult run() { - LOG.debug("Running swap-restore task."); - // Generate swap-restore plan. - Pair, List> swapRestorePlan = getSwapRestorePlan(); - LOG.debug("Generated swap-restore plan with {} deletions and {} transfers.", - swapRestorePlan.getFirst().size(), swapRestorePlan.getSecond().size()); - - BlockManagementTaskResult result = new BlockManagementTaskResult(); - // Execute to-be-removed blocks from the plan. - int removalFailCount = 0; - for (Long blockId : swapRestorePlan.getFirst()) { - try { - mBlockStore.removeBlock(Sessions.createInternalSessionId(), blockId); - } catch (Exception e) { - LOG.warn("Failed to remove block: {} during swap-restore task.", blockId); - removalFailCount++; - } - } - result.addOpResults(BlockOperationType.SWAP_RESTORE_REMOVE, - new BlockOperationResult(swapRestorePlan.getFirst().size(), removalFailCount, 0)); - - // Execute to-be-transferred blocks from the plan. - BlockOperationResult flushResult = - mTransferExecutor.executeTransferList(swapRestorePlan.getSecond()); - result.addOpResults(BlockOperationType.SWAP_RESTORE_FLUSH, flushResult); - - // Re-balance each tier. - BlockOperationResult balanceResult = - mTransferExecutor.executeTransferList(getBalancingTransfersList()); - result.addOpResults(BlockOperationType.SWAP_RESTORE_BALANCE, balanceResult); - - return result; - } - - /** - * @return the pair of <blocks-to-remove, blocks-to-transfer> for swap-restore plan - */ - private Pair, List> getSwapRestorePlan() { - StorageTierAssoc storageTierAssoc = mMetadataManager.getStorageTierAssoc(); - - List blocksToRemove = new LinkedList<>(); - List blocksToTransfer = new LinkedList<>(); - - long cascadingFromAbove = 0; - for (StorageTierView tierView : mEvictorView.getTierViews()) { - int currentTierOrdinal = tierView.getTierViewOrdinal(); - String tierAlias = storageTierAssoc.getAlias(currentTierOrdinal); - BlockStoreLocation destLocation = BlockStoreLocation.anyDirInTier(tierAlias); - if (currentTierOrdinal < storageTierAssoc.size() - 1) { - destLocation = - BlockStoreLocation.anyDirInTier(storageTierAssoc.getAlias(currentTierOrdinal + 1)); - } - boolean lastTier = currentTierOrdinal == storageTierAssoc.size() - 1; - - long tierReservedBytes = 0; - long tierCapacityBytes = 0; - long tierCommittedBytes = 0; - for (StorageDirView dirView : tierView.getDirViews()) { - tierReservedBytes += dirView.getReservedBytes(); - tierCapacityBytes += dirView.getCapacityBytes(); - tierCommittedBytes += dirView.getCommittedBytes(); - } - - long bytesBeyondReserve = - (tierCommittedBytes + cascadingFromAbove) - (tierCapacityBytes - tierReservedBytes); - long moveOutBytes = Math.max(0, bytesBeyondReserve); - - // Store for the next tier. - cascadingFromAbove = moveOutBytes; - - Iterator tierIterator = mMetadataManager.getBlockIterator() - .getIterator(BlockStoreLocation.anyDirInTier(tierAlias), BlockOrder.NATURAL); - while (tierIterator.hasNext() && moveOutBytes > 0) { - long blockId = tierIterator.next(); - Optional nextBlockFromTier = mEvictorView.getBlockMeta(blockId); - if (!nextBlockFromTier.isPresent()) { - LOG.debug("Block:{} exist but not available for moving.", blockId); - continue; - } - moveOutBytes -= nextBlockFromTier.get().getBlockSize(); - if (lastTier) { - blocksToRemove.add(nextBlockFromTier.get().getBlockId()); - } else { - blocksToTransfer.add(BlockTransferInfo.createMove( - nextBlockFromTier.get().getBlockLocation(), - nextBlockFromTier.get().getBlockId(), destLocation)); - } - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug( - "Generated swap-restore plan with {} deletions and {} transfers.\n" - + "Block deletions:\n ->{}\nBlock transfers:\n ->{}", - blocksToRemove.size(), blocksToTransfer.size(), - blocksToRemove.stream().map(Object::toString).collect(Collectors.joining(",\n ->")), - blocksToTransfer.stream().map(Object::toString).collect(Collectors.joining(",\n ->"))); - } - return new Pair<>(blocksToRemove, blocksToTransfer); - } - - /** - * @return the list of transfer in order to balance swap-space within tier - */ - private List getBalancingTransfersList() { - List transferInfos = new LinkedList<>(); - for (StorageTierView tierView : mEvictorView.getTierViews()) { - for (StorageDirView dirView : tierView.getDirViews()) { - Iterator dirBlockIter = mMetadataManager.getBlockIterator().getIterator( - new BlockStoreLocation(tierView.getTierViewAlias(), dirView.getDirViewIndex()), - BlockOrder.NATURAL); - while (dirBlockIter.hasNext() && dirView.getAvailableBytes() < dirView.getReservedBytes()) { - long blockId = dirBlockIter.next(); - Optional movingOutBlock = mEvictorView.getBlockMeta(blockId); - if (!movingOutBlock.isPresent()) { - LOG.debug("Block:{} exist but not available for balancing.", blockId); - continue; - } - // Find where to move the block. - StorageDirView dstDirView = null; - for (StorageDirView candidateDirView : tierView.getDirViews()) { - if (candidateDirView.getDirViewIndex() == dirView.getDirViewIndex()) { - continue; - } - if (candidateDirView.getAvailableBytes() - - candidateDirView.getReservedBytes() >= movingOutBlock.get().getBlockSize()) { - dstDirView = candidateDirView; - break; - } - } - if (dstDirView == null) { - LOG.warn("Could not balance swap-restore space for location: {}", - dirView.toBlockStoreLocation()); - break; - } - - // TODO(ggezer): Consider allowing evictions for this move. - transferInfos.add(BlockTransferInfo.createMove(movingOutBlock.get().getBlockLocation(), - blockId, dstDirView.toBlockStoreLocation())); - - // Account for moving-out blocks. - ((StorageDirEvictorView) dirView).markBlockMoveOut(blockId, - movingOutBlock.get().getBlockSize()); - // Account for moving-in blocks. - ((StorageDirEvictorView) dstDirView).markBlockMoveIn(blockId, - movingOutBlock.get().getBlockSize()); - } - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Generated {} balance transfers:\n ->{}", transferInfos.size(), - transferInfos.stream().map(Object::toString).collect(Collectors.joining(",\n ->"))); - } - return transferInfos; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/management/tier/TierManagementTaskProvider.java b/core/server/worker/src/main/java/alluxio/worker/block/management/tier/TierManagementTaskProvider.java deleted file mode 100644 index 5e682e8ec3d4..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/management/tier/TierManagementTaskProvider.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.LocalBlockStore; -import alluxio.worker.block.annotator.BlockOrder; -import alluxio.worker.block.management.BlockManagementTask; -import alluxio.worker.block.management.ManagementTaskProvider; -import alluxio.worker.block.management.StoreLoadTracker; -import alluxio.worker.block.meta.StorageTier; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; -import java.util.concurrent.ExecutorService; -import java.util.function.Supplier; - -/** - * {@link ManagementTaskProvider} implementation for tier management tasks. - * - * It currently creates three types of tasks: - * 1- {@link AlignTask} for aligning tiers based on user access pattern. - * 2- {@link SwapRestoreTask} for when swap task can't run due to reserved space exhaustion. - * 3- {@link PromoteTask} for utilizing speed of higher tiers by moving blocks from below. - */ -public class TierManagementTaskProvider implements ManagementTaskProvider { - private static final Logger LOG = LoggerFactory.getLogger(TierManagementTaskProvider.class); - - private final LocalBlockStore mBlockStore; - private final BlockMetadataManager mMetadataManager; - private final Supplier mEvictorViewSupplier; - private final StoreLoadTracker mLoadTracker; - private final ExecutorService mExecutor; - - /** Used to set whether swap-restore task is required. */ - private static boolean sSwapRestoreRequired = false; - - /** - * Used to set whether swap-restore task is required. - * It's used by {@link AlignTask} when a swap fails due to insufficient reserved space. - * - * @param swapRestoreRequired whether swap-restore task needs to run - */ - public static void setSwapRestoreRequired(boolean swapRestoreRequired) { - sSwapRestoreRequired = swapRestoreRequired; - } - - /** - * Creates a task provider for tier management functions. - * - * @param blockStore the block store - * @param metadataManager the meta manager - * @param evictorViewSupplier the evictor view supplier - * @param loadTracker the load tracker - * @param executor the executor - */ - public TierManagementTaskProvider(LocalBlockStore blockStore, - BlockMetadataManager metadataManager, Supplier evictorViewSupplier, - StoreLoadTracker loadTracker, ExecutorService executor) { - mBlockStore = blockStore; - mMetadataManager = metadataManager; - mEvictorViewSupplier = evictorViewSupplier; - mLoadTracker = loadTracker; - mExecutor = executor; - } - - @Override - public BlockManagementTask getTask() { - switch (findNextTask()) { - case NONE: - return null; - case ALIGN: - return new AlignTask(mBlockStore, mMetadataManager, mEvictorViewSupplier.get(), - mLoadTracker, mExecutor); - case PROMOTE: - return new PromoteTask(mBlockStore, mMetadataManager, mEvictorViewSupplier.get(), - mLoadTracker, mExecutor); - case SWAP_RESTORE: - return new SwapRestoreTask(mBlockStore, mMetadataManager, mEvictorViewSupplier.get(), - mLoadTracker, mExecutor); - default: - throw new IllegalArgumentException("Unknown task type."); - } - } - - /** - * @return the next tier management task that is required - */ - private TierManagementTaskType findNextTask() { - // Fetch the configuration for supported tasks types. - boolean alignEnabled = - Configuration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED); - boolean swapRestoreEnabled = - Configuration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_SWAP_RESTORE_ENABLED); - boolean promotionEnabled = - Configuration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED); - - // Return swap-restore task if marked. - if (swapRestoreEnabled && sSwapRestoreRequired) { - setSwapRestoreRequired(false); - LOG.debug("Swap-restore needed."); - return TierManagementTaskType.SWAP_RESTORE; - } - - // Acquire a recent evictor view. - BlockMetadataEvictorView evictorView = mEvictorViewSupplier.get(); - - // Iterate all tier intersections and decide which task to run. - for (Pair intersection : mMetadataManager - .getStorageTierAssoc().intersectionList()) { - // Check if the intersection needs alignment. - if (alignEnabled && !mMetadataManager.getBlockIterator().aligned(intersection.getFirst(), - intersection.getSecond(), BlockOrder.NATURAL, - (blockId) -> !evictorView.isBlockEvictable(blockId))) { - LOG.debug("Alignment needed between: {} - {}", intersection.getFirst().tierAlias(), - intersection.getSecond().tierAlias()); - return TierManagementTaskType.ALIGN; - } - - // Check if the intersection allows for promotions. - if (promotionEnabled) { - StorageTier highTier = mMetadataManager.getTier(intersection.getFirst().tierAlias()); - // Current used percent on the high tier of intersection. - double currentUsedRatio = - 1.0 - (double) highTier.getAvailableBytes() / highTier.getCapacityBytes(); - // Configured promotion quota percent for tiers. - double quotaRatio = (double) Configuration - .getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_QUOTA_PERCENT) / 100; - // Check if the high tier allows for promotions. - if (currentUsedRatio < quotaRatio) { - // Check if there is anything to move from lower tier. - Iterator lowBlocks = mMetadataManager.getBlockIterator() - .getIterator(intersection.getSecond(), BlockOrder.REVERSE); - while (lowBlocks.hasNext()) { - if (evictorView.isBlockEvictable(lowBlocks.next())) { - LOG.debug("Promotions needed from {} to {}", intersection.getSecond().tierAlias(), - intersection.getFirst().tierAlias()); - return TierManagementTaskType.PROMOTE; - } - } - } - } - } - - // No tier management task is required/allowed. - return TierManagementTaskType.NONE; - } - - /** - * Supported tier management tasks. - */ - enum TierManagementTaskType { - NONE, - ALIGN, - PROMOTE, - SWAP_RESTORE - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultBlockMeta.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultBlockMeta.java deleted file mode 100644 index 2d98a91fd4c9..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultBlockMeta.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.util.io.PathUtils; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.base.Preconditions; - -import java.io.File; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Represents the metadata of a block in Alluxio managed storage. - */ -@ThreadSafe -public class DefaultBlockMeta implements BlockMeta { - private final long mBlockId; - private final StorageDir mDir; - private final long mBlockSize; - - /** - * Committed block is stored in BlockStore under its {@link StorageDir} as a block file named - * after its blockId. e.g. Block 100 of StorageDir "/mnt/mem/0" has path: - *

- * /mnt/mem/0/100 - * - * @param blockId the block id - * @param dir the parent directory - * @return committed file path - */ - public static String commitPath(StorageDir dir, long blockId) { - return PathUtils.concatPath(dir.getDirPath(), blockId); - } - - /** - * Creates a new instance of {@link DefaultBlockMeta}. - * - * @param blockId the block id - * @param blockSize the block size - * @param dir the parent directory - */ - public DefaultBlockMeta(long blockId, long blockSize, StorageDir dir) { - mBlockId = blockId; - mDir = Preconditions.checkNotNull(dir, "dir"); - mBlockSize = blockSize; - } - - /** - * Creates a new instance of {@link DefaultBlockMeta} from {@link DefaultTempBlockMeta}. - * - * @param tempBlock uncommitted block metadata - */ - public DefaultBlockMeta(TempBlockMeta tempBlock) { - this(tempBlock.getBlockId(), - // NOTE: TempBlockMeta must be committed after the actual data block file is moved. - new File(tempBlock.getCommitPath()).length(), - tempBlock.getParentDir()); - } - - @Override - public long getBlockId() { - return mBlockId; - } - - @Override - public BlockStoreLocation getBlockLocation() { - StorageTier tier = mDir.getParentTier(); - return new BlockStoreLocation(tier.getTierAlias(), mDir.getDirIndex(), mDir.getDirMedium()); - } - - @Override - public long getBlockSize() { - return mBlockSize; - } - - @Override - public String getPath() { - return commitPath(mDir, mBlockId); - } - - @Override - public StorageDir getParentDir() { - return mDir; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultStorageDir.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultStorageDir.java deleted file mode 100644 index 052249f085f9..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultStorageDir.java +++ /dev/null @@ -1,378 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import static com.google.common.base.Preconditions.checkState; - -import alluxio.annotation.SuppressFBWarnings; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.util.io.FileUtils; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Represents a directory in a storage tier. It has a fixed capacity allocated to it on - * instantiation. It contains the set of blocks currently in the storage directory. - * - * Portion of capacity will be accounted as reserved space. - * Through {@link StorageDirView}, this space will be reflected as: - * - committed for user I/Os - * - available for internal I/Os. - */ -@NotThreadSafe -public final class DefaultStorageDir implements StorageDir { - private static final Logger LOG = LoggerFactory.getLogger(DefaultStorageDir.class); - - private final long mCapacityBytes; - private final String mDirMedium; - /** A map from block id to block metadata. */ - private final Map mBlockIdToBlockMap = new HashMap<>(200); - /** A map from block id to temp block metadata. */ - private final Map mBlockIdToTempBlockMap = new HashMap<>(200); - /** A map from session id to the set of temp blocks created by this session. */ - private final Map> mSessionIdToTempBlockIdsMap = new HashMap<>(200); - private final AtomicLong mAvailableBytes; - private final AtomicLong mCommittedBytes; - private final AtomicLong mReservedBytes; - private final String mDirPath; - private final int mDirIndex; - private final StorageTier mTier; - - private DefaultStorageDir(StorageTier tier, int dirIndex, long capacityBytes, long reservedBytes, - String dirPath, String dirMedium) { - mTier = Preconditions.checkNotNull(tier, "tier"); - mDirIndex = dirIndex; - mCapacityBytes = capacityBytes; - mReservedBytes = new AtomicLong(reservedBytes); - mAvailableBytes = new AtomicLong(capacityBytes - reservedBytes); - mCommittedBytes = new AtomicLong(0); - mDirPath = dirPath; - mDirMedium = dirMedium; - } - - /** - * Factory method to create {@link StorageDir}. - * - * It will load metadata of existing committed blocks in the dirPath specified. Only files with - * directory depth 1 under dirPath and whose file name can be parsed into {@code long} will be - * considered as existing committed blocks, these files will be preserved, others files or - * directories will be deleted. - * - * @param tier the {@link StorageTier} this dir belongs to - * @param dirIndex the index of this dir in its tier - * @param capacityBytes the initial capacity of this dir, can not be modified later - * @param reservedBytes the amount of reserved space for internal management - * @param dirPath filesystem path of this dir for actual storage - * @param dirMedium the medium type of the storage dir - * @return the new created {@link StorageDir} - */ - public static StorageDir newStorageDir(StorageTier tier, int dirIndex, long capacityBytes, - long reservedBytes, String dirPath, String dirMedium) { - DefaultStorageDir dir = - new DefaultStorageDir(tier, dirIndex, capacityBytes, reservedBytes, dirPath, dirMedium); - dir.initializeMeta(); - LOG.info("StorageDir initialized: path={}, tier={}, dirIndex={}, medium={}, capacityBytes={}, " - + "reservedBytes={}, availableBytes={}", - dirPath, tier, dirIndex, dirMedium, capacityBytes, reservedBytes, dir.mAvailableBytes); - return dir; - } - - /** - * Initializes metadata for existing blocks in this {@link StorageDir}. - * - * Only paths satisfying the contract defined in - * {@link DefaultBlockMeta#commitPath(StorageDir, long)} are legal, should be in format like - * {dir}/{blockId}. other paths will be deleted. - */ - private void initializeMeta() { - // Create the storage directory path - boolean isDirectoryNewlyCreated = FileUtils.createStorageDirPath(mDirPath, - Configuration.getString(PropertyKey.WORKER_DATA_FOLDER_PERMISSIONS)); - String tmpDir = Paths.get(Configuration.getString(PropertyKey.WORKER_DATA_TMP_FOLDER)) - .getName(0).toString(); - if (isDirectoryNewlyCreated) { - LOG.info("Folder {} was created!", mDirPath); - } - - File dir = new File(mDirPath); - File[] paths = dir.listFiles(); - if (paths == null) { - return; - } - for (File path : paths) { - if (!path.isFile()) { - if (!path.getName().equals(tmpDir)) { - LOG.error("{} in StorageDir is not a file", path.getAbsolutePath()); - } - try { - // TODO(calvin): Resolve this conflict in class names. - org.apache.commons.io.FileUtils.deleteDirectory(path); - } catch (IOException e) { - LOG.error("can not delete directory {}", path.getAbsolutePath(), e); - } - } else { - try { - long blockId = Long.parseLong(path.getName()); - addBlockMeta(new DefaultBlockMeta(blockId, path.length(), this)); - } catch (NumberFormatException e) { - LOG.error("filename of {} in StorageDir can not be parsed into long", - path.getAbsolutePath(), e); - if (path.delete()) { - LOG.warn("file {} has been deleted", path.getAbsolutePath()); - } else { - LOG.error("can not delete file {}", path.getAbsolutePath()); - } - } - } - } - } - - @Override - public long getCapacityBytes() { - return mCapacityBytes; - } - - @Override - public long getAvailableBytes() { - return mAvailableBytes.get(); - } - - @Override - public long getCommittedBytes() { - return mCommittedBytes.get(); - } - - @Override - public String getDirPath() { - return mDirPath; - } - - @Override - public String getDirMedium() { - return mDirMedium; - } - - @Override - public StorageTier getParentTier() { - return mTier; - } - - @Override - public int getDirIndex() { - return mDirIndex; - } - - @Override - public List getBlockIds() { - return new ArrayList<>(mBlockIdToBlockMap.keySet()); - } - - @Override - public List getBlocks() { - return new ArrayList<>(mBlockIdToBlockMap.values()); - } - - @Override - public boolean hasBlockMeta(long blockId) { - return mBlockIdToBlockMap.containsKey(blockId); - } - - @Override - public boolean hasTempBlockMeta(long blockId) { - return mBlockIdToTempBlockMap.containsKey(blockId); - } - - @Override - public Optional getBlockMeta(long blockId) { - return Optional.ofNullable(mBlockIdToBlockMap.get(blockId)); - } - - @Override - public Optional getTempBlockMeta(long blockId) { - return Optional.ofNullable(mBlockIdToTempBlockMap.get(blockId)); - } - - @Override - public void addBlockMeta(BlockMeta blockMeta) { - Preconditions.checkNotNull(blockMeta, "blockMeta"); - long blockId = blockMeta.getBlockId(); - long blockSize = blockMeta.getBlockSize(); - if (getAvailableBytes() + getReservedBytes() < blockSize) { - throw new ResourceExhaustedRuntimeException( - ExceptionMessage.NO_SPACE_FOR_BLOCK_META.getMessage(blockId, blockSize, - getAvailableBytes(), blockMeta.getBlockLocation().tierAlias()), false); - } - checkState(!hasBlockMeta(blockId), ExceptionMessage.ADD_EXISTING_BLOCK.getMessage(blockId, - blockMeta.getBlockLocation().tierAlias())); - mBlockIdToBlockMap.put(blockId, blockMeta); - reserveSpace(blockSize, true); - } - - @Override - public void addTempBlockMeta(TempBlockMeta tempBlockMeta) { - Preconditions.checkNotNull(tempBlockMeta, "tempBlockMeta"); - long sessionId = tempBlockMeta.getSessionId(); - long blockId = tempBlockMeta.getBlockId(); - long blockSize = tempBlockMeta.getBlockSize(); - if (getAvailableBytes() + getReservedBytes() < blockSize) { - throw new ResourceExhaustedRuntimeException( - ExceptionMessage.NO_SPACE_FOR_BLOCK_META.getMessage(blockId, blockSize, - getAvailableBytes(), tempBlockMeta.getBlockLocation().tierAlias()), false); - } - checkState(!hasTempBlockMeta(blockId), ExceptionMessage.ADD_EXISTING_BLOCK.getMessage(blockId, - tempBlockMeta.getBlockLocation().tierAlias())); - mBlockIdToTempBlockMap.put(blockId, tempBlockMeta); - Set sessionTempBlocks = mSessionIdToTempBlockIdsMap.get(sessionId); - if (sessionTempBlocks == null) { - mSessionIdToTempBlockIdsMap.put(sessionId, Sets.newHashSet(blockId)); - } else { - sessionTempBlocks.add(blockId); - } - reserveSpace(blockSize, false); - } - - @Override - public void removeBlockMeta(BlockMeta blockMeta) { - Preconditions.checkNotNull(blockMeta, "blockMeta"); - long blockId = blockMeta.getBlockId(); - BlockMeta deletedBlockMeta = mBlockIdToBlockMap.remove(blockId); - if (deletedBlockMeta != null) { - reclaimSpace(blockMeta.getBlockSize(), true); - } - } - - @Override - @SuppressFBWarnings("NP_NULL_ON_SOME_PATH") - public void removeTempBlockMeta(TempBlockMeta tempBlockMeta) { - Preconditions.checkNotNull(tempBlockMeta, "tempBlockMeta"); - final long blockId = tempBlockMeta.getBlockId(); - final long sessionId = tempBlockMeta.getSessionId(); - TempBlockMeta deletedTempBlockMeta = mBlockIdToTempBlockMap.remove(blockId); - checkState(deletedTempBlockMeta != null, - ExceptionMessage.BLOCK_META_NOT_FOUND.getMessage(blockId)); - Set sessionBlocks = mSessionIdToTempBlockIdsMap.get(sessionId); - checkState(sessionBlocks != null && sessionBlocks.remove(blockId), - ExceptionMessage.BLOCK_NOT_FOUND_FOR_SESSION.getMessage(blockId, mTier.getTierAlias(), - sessionId)); - if (sessionBlocks.isEmpty()) { - mSessionIdToTempBlockIdsMap.remove(sessionId); - } - reclaimSpace(tempBlockMeta.getBlockSize(), false); - } - - @Override - public void resizeTempBlockMeta(TempBlockMeta tempBlockMeta, long newSize) { - long oldSize = tempBlockMeta.getBlockSize(); - checkState(oldSize < newSize, "Shrinking block, not supported!"); - if (newSize > oldSize) { - reserveSpace(newSize - oldSize, false); - tempBlockMeta.setBlockSize(newSize); - } - } - - @Override - public void cleanupSessionTempBlocks(long sessionId, List tempBlockIds) { - Set sessionTempBlocks = mSessionIdToTempBlockIdsMap.get(sessionId); - // The session's temporary blocks have already been removed. - if (sessionTempBlocks == null) { - return; - } - for (Long tempBlockId : tempBlockIds) { - if (!mBlockIdToTempBlockMap.containsKey(tempBlockId)) { - // This temp block does not exist in this dir, this is expected for some blocks since the - // input list is across all dirs - continue; - } - sessionTempBlocks.remove(tempBlockId); - TempBlockMeta tempBlockMeta = mBlockIdToTempBlockMap.remove(tempBlockId); - if (tempBlockMeta != null) { - reclaimSpace(tempBlockMeta.getBlockSize(), false); - } else { - LOG.error("Cannot find blockId {} when cleanup sessionId {}", tempBlockId, sessionId); - } - } - if (sessionTempBlocks.isEmpty()) { - mSessionIdToTempBlockIdsMap.remove(sessionId); - } else { - // This may happen if the client comes back during clean up and creates more blocks or some - // temporary blocks failed to be deleted - LOG.warn("Blocks still owned by session {} after cleanup.", sessionId); - } - } - - @Override - public List getSessionTempBlocks(long sessionId) { - Set sessionTempBlockIds = mSessionIdToTempBlockIdsMap.get(sessionId); - - if (sessionTempBlockIds == null || sessionTempBlockIds.isEmpty()) { - return Collections.emptyList(); - } - List sessionTempBlocks = new ArrayList<>(); - for (long blockId : sessionTempBlockIds) { - sessionTempBlocks.add(mBlockIdToTempBlockMap.get(blockId)); - } - return sessionTempBlocks; - } - - @Override - public BlockStoreLocation toBlockStoreLocation() { - return new BlockStoreLocation(mTier.getTierAlias(), mDirIndex, mDirMedium); - } - - @Override - public long getReservedBytes() { - return mReservedBytes.get(); - } - - private void reclaimSpace(long size, boolean committed) { - mAvailableBytes.getAndUpdate(oldAvailableBytes -> { - long newAvailableBytes = oldAvailableBytes + size; - checkState(mCapacityBytes >= newAvailableBytes, - "Available bytes should always be less than total capacity bytes"); - return newAvailableBytes; - }); - if (committed) { - mCommittedBytes.addAndGet(-size); - } - } - - private void reserveSpace(long size, boolean committed) { - mAvailableBytes.getAndUpdate(oldAvailableBytes -> { - checkState(size <= oldAvailableBytes + getReservedBytes(), - "Available bytes should always be non-negative"); - return oldAvailableBytes - size; - }); - if (committed) { - mCommittedBytes.addAndGet(size); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultStorageTier.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultStorageTier.java deleted file mode 100644 index 32723dee8696..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultStorageTier.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.InvalidPathException; -import alluxio.util.CommonUtils; -import alluxio.util.FormatUtils; -import alluxio.util.OSUtils; -import alluxio.util.ShellUtils; -import alluxio.util.UnixMountInfo; -import alluxio.util.io.FileUtils; -import alluxio.util.io.PathUtils; - -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Represents a tier of storage, for example memory or SSD. It serves as a container of - * {@link StorageDir} which actually contains metadata information about blocks stored and space - * used/available. - */ -@NotThreadSafe -public final class DefaultStorageTier implements StorageTier { - private static final Logger LOG = LoggerFactory.getLogger(DefaultStorageTier.class); - - /** Alias value of this tier in tiered storage. */ - private final String mTierAlias; - /** Ordinal value of this tier in tiered storage, the highest level is 0. */ - private final int mTierOrdinal; - private final HashMap mDirs; - /** The lost storage paths that are failed to initialize or lost. */ - private final List mLostStorage; - - private DefaultStorageTier(String tierAlias, int tierOrdinal) { - mTierAlias = tierAlias; - mTierOrdinal = tierOrdinal; - mDirs = new HashMap<>(); - mLostStorage = new ArrayList<>(); - } - - private void initStorageTier(boolean isMultiTier) { - String tmpDir = Configuration.getString(PropertyKey.WORKER_DATA_TMP_FOLDER); - PropertyKey tierDirPathConf = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(mTierOrdinal); - List dirPaths = Configuration.getList(tierDirPathConf).stream() - .map(path -> CommonUtils.getWorkerDataDirectory(path, Configuration.global())) - .collect(ImmutableList.toImmutableList()); - - PropertyKey tierDirCapacityConf = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA.format(mTierOrdinal); - List dirQuotas = Configuration.getList(tierDirCapacityConf); - Preconditions.checkState(dirQuotas.size() > 0, - "Tier capacity configuration should not be blank"); - - PropertyKey tierDirMediumConf = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_MEDIUMTYPE.format(mTierOrdinal); - List dirMedium = Configuration.getList(tierDirMediumConf); - Preconditions.checkState(dirMedium.size() > 0, - "Tier medium type configuration should not be blank"); - - // Set reserved bytes on directories if tier aligning is enabled. - long reservedBytes = 0; - if (isMultiTier - && Configuration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED)) { - reservedBytes = - Configuration.getBytes(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_RESERVED_BYTES); - } - - for (int i = 0; i < dirPaths.size(); i++) { - int index = i >= dirQuotas.size() ? dirQuotas.size() - 1 : i; - int mediumTypeindex = i >= dirMedium.size() ? dirMedium.size() - 1 : i; - long capacity = FormatUtils.parseSpaceSize(dirQuotas.get(index)); - try { - StorageDir dir = DefaultStorageDir.newStorageDir(this, i, capacity, reservedBytes, - dirPaths.get(i), dirMedium.get(mediumTypeindex)); - mDirs.put(i, dir); - } catch (Exception e) { - LOG.error("Unable to initialize storage directory at {}", dirPaths.get(i), e); - mLostStorage.add(dirPaths.get(i)); - continue; - } - - // Delete tmp directory. - String tmpDirPath = PathUtils.concatPath(dirPaths.get(i), tmpDir); - try { - FileUtils.deletePathRecursively(tmpDirPath); - } catch (IOException e) { - if (FileUtils.exists(tmpDirPath)) { - LOG.error("Failed to clean up temporary directory: {}.", tmpDirPath); - } - } - } - if (mTierAlias.equals(Constants.MEDIUM_MEM) && mDirs.size() == 1) { - checkEnoughMemSpace(mDirs.values().iterator().next()); - } - } - - /** - * Checks that a tmpfs/ramfs backing the storage directory has enough capacity. If the storage - * directory is not backed by tmpfs/ramfs or the size of the tmpfs/ramfs cannot be determined, a - * warning is logged but no exception is thrown. - * - * @param storageDir the storage dir to check - * @throws IllegalStateException if the tmpfs/ramfs is smaller than the configured memory size - */ - private void checkEnoughMemSpace(StorageDir storageDir) { - if (!OSUtils.isLinux()) { - return; - } - List info; - try { - info = ShellUtils.getUnixMountInfo(); - } catch (IOException e) { - LOG.warn("Failed to get mount information for verifying memory capacity: {}", e.toString()); - return; - } - boolean foundMountInfo = false; - for (UnixMountInfo mountInfo : info) { - Optional mountPointOption = mountInfo.getMountPoint(); - Optional fsTypeOption = mountInfo.getFsType(); - Optional sizeOption = mountInfo.getOptions().getSize(); - if (!mountPointOption.isPresent() || !fsTypeOption.isPresent() || !sizeOption.isPresent()) { - continue; - } - String mountPoint = mountPointOption.get(); - String fsType = fsTypeOption.get(); - long size = sizeOption.get(); - try { - // getDirPath gives something like "/mnt/tmpfs/alluxioworker". - String rootStoragePath = PathUtils.getParent(storageDir.getDirPath()); - if (!PathUtils.cleanPath(mountPoint).equals(rootStoragePath)) { - continue; - } - } catch (InvalidPathException e) { - continue; - } - foundMountInfo = true; - if (fsType.equalsIgnoreCase("tmpfs") && size < storageDir.getCapacityBytes()) { - throw new IllegalStateException(String.format( - "%s is smaller than the configured size: %s size: %s, configured size: %s", - fsType, fsType, size, storageDir.getCapacityBytes())); - } - break; - } - if (!foundMountInfo) { - LOG.warn("Failed to verify memory capacity"); - } - } - - /** - * Factory method to create {@link StorageTier}. - * - * @param tierAlias the tier alias - * @param tierOrdinal the tier ordinal - * @param isMultiTier whether this tier is part of a multi-tier setup - * @return a new storage tier - */ - public static StorageTier newStorageTier(String tierAlias, int tierOrdinal, boolean isMultiTier) { - DefaultStorageTier ret = new DefaultStorageTier(tierAlias, tierOrdinal); - ret.initStorageTier(isMultiTier); - return ret; - } - - @Override - public int getTierOrdinal() { - return mTierOrdinal; - } - - @Override - public String getTierAlias() { - return mTierAlias; - } - - @Override - public long getCapacityBytes() { - long totalCapacity = 0; - for (StorageDir dir : mDirs.values()) { - totalCapacity += dir.getCapacityBytes(); - } - return totalCapacity; - } - - @Override - public long getAvailableBytes() { - long availableBytes = 0; - for (StorageDir dir : mDirs.values()) { - availableBytes += dir.getAvailableBytes(); - } - return availableBytes; - } - - @Override - @Nullable - public StorageDir getDir(int dirIndex) { - return mDirs.get(dirIndex); - } - - @Override - public List getStorageDirs() { - return new ArrayList<>(mDirs.values()); - } - - @Override - public List getLostStorage() { - return new ArrayList<>(mLostStorage); - } - - /** - * @throws IllegalArgumentException if dir does not belong to this tier - */ - @Override - public void removeStorageDir(StorageDir dir) { - Preconditions.checkArgument(this == dir.getParentTier(), - "storage dir %s does not belong to tier %s", dir.getDirPath(), getTierAlias()); - mDirs.remove(dir.getDirIndex()); - mLostStorage.add(dir.getDirPath()); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultTempBlockMeta.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultTempBlockMeta.java deleted file mode 100644 index 97f74ed8b685..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/DefaultTempBlockMeta.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.io.PathUtils; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.base.Preconditions; - -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Represents the metadata of an uncommitted block in Alluxio managed storage. - */ -@NotThreadSafe -public class DefaultTempBlockMeta implements TempBlockMeta { - private static final String TMP_DIR = Configuration.getString( - PropertyKey.WORKER_DATA_TMP_FOLDER); - private static final int SUB_DIR_MAX = Configuration.getInt( - PropertyKey.WORKER_DATA_TMP_SUBDIR_MAX); - private final long mBlockId; - private final StorageDir mDir; - private final long mSessionId; - private long mTempBlockSize; - - /** - * All blocks are created as temp blocks before committed. They are stored in BlockStore under a - * subdir of its {@link StorageDir}, the subdir is tmpFolder/sessionId % maxSubdirMax. - * tmpFolder is a property of {@link PropertyKey#WORKER_DATA_TMP_FOLDER}. - * maxSubdirMax is a property of {@link PropertyKey#WORKER_DATA_TMP_SUBDIR_MAX}. - * The block file name is "sessionId-blockId". e.g. sessionId 2 creates a temp Block 100 in - * {@link StorageDir} "/mnt/mem/0", this temp block has path: - *

- * /mnt/mem/0/.tmp_blocks/2/2-100 - * - * @param dir the parent directory - * @param sessionId the session id - * @param blockId the block id - * @return temp file path - */ - public static String tempPath(StorageDir dir, long sessionId, long blockId) { - return PathUtils.concatPath(dir.getDirPath(), TMP_DIR, sessionId % SUB_DIR_MAX, - String.format("%x-%d", sessionId, blockId)); - } - - /** - * Creates a new instance of {@link DefaultTempBlockMeta}. - * - * @param sessionId the session id - * @param blockId the block id - * @param initialBlockSize initial size of this block in bytes - * @param dir {@link StorageDir} of this temp block belonging to - */ - public DefaultTempBlockMeta(long sessionId, long blockId, long initialBlockSize, StorageDir dir) { - mBlockId = blockId; - mDir = Preconditions.checkNotNull(dir, "dir"); - mSessionId = sessionId; - mTempBlockSize = initialBlockSize; - } - - @Override - public long getBlockSize() { - return mTempBlockSize; - } - - @Override - public String getPath() { - return tempPath(mDir, mSessionId, mBlockId); - } - - @Override - public long getBlockId() { - return mBlockId; - } - - @Override - public BlockStoreLocation getBlockLocation() { - StorageTier tier = mDir.getParentTier(); - return new BlockStoreLocation(tier.getTierAlias(), mDir.getDirIndex(), mDir.getDirMedium()); - } - - @Override - public StorageDir getParentDir() { - return mDir; - } - - @Override - public String getCommitPath() { - return DefaultBlockMeta.commitPath(mDir, mBlockId); - } - - @Override - public long getSessionId() { - return mSessionId; - } - - @Override - public void setBlockSize(long newSize) { - mTempBlockSize = newSize; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirAllocatorView.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirAllocatorView.java deleted file mode 100644 index f52cb2edc1f4..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirAllocatorView.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -/** - * This class is a wrapper of {@link StorageDir} to provide more limited access for allocators. - */ -public class StorageDirAllocatorView extends StorageDirView { - - /** - * Creates a {@link StorageDirAllocatorView} using the actual {@link StorageDir}. - * - * @param dir which the dirView is constructed from - * @param tierView which the dirView is under - */ - public StorageDirAllocatorView(StorageDir dir, StorageTierView tierView) { - super(dir, tierView); - } - - @Override - public long getAvailableBytes() { - long reservedBytes = mDir.getReservedBytes(); - long availableBytes = mDir.getAvailableBytes(); - long capacityBytes = mDir.getCapacityBytes(); - if (mTierView.mUseReservedSpace) { - return capacityBytes - mDir.getCommittedBytes(); - } else { - return availableBytes; - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirEvictorView.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirEvictorView.java deleted file mode 100644 index 60dea7817799..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirEvictorView.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.worker.block.BlockMetadataEvictorView; - -import com.google.common.base.Preconditions; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * This class is a wrapper of {@link StorageDir} to provide more limited access for evictors. - */ -public class StorageDirEvictorView extends StorageDirView { - /** The {@link BlockMetadataEvictorView} this view is associated with. */ - private final BlockMetadataEvictorView mMetadataView; - - // The below data structures are used by the evictor to mark blocks to move in/out during - // generating an eviction plan. - private final Set mBlocksToMoveIn = new HashSet<>(); - private final Set mBlocksToMoveOut = new HashSet<>(); - private long mBlocksToMoveInSize = 0L; - private long mBlocksToMoveOutSize = 0L; - - /** - * Creates a {@link StorageDirEvictorView} using the actual {@link StorageDir} - * and the associated {@link BlockMetadataEvictorView}. - * - * @param dir which the dirView is constructed from - * @param tierView which the dirView is under - * @param managerView which the dirView is associated with - */ - public StorageDirEvictorView(StorageDir dir, StorageTierEvictorView tierView, - BlockMetadataEvictorView managerView) { - super(dir, tierView); - mMetadataView = Preconditions.checkNotNull(managerView, "view"); - } - - @Override - public long getAvailableBytes() { - return mDir.getAvailableBytes() + mBlocksToMoveOutSize - mBlocksToMoveInSize; - } - - /** - * Gets a filtered list of block metadata, for blocks that are neither pinned or being blocked. - * - * @return a list of metadata for all evictable blocks - */ - public List getEvictableBlocks() { - List filteredList = new ArrayList<>(); - - for (BlockMeta blockMeta : mDir.getBlocks()) { - long blockId = blockMeta.getBlockId(); - if (mMetadataView.isBlockEvictable(blockId)) { - filteredList.add(blockMeta); - } - } - return filteredList; - } - - /** - * Gets evictable bytes for this dir, i.e., the total bytes of total evictable blocks. - * - * @return evictable bytes for this dir - */ - public long getEvitableBytes() { - long bytes = 0; - for (BlockMeta blockMeta : mDir.getBlocks()) { - long blockId = blockMeta.getBlockId(); - if (mMetadataView.isBlockEvictable(blockId)) { - bytes += blockMeta.getBlockSize(); - } - } - return bytes; - } - - /** - * Clears all marks about blocks to move in/out in this view. - */ - public void clearBlockMarks() { - mBlocksToMoveIn.clear(); - mBlocksToMoveOut.clear(); - mBlocksToMoveInSize = mBlocksToMoveOutSize = 0L; - } - - /** - * Returns an indication whether the given block is marked to be moved out. - * - * @param blockId the block ID - * @return whether the block is marked to be moved out - */ - public boolean isMarkedToMoveOut(long blockId) { - return mBlocksToMoveOut.contains(blockId); - } - - /** - * Marks a block to move into this dir view, which is used by the evictor. - * - * @param blockId the Id of the block - * @param blockSize the block size - */ - public void markBlockMoveIn(long blockId, long blockSize) { - if (mBlocksToMoveIn.add(blockId)) { - mBlocksToMoveInSize += blockSize; - } - } - - /** - * Marks a block to move out of this dir view, which is used by the evictor. - * - * @param blockId the Id of the block - * @param blockSize the block size - */ - public void markBlockMoveOut(long blockId, long blockSize) { - if (mBlocksToMoveOut.add(blockId)) { - mBlocksToMoveOutSize += blockSize; - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirView.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirView.java deleted file mode 100644 index 9d4ca0be2267..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageDirView.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.base.Preconditions; - -/** - * This class is a wrapper of {@link StorageDir} to provide more limited access. - */ -public abstract class StorageDirView { - - /** The {@link StorageDir} this view is derived from. */ - final StorageDir mDir; - /** The {@link StorageTierView} this view under. */ - final StorageTierView mTierView; - - /** - * Creates a {@link StorageDirView} using the actual {@link StorageDir}. - * - * @param dir which the dirView is constructed from - * @param tierView which the dirView is under - */ - public StorageDirView(StorageDir dir, StorageTierView tierView) { - mDir = Preconditions.checkNotNull(dir, "dir"); - mTierView = Preconditions.checkNotNull(tierView, "tierView"); - } - - /** - * Gets available bytes for this dir. - * - * @return available bytes for this dir - */ - public abstract long getAvailableBytes(); - - /** - * @return the amount of bytes reserved for internal management - */ - public long getReservedBytes() { - return mDir.getReservedBytes(); - } - - /** - * Gets the index of this Dir. - * - * @return index of the dir - */ - public int getDirViewIndex() { - return mDir.getDirIndex(); - } - - /** - * Gets capacity bytes for this dir. - * - * @return capacity bytes for this dir - */ - public long getCapacityBytes() { - return mDir.getCapacityBytes(); - } - - /** - * Gets committed bytes for this dir. This includes all blocks, locked, pinned, committed etc. - * - * @return committed bytes for this dir - */ - public long getCommittedBytes() { - return mDir.getCommittedBytes(); - } - - /** - * Creates a {@link TempBlockMeta} given sessionId, blockId, and initialBlockSize. - * - * @param sessionId of the owning session - * @param blockId of the new block - * @param initialBlockSize of the new block - * @return a new {@link TempBlockMeta} under the underlying directory - */ - public TempBlockMeta createTempBlockMeta(long sessionId, long blockId, long initialBlockSize) { - return new DefaultTempBlockMeta(sessionId, blockId, initialBlockSize, mDir); - } - - /** - * @return parent tier view - */ - public StorageTierView getParentTierView() { - return mTierView; - } - - /** - * Get medium type of the dir view, which is derived from the dir. - * - * @return the medium type - */ - public String getMediumType() { - return mDir.getDirMedium(); - } - - /** - * Creates a {@link BlockStoreLocation} for this directory view. Redirecting to - * {@link StorageDir#toBlockStoreLocation} - * - * @return {@link BlockStoreLocation} created - */ - public BlockStoreLocation toBlockStoreLocation() { - return mDir.toBlockStoreLocation(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierAllocatorView.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierAllocatorView.java deleted file mode 100644 index 673d268c7308..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierAllocatorView.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * This class is a wrapper of {@link StorageTier} to provide more limited access for allocators. - */ -@ThreadSafe -public class StorageTierAllocatorView extends StorageTierView { - - /** - * Creates a {@link StorageTierView} using the actual {@link StorageTier}. - * - * @param tier which the tierView is constructed from - * @param useReservedSpace whether to include reserved space in dir's available bytes - */ - public StorageTierAllocatorView(StorageTier tier, boolean useReservedSpace) { - super(tier, useReservedSpace); - for (StorageDir dir : mTier.getStorageDirs()) { - StorageDirAllocatorView dirView = new StorageDirAllocatorView(dir, this); - mDirViews.put(dirView.getDirViewIndex(), dirView); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierEvictorView.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierEvictorView.java deleted file mode 100644 index 0dd1307091fe..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierEvictorView.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.worker.block.BlockMetadataEvictorView; - -import com.google.common.base.Preconditions; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * This class is a wrapper of {@link StorageTier} to provide more limited access for evictors. - */ -@ThreadSafe -public class StorageTierEvictorView extends StorageTierView { - - /** The {@link BlockMetadataEvictorView} this {@link StorageTierEvictorView} is under. */ - private final BlockMetadataEvictorView mMetadataView; - - /** - * Creates a {@link StorageTierEvictorView} using the actual {@link StorageTier} and the above - * {@link BlockMetadataEvictorView}. - * - * @param tier which the tierView is constructed from - * @param view the {@link BlockMetadataEvictorView} this tierView is associated with - */ - public StorageTierEvictorView(StorageTier tier, BlockMetadataEvictorView view) { - super(tier); - mMetadataView = Preconditions.checkNotNull(view, "view"); - - for (StorageDir dir : mTier.getStorageDirs()) { - StorageDirEvictorView dirView = new StorageDirEvictorView(dir, this, view); - mDirViews.put(dirView.getDirViewIndex(), dirView); - } - } - - /** - * @return the block metadata evictor view for this storage tier view - */ - public BlockMetadataEvictorView getBlockMetadataEvictorView() { - return mMetadataView; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierView.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierView.java deleted file mode 100644 index 3a094670f3fb..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/StorageTierView.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import com.google.common.base.Preconditions; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -/** - * This class is an abstract class for allocators and evictors to extend to provide - * limited access to {@link StorageTier}. - */ -public abstract class StorageTierView { - - /** The {@link StorageTier} this view is derived from. */ - final StorageTier mTier; - /** A list of {@link StorageDirView} under this StorageTierView. */ - final Map mDirViews = new HashMap<>(); - /** Whether to include reserved space into availability calculations. */ - final boolean mUseReservedSpace; - - /** - * Creates a {@link StorageTierView} using the actual {@link StorageTier}. - * - * @param tier which the tierView is constructed from - */ - public StorageTierView(StorageTier tier) { - this(tier, false); - } - - /** - * Creates a {@link StorageTierView} using the actual {@link StorageTier}. - * - * @param tier which the tierView is constructed from - * @param useReservedSpace whether to include reserved space in available bytes - */ - public StorageTierView(StorageTier tier, boolean useReservedSpace) { - mTier = Preconditions.checkNotNull(tier, "tier"); - mUseReservedSpace = useReservedSpace; - } - - /** - * @return a list of directory views in this storage tier view - */ - public Collection getDirViews() { - return mDirViews.values(); - } - - /** - * Returns a directory view for the given index. - * - * @param dirIndex the directory view index - * @return a directory view - */ - public StorageDirView getDirView(int dirIndex) { - return mDirViews.get(dirIndex); - } - - /** - * @return the storage tier view alias - */ - public String getTierViewAlias() { - return mTier.getTierAlias(); - } - - /** - * @return the ordinal value of the storage tier view - */ - public int getTierViewOrdinal() { - return mTier.getTierOrdinal(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/meta/UnderFileSystemBlockMeta.java b/core/server/worker/src/main/java/alluxio/worker/block/meta/UnderFileSystemBlockMeta.java deleted file mode 100644 index e1473b0ba959..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/meta/UnderFileSystemBlockMeta.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.proto.dataserver.Protocol; - -/** - * This class represents the metadata of a block that is in UFS. This class is immutable. - */ -public final class UnderFileSystemBlockMeta { - private final long mSessionId; - private final long mBlockId; - private final String mUnderFileSystemPath; - /** The offset in bytes of the first byte of the block in its corresponding UFS file. */ - private final long mOffset; - /** The block size in bytes. */ - private final long mBlockSize; - /** The id of the mount point. */ - private final long mMountId; - /** Do not cache the block to the local Alluxio worker if set. */ - private final boolean mNoCache; - private final String mUser; - - /** - * Creates an instance of {@link UnderFileSystemBlockMeta}. - * - * @param sessionId the session ID - * @param blockId the block ID - * @param options the {@link Protocol.OpenUfsBlUfsInputStreamCacheTestockOptions} - */ - public UnderFileSystemBlockMeta(long sessionId, long blockId, - Protocol.OpenUfsBlockOptions options) { - mSessionId = sessionId; - mBlockId = blockId; - mUnderFileSystemPath = options.getUfsPath(); - mOffset = options.getOffsetInFile(); - mBlockSize = options.getBlockSize(); - mMountId = options.getMountId(); - mNoCache = options.getNoCache(); - mUser = options.getUser(); - } - - /** - * @return the session ID - */ - public long getSessionId() { - return mSessionId; - } - - /** - * @return the block ID - */ - public long getBlockId() { - return mBlockId; - } - - /** - * @return the UFS path - */ - public String getUnderFileSystemPath() { - return mUnderFileSystemPath; - } - - /** - * @return the offset of the block in the UFS file - */ - public long getOffset() { - return mOffset; - } - - /** - * @return the block size in bytes - */ - public long getBlockSize() { - return mBlockSize; - } - - /** - * @return the id of the mount of this file is mapped to - */ - public long getMountId() { - return mMountId; - } - - /** - * @return true if mNoCache is set - */ - public boolean isNoCache() { - return mNoCache; - } - - /** - * @return the user - */ - public String getUser() { - return mUser; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/reviewer/AcceptingReviewer.java b/core/server/worker/src/main/java/alluxio/worker/block/reviewer/AcceptingReviewer.java deleted file mode 100644 index 2a9079661291..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/reviewer/AcceptingReviewer.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.reviewer; - -import alluxio.worker.block.meta.StorageDirView; - -/** - * An implementation of {@link Reviewer} that never rejects. - * This is used to turn the Reviewer logic off. - * */ -public class AcceptingReviewer implements Reviewer { - @Override - public boolean acceptAllocation(StorageDirView dirView) { - return true; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/reviewer/ProbabilisticBufferReviewer.java b/core/server/worker/src/main/java/alluxio/worker/block/reviewer/ProbabilisticBufferReviewer.java deleted file mode 100644 index 5bf01afe5534..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/reviewer/ProbabilisticBufferReviewer.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.reviewer; - -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageDirView; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.ThreadLocalRandom; - -/** - * Reviews a block allocation, and probably rejects a bad block allocation. - * The more full a StorageDir, the higher the chance that this reviewer decides - * NOT to use it for a new block. - * - * The intention is to leave a buffer to each {@link StorageDir} by early stopping putting - * new blocks into it. - * The existing blocks in the {@link StorageDir} will likely read more data in and expand in size. - * We want to leave some space for the expansion, to lower the chance of eviction. - * */ -public class ProbabilisticBufferReviewer implements Reviewer { - private static final Logger LOG = LoggerFactory.getLogger(ProbabilisticBufferReviewer.class); - private static final ThreadLocalRandom RANDOM = ThreadLocalRandom.current(); - - private final long mHardLimitBytes; - private final long mSoftLimitBytes; - - /** - * Constructor the instance from configuration. - * */ - public ProbabilisticBufferReviewer() { - AlluxioConfiguration conf = Configuration.global(); - mHardLimitBytes = conf.getBytes(PropertyKey.WORKER_REVIEWER_PROBABILISTIC_HARDLIMIT_BYTES); - long stopSoftBytes = conf.getBytes(PropertyKey.WORKER_REVIEWER_PROBABILISTIC_SOFTLIMIT_BYTES); - if (stopSoftBytes <= mHardLimitBytes) { - LOG.warn("{} should be greater than or equal to {}. Setting {} to {}.", - PropertyKey.WORKER_REVIEWER_PROBABILISTIC_SOFTLIMIT_BYTES, - PropertyKey.WORKER_REVIEWER_PROBABILISTIC_HARDLIMIT_BYTES, - PropertyKey.WORKER_REVIEWER_PROBABILISTIC_SOFTLIMIT_BYTES, - mHardLimitBytes); - mSoftLimitBytes = mHardLimitBytes; - } else { - mSoftLimitBytes = stopSoftBytes; - } - } - - /** - * Calculates the probability of allowing a new block into this dir, - * based on how much available space there is. - * */ - double getProbability(StorageDirView dirView) { - long availableBytes = dirView.getAvailableBytes(); - long capacityBytes = dirView.getCapacityBytes(); - - // Rules: - // 1. If more than the SOFT limit left, we use this tier. Prob=100% - // 2. If the tier is less than block size, ignore this tier. Prob=0% - // 3. If in the middle, the probability is linear to the space left, - // the less space the lower. - if (availableBytes > mSoftLimitBytes) { - return 1.0; - } - if (availableBytes <= mHardLimitBytes) { - return 0.0; - } - // 2 points: - // Axis X: space usage (commitment) - // Axis Y: Probability of using this tier - // (capacity - soft, 1.0) - // (capacity - hard, 0.0) - double x = capacityBytes - availableBytes; - // If HardLimit = SoftLimit, then we would have returned in the previous if-else - double k = 1.0 / (mHardLimitBytes - mSoftLimitBytes); - double b = (capacityBytes - mHardLimitBytes + 0.0) / (mSoftLimitBytes - mHardLimitBytes); - double y = k * x + b; - LOG.debug("{} bytes available in {}. Probability of staying is {}.", availableBytes, - dirView.toBlockStoreLocation(), y); - return y; - } - - @Override - public boolean acceptAllocation(StorageDirView dirView) { - double chance = getProbability(dirView); - // Throw a dice - double dice = RANDOM.nextDouble(); - return dice < chance; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/block/reviewer/Reviewer.java b/core/server/worker/src/main/java/alluxio/worker/block/reviewer/Reviewer.java deleted file mode 100644 index c6351927dc32..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/block/reviewer/Reviewer.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.reviewer; - -import alluxio.annotation.PublicApi; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.worker.block.allocator.Allocator; -import alluxio.worker.block.meta.StorageDirView; - -/** - * (Experimental) The API is subject to change in the future. - * Interface for the review policy of allocation decision made by {@link Allocator}. - * - * Each {@link Allocator} has a Reviewer instance according to the policy definition. - * For each block allocation decision, the Reviewer reviews it according to the criteria - * defined in the policy. - * If the allocation does not meet the criteria, the Reviewer will reject it. - * */ -@PublicApi -public interface Reviewer { - /** - * Reviews an allocation proposed by the {@link Allocator}. - * Returning true means the allocation is accepted. - * Returning false meanes the allocation is rejected. - * - * @param dirView the storage dir that the block is allocated to - * @return whether the allocation is accepted - * */ - boolean acceptAllocation(StorageDirView dirView); - - /** - * Factory for {@link Reviewer}. - */ - class Factory { - - private Factory() {} // prevent instantiation - - /** - * Factory for {@link Reviewer}. - * - * @return the generated {@link Reviewer}, it will be a {@link ProbabilisticBufferReviewer} - * by default - */ - public static Reviewer create() { - return CommonUtils.createNewClassInstance( - Configuration.getClass(PropertyKey.WORKER_REVIEWER_CLASS), - new Class[] {}, new Object[] {}); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/file/FileSystemMasterClient.java b/core/server/worker/src/main/java/alluxio/worker/file/FileSystemMasterClient.java deleted file mode 100644 index e8ffb79c5dd4..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/file/FileSystemMasterClient.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.file; - -import alluxio.AbstractMasterClient; -import alluxio.Constants; -import alluxio.conf.PropertyKey; -import alluxio.grpc.FileSystemMasterWorkerServiceGrpc; -import alluxio.grpc.GetFileInfoPRequest; -import alluxio.grpc.GetPinnedFileIdsPRequest; -import alluxio.grpc.GetUfsInfoPRequest; -import alluxio.grpc.GrpcUtils; -import alluxio.grpc.ServiceType; -import alluxio.grpc.UfsInfo; -import alluxio.master.MasterClientContext; -import alluxio.wire.FileInfo; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import javax.annotation.concurrent.ThreadSafe; - -/** - * A wrapper for the gRPC client to interact with the file system master, used by Alluxio worker. - */ -@ThreadSafe -public class FileSystemMasterClient extends AbstractMasterClient { - private static final Logger LOG = LoggerFactory.getLogger(FileSystemMasterClient.class); - private FileSystemMasterWorkerServiceGrpc.FileSystemMasterWorkerServiceBlockingStub mClient = - null; - - /** - * Creates a instance of {@link FileSystemMasterClient}. - * - * @param conf master client configuration - */ - public FileSystemMasterClient(MasterClientContext conf) { - super(conf); - } - - @Override - protected ServiceType getRemoteServiceType() { - return ServiceType.FILE_SYSTEM_MASTER_WORKER_SERVICE; - } - - @Override - protected String getServiceName() { - return Constants.FILE_SYSTEM_MASTER_WORKER_SERVICE_NAME; - } - - @Override - protected long getServiceVersion() { - return Constants.FILE_SYSTEM_MASTER_WORKER_SERVICE_VERSION; - } - - @Override - protected void afterConnect() throws IOException { - mClient = FileSystemMasterWorkerServiceGrpc.newBlockingStub(mChannel); - } - - /** - * @param fileId the id of the file for which to get the {@link FileInfo} - * @return the file info for the given file id - */ - public FileInfo getFileInfo(final long fileId) throws IOException { - return retryRPC(() -> GrpcUtils.fromProto(mClient - .getFileInfo(GetFileInfoPRequest.newBuilder().setFileId(fileId).build()).getFileInfo()), - LOG, "GetFileInfo", "fileId=%d", fileId); - } - - /** - * @return the set of pinned file ids - */ - public Set getPinList() throws IOException { - return retryRPC(() -> new HashSet<>(mClient.withDeadlineAfter(mContext.getClusterConf() - .getMs(PropertyKey.WORKER_MASTER_PERIODICAL_RPC_TIMEOUT), TimeUnit.MILLISECONDS) - .getPinnedFileIds(GetPinnedFileIdsPRequest.newBuilder().build()) - .getPinnedFileIdsList()), - LOG, "GetPinList", ""); - } - - /** - * @param mountId the id of the mount - * @return the ufs information for the give ufs - * @throws IOException if an I/O error occurs - */ - public UfsInfo getUfsInfo(final long mountId) throws IOException { - return retryRPC(() -> mClient - .getUfsInfo(GetUfsInfoPRequest.newBuilder().setMountId(mountId).build()).getUfsInfo(), - LOG, "GetUfsInfo", "mountId=%d", mountId); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/AbstractWriteHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/AbstractWriteHandler.java deleted file mode 100644 index 8e6c6659bc85..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/AbstractWriteHandler.java +++ /dev/null @@ -1,458 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.Constants; -import alluxio.RpcSensitiveConfigMask; -import alluxio.client.block.stream.GrpcDataWriter; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.InvalidArgumentException; -import alluxio.grpc.WriteRequest; -import alluxio.grpc.WriteRequestCommand; -import alluxio.grpc.WriteResponse; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.network.protocol.databuffer.NioDataBuffer; -import alluxio.security.authentication.AuthenticatedUserInfo; -import alluxio.util.LogUtils; -import alluxio.util.logging.SamplingLogger; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; -import com.google.common.base.Preconditions; -import com.google.protobuf.ByteString; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.internal.SerializingExecutor; -import io.grpc.stub.StreamObserver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.Semaphore; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class handles {@link WriteRequest}s. - * - * Protocol: Check {@link GrpcDataWriter} for more information. - * 1. The write handler streams data from the client and writes them. The flow control is handled - * by gRPC server. - * 3. A complete or cancel event signifies the completion of this request. - * 4. When an error occurs, the response stream is closed. - * - * Threading model: - * The write handler runs the gRPC event threads. gRPC serializes the events so there will not be - * events running concurrently for same request. - * - * @param type of write request - */ -@NotThreadSafe -abstract class AbstractWriteHandler> { - private static final Logger LOG = LoggerFactory.getLogger(AbstractWriteHandler.class); - private static final Logger SLOW_WRITE_LOG = new SamplingLogger(LOG, 5 * Constants.MINUTE_MS); - private static final long SLOW_WRITE_MS = - Configuration.getMs(PropertyKey.WORKER_REMOTE_IO_SLOW_THRESHOLD); - public static final long FILE_BUFFER_SIZE = Constants.MB; - - /** The observer for sending response messages. */ - private final StreamObserver mResponseObserver; - /** The executor for running write tasks asynchronously in the submission order. */ - private final SerializingExecutor mSerializingExecutor; - /** The semaphore to control the number of write tasks queued up in the executor.*/ - private final Semaphore mSemaphore = new Semaphore( - Configuration.getInt(PropertyKey.WORKER_NETWORK_WRITER_BUFFER_SIZE_MESSAGES), true); - - /** - * This is initialized only once for a whole file or block in - * {@link AbstractWriteHandler#write(WriteRequest)}. - * It is safe to read those final primitive fields (e.g. mId, mSessionId) if mError is not set. - * - * Using "volatile" because we want any value change of this variable to be - * visible across all gRPC threads. - */ - private volatile T mContext; - - protected AuthenticatedUserInfo mUserInfo; - - /** - * Creates an instance of {@link AbstractWriteHandler}. - * - * @param responseObserver the response observer for the write request - * @param userInfo the authenticated user info - */ - AbstractWriteHandler(StreamObserver responseObserver, - AuthenticatedUserInfo userInfo) { - mResponseObserver = responseObserver; - mUserInfo = userInfo; - mSerializingExecutor = new SerializingExecutor(GrpcExecutors.BLOCK_WRITER_EXECUTOR); - } - - /** - * Handles write request. - * - * @param writeRequest the request from the client - */ - public void write(WriteRequest writeRequest) { - if (!tryAcquireSemaphore()) { - return; - } - mSerializingExecutor.execute(() -> { - try { - if (mContext == null) { - LOG.debug("Received write request {}.", - RpcSensitiveConfigMask.CREDENTIAL_FIELD_MASKER.maskObjects(LOG, writeRequest)); - try { - mContext = createRequestContext(writeRequest); - } catch (Exception e) { - // abort() assumes context is initialized. - // Reply with the error in order to prevent clients getting stuck. - replyError(new Error(AlluxioStatusException.fromThrowable(e), true)); - throw e; - } - } else { - Preconditions.checkState(!mContext.isDoneUnsafe(), - "invalid request after write request is completed."); - } - if (mContext.isDoneUnsafe() || mContext.getError() != null) { - return; - } - validateWriteRequest(writeRequest); - if (writeRequest.hasCommand()) { - WriteRequestCommand command = writeRequest.getCommand(); - if (command.getFlush()) { - flush(); - } else { - handleCommand(command, mContext); - } - } else { - Preconditions.checkState(writeRequest.hasChunk(), - "write request is missing data chunk in non-command message"); - ByteString data = writeRequest.getChunk().getData(); - Preconditions.checkState(data != null && data.size() > 0, - "invalid data size from write request message"); - writeData(new NioDataBuffer(data.asReadOnlyByteBuffer(), data.size())); - } - } catch (Exception e) { - LogUtils.warnWithException(LOG, "Exception occurred while processing write request {}.", - writeRequest, e); - abort(new Error(AlluxioStatusException.fromThrowable(e), true)); - } finally { - mSemaphore.release(); - } - }); - } - - /** - * Handles write request with data message. - * - * @param request the request from the client - * @param buffer the data associated with the request - */ - public void writeDataMessage(WriteRequest request, DataBuffer buffer) { - if (buffer == null) { - write(request); - return; - } - boolean releaseBuf = true; - try { - Preconditions.checkState(!request.hasCommand(), - "write request command should not come with data buffer"); - Preconditions.checkState(buffer.readableBytes() > 0, - "invalid data size from write request message"); - if (!tryAcquireSemaphore()) { - return; - } - releaseBuf = false; - mSerializingExecutor.execute(() -> { - try { - writeData(buffer); - } finally { - mSemaphore.release(); - } - }); - } finally { - if (releaseBuf) { - buffer.release(); - } - } - } - - /** - * Handles request complete event. - */ - public void onCompleted() { - mSerializingExecutor.execute(() -> { - Preconditions.checkState(mContext != null); - try { - completeRequest(mContext); - replySuccess(); - } catch (Exception e) { - LogUtils.warnWithException(LOG, "Exception occurred while completing write request {}.", - mContext.getRequest(), e); - abort(new Error(AlluxioStatusException.fromThrowable(e), true)); - } - }); - } - - /** - * Handles request cancellation event. - */ - public void onCancel() { - mSerializingExecutor.execute(() -> { - try { - cancelRequest(mContext); - replyCancel(); - } catch (Exception e) { - LogUtils.warnWithException(LOG, "Exception occurred while cancelling write request {}.", - mContext.getRequest(), e); - abort(new Error(AlluxioStatusException.fromThrowable(e), true)); - } - }); - } - - /** - * Handles errors from the request. - * - * @param cause the exception - */ - public void onError(Throwable cause) { - if (cause instanceof StatusRuntimeException - && ((StatusRuntimeException) cause).getStatus().getCode() == Status.Code.CANCELLED) { - // Cancellation is already handled. - return; - } - mSerializingExecutor.execute(() -> { - LogUtils.warnWithException(LOG, "Exception thrown while handling write request {}", - mContext == null ? "unknown" : mContext.getRequest(), cause); - abort(new Error(AlluxioStatusException.fromThrowable(cause), false)); - }); - } - - private boolean tryAcquireSemaphore() { - try { - mSemaphore.acquire(); - } catch (InterruptedException e) { - LOG.warn("write data request {} is interrupted: {}", - mContext == null ? "unknown" : mContext.getRequest(), e.getMessage()); - abort(new Error(AlluxioStatusException.fromThrowable(e), true)); - Thread.currentThread().interrupt(); - return false; - } - return true; - } - - /** - * Validates a block write request. - * - * @param request the block write request - * @throws InvalidArgumentException if the write request is invalid - */ - @GuardedBy("mLock") - private void validateWriteRequest(alluxio.grpc.WriteRequest request) - throws InvalidArgumentException { - if (request.hasCommand() && request.getCommand().hasOffset() - && request.getCommand().getOffset() != mContext.getPos()) { - throw new InvalidArgumentException(String.format( - "Offsets do not match [received: %d, expected: %d].", - request.getCommand().getOffset(), mContext.getPos())); - } - } - - private void writeData(DataBuffer buf) { - try { - if (mContext.isDoneUnsafe() || mContext.getError() != null) { - return; - } - int readableBytes = buf.readableBytes(); - mContext.setPos(mContext.getPos() + readableBytes); - - long writeStartMs = System.currentTimeMillis(); - writeBuf(mContext, mResponseObserver, buf, mContext.getPos()); - long writeMs = System.currentTimeMillis() - writeStartMs; - - if (writeMs >= SLOW_WRITE_MS) { - // A single write call took much longer than expected. - - String prefix = String - .format("Writing buffer for remote write took longer than %s ms. handler: %s", - SLOW_WRITE_MS, this.getClass().getName()); - - // Do not template the handler class, so the sampling log can distinguish between - // different handler types - SLOW_WRITE_LOG.warn(prefix + " id: {} location: {} bytes: {} durationMs: {}", - mContext.getRequest().getId(), getLocation(), readableBytes, writeMs); - } - - incrementMetrics(readableBytes); - } catch (Exception e) { - LOG.error("Failed to write data for request {}", mContext.getRequest(), e); - abort(new Error(AlluxioStatusException.fromThrowable(e), true)); - } finally { - buf.release(); - } - } - - private void flush() { - try { - flushRequest(mContext); - replyFlush(); - } catch (Exception e) { - LOG.error("Failed to flush for write request {}", mContext.getRequest(), e); - abort(new Error(AlluxioStatusException.fromThrowable(e), true)); - } - } - - /** - * Abort the write process due to error. - * - * @param error the error - */ - private void abort(Error error) { - try { - if (mContext == null || mContext.getError() != null || mContext.isDoneUnsafe()) { - // Note, we may reach here via events due to network errors bubbling up before - // mContext is initialized, or stream error after the request is finished. - return; - } - mContext.setError(error); - cleanupRequest(mContext); - } catch (Exception e) { - LOG.warn("Failed to cleanup states with error {}.", e.toString()); - } finally { - replyError(); - } - } - - /** - * Creates a new request context. - * - * @param msg the block write request - */ - protected abstract T createRequestContext(WriteRequest msg) throws Exception; - - /** - * Completes this write. This is called when the write completes. - * @param context context of the request to complete - */ - protected abstract void completeRequest(T context) throws Exception; - - /** - * Cancels this write. This is called when the client issues a cancel request. - * - * @param context context of the request to complete - */ - protected abstract void cancelRequest(T context) throws Exception; - - /** - * Cleans up this write. This is called when the write request is aborted due to any exception - * or session timeout. - * - * @param context context of the request to complete - */ - protected abstract void cleanupRequest(T context) throws Exception; - - /** - * Flushes the buffered data. Flush only happens after write. - * - * @param context context of the request to complete - */ - protected abstract void flushRequest(T context) throws Exception; - - /** - * Writes the buffer. - * @param context context of the request to complete - * @param responseObserver the response observer - * @param buf the buffer - * @param pos the pos - */ - protected abstract void writeBuf(T context, StreamObserver responseObserver, - DataBuffer buf, long pos) throws Exception; - - /** - * @param context the context of the request - * @return an informational string of the location the writer is writing to - */ - protected abstract String getLocationInternal(T context); - - /** - * @return an informational string of the location the writer is writing to - */ - public String getLocation() { - if (mContext == null) { - return "null"; - } - return getLocationInternal(mContext); - } - - /** - * Handles a command in the write request. - * - * @param command the command to be handled - */ - protected void handleCommand(WriteRequestCommand command, T context) throws Exception { - // no additional command is handled by default - } - - /** - * Writes a response to signify the success of the write request. - */ - private void replySuccess() { - mContext.setDoneUnsafe(true); - mResponseObserver.onCompleted(); - } - - /** - * Writes a response to signify the successful cancellation of the write request. - */ - private void replyCancel() { - mContext.setDoneUnsafe(true); - mResponseObserver.onCompleted(); - } - - /** - * Writes an error response. - */ - private void replyError() { - replyError(Preconditions.checkNotNull(mContext.getError())); - } - - /** - * Writes an error response. - */ - private void replyError(Error error) { - if (error.isNotifyClient()) { - mResponseObserver.onError(error.getCause().toGrpcStatusException()); - } - } - - /** - * Writes a response to signify the successful flush. - */ - private void replyFlush() { - mResponseObserver.onNext( - WriteResponse.newBuilder().setOffset(mContext.getPos()).build()); - } - - /** - * @param bytesWritten bytes written - */ - private void incrementMetrics(long bytesWritten) { - Counter counter = mContext.getCounter(); - Meter meter = mContext.getMeter(); - Preconditions.checkState(counter != null, "counter"); - Preconditions.checkState(meter != null, "meter"); - counter.inc(bytesWritten); - meter.mark(bytesWritten); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockReadHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/BlockReadHandler.java deleted file mode 100644 index bf0cf4c5603e..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockReadHandler.java +++ /dev/null @@ -1,673 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import static alluxio.util.CommonUtils.isFatalError; -import static alluxio.worker.block.BlockMetadataManager.WORKER_STORAGE_TIER_ASSOC; - -import alluxio.Constants; -import alluxio.ProcessUtils; -import alluxio.RpcSensitiveConfigMask; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.InvalidArgumentException; -import alluxio.grpc.Chunk; -import alluxio.grpc.DataMessage; -import alluxio.grpc.ReadResponse; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.network.protocol.databuffer.NettyDataBuffer; -import alluxio.network.protocol.databuffer.PooledDirectNioByteBuf; -import alluxio.resource.LockResource; -import alluxio.util.LogUtils; -import alluxio.util.logging.SamplingLogger; -import alluxio.wire.BlockReadRequest; -import alluxio.worker.block.AllocateOptions; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.BlockStoreType; -import alluxio.worker.block.DefaultBlockWorker; -import alluxio.worker.block.io.BlockReader; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; -import com.google.common.base.Preconditions; -import com.google.protobuf.UnsafeByteOperations; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.internal.SerializingExecutor; -import io.grpc.stub.CallStreamObserver; -import io.grpc.stub.StreamObserver; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; -import io.netty.buffer.Unpooled; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.locks.ReentrantLock; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This class handles {@link BlockReadRequest}s. - * - * Protocol: Check {@link alluxio.client.block.stream.GrpcDataReader} for additional information. - * 1. Once a read request is received, the handler creates a {@link DataReader} which reads - * chunks of data from the block worker and pushes them to the buffer. - * 2. The {@link DataReader} pauses if there are too many packets in flight, and resumes if there - * is room available. - * 3. The channel is closed if there is any exception during the data read/write. - * - * Threading model: - * Only two threads are involved at a given point of time: gRPC event thread, data reader thread. - * 1. The gRPC event thread accepts the read request, handles write callbacks. If any exception - * occurs (e.g. failed to read from stream or respond to stream) or the read request is cancelled - * by the client, the gRPC event thread notifies the data reader thread. - * 2. The data reader thread keeps reading from the file and writes to buffer. Before reading a - * new data chunk, it checks whether there are notifications (e.g. cancel, error), if - * there is, handle them properly. See more information about the notifications in the javadoc - * of {@link BlockReadRequestContext} about CANCEL, EOF, and ERROR flags. - * - * @see BlockReadRequestContext - */ -@NotThreadSafe -public class BlockReadHandler implements StreamObserver { - private static final Logger LOG = LoggerFactory.getLogger(BlockReadHandler.class); - private static final long MAX_CHUNK_SIZE = - Configuration.getBytes(PropertyKey.WORKER_NETWORK_READER_MAX_CHUNK_SIZE_BYTES); - private static final long MAX_BYTES_IN_FLIGHT = - Configuration.getBytes(PropertyKey.WORKER_NETWORK_READER_BUFFER_SIZE_BYTES); - private static final Logger SLOW_BUFFER_LOG = new SamplingLogger(LOG, Constants.MINUTE_MS); - private static final long SLOW_BUFFER_MS = - Configuration.getMs(PropertyKey.WORKER_REMOTE_IO_SLOW_THRESHOLD); - /** Metrics. */ - private static final Counter RPC_READ_COUNT = - MetricsSystem.counterWithTags(MetricKey.WORKER_ACTIVE_RPC_READ_COUNT.getName(), - MetricKey.WORKER_ACTIVE_RPC_READ_COUNT.isClusterAggregated()); - - /** The executor to run {@link DataReader}. */ - private final ExecutorService mDataReaderExecutor; - /** A serializing executor for sending responses. */ - private final Executor mSerializingExecutor; - /** The Block Worker. */ - private final DefaultBlockWorker mWorker; - private final ReentrantLock mLock = new ReentrantLock(); - private final boolean mDomainSocketEnabled; - private final boolean mIsReaderBufferPooled; - - private final BlockStoreType mBlockStoreType; - - /** - * This is only created in the gRPC event thread when a read request is received. - * Using "volatile" because we want any value change of this variable to be - * visible across both gRPC and I/O threads, meanwhile no atomicity of operation is assumed; - */ - private volatile BlockReadRequestContext mContext; - private final StreamObserver mResponseObserver; - - /** - * Creates an instance of {@link BlockReadHandler}. - * - * @param executorService the executor service to run {@link DataReader}s - * @param blockWorker block worker - * @param responseObserver the response observer of the - * @param domainSocketEnabled if domain socket is enabled - */ - BlockReadHandler(ExecutorService executorService, - DefaultBlockWorker blockWorker, - StreamObserver responseObserver, - boolean domainSocketEnabled) { - mDataReaderExecutor = executorService; - mResponseObserver = responseObserver; - mSerializingExecutor = - new SerializingExecutor(GrpcExecutors.BLOCK_READER_SERIALIZED_RUNNER_EXECUTOR); - mWorker = blockWorker; - mDomainSocketEnabled = domainSocketEnabled; - mIsReaderBufferPooled = - Configuration.getBoolean(PropertyKey.WORKER_NETWORK_READER_BUFFER_POOLED); - mBlockStoreType = - Configuration.getEnum(PropertyKey.WORKER_BLOCK_STORE_TYPE, BlockStoreType.class); - } - - @Override - public void onNext(alluxio.grpc.ReadRequest request) { - // Expected state: context equals null as this handler is new for request. - // Otherwise, notify the client an illegal state. Note that, we reset the context before - // validation msg as validation may require to update error in context. - LOG.debug("Received read request {}.", - RpcSensitiveConfigMask.CREDENTIAL_FIELD_MASKER.maskObjects(LOG, request)); - try (LockResource lr = new LockResource(mLock)) { - if (request.hasOffsetReceived()) { - mContext.setPosReceived(request.getOffsetReceived()); - if (!tooManyPendingChunks()) { - onReady(); - } - return; - } - Preconditions.checkState(mContext == null || !mContext.isDataReaderActive()); - mContext = createRequestContext(request); - validateReadRequest(request); - mContext.setPosToQueue(mContext.getRequest().getStart()); - mContext.setPosReceived(mContext.getRequest().getStart()); - mDataReaderExecutor.submit(createDataReader(mContext, mResponseObserver)); - mContext.setDataReaderActive(true); - } catch (RejectedExecutionException e) { - handleStreamEndingException(Status.RESOURCE_EXHAUSTED.withCause(e) - .withDescription("Failed to create a new data reader")); - } catch (Exception e) { - handleStreamEndingException( - AlluxioStatusException.fromThrowable(e).toGrpcStatusException().getStatus()); - } - } - - /** - * Handles any exception which should abort the client's read request. - * - * @param status the type of {@link Status} exception which should be returned to the user - */ - private void handleStreamEndingException(Status status) { - Long sessionId = mContext.getRequest() == null ? -1 : mContext.getRequest().getSessionId(); - LogUtils.warnWithException(LOG, "Error occurred while handling read. sessionId: {}. Ending " - + "stream", - sessionId, status); - AlluxioStatusException statusExc = AlluxioStatusException.from(status); - try (LockResource lr = new LockResource(mLock)) { - if (mContext == null) { - mContext = createRequestContext(alluxio.grpc.ReadRequest.newBuilder().build()); - } - setError(new Error(statusExc, true)); - } - } - - /** - * @return true if there are too many chunks in-flight - */ - @GuardedBy("mLock") - public boolean tooManyPendingChunks() { - return mContext.getPosToQueue() - mContext.getPosReceived() >= MAX_BYTES_IN_FLIGHT; - } - - @Override - public void onError(Throwable cause) { - BlockReadRequest r = mContext == null ? null : mContext.getRequest(); - LogUtils.warnWithException(LOG, "Exception occurred while processing read request onError " - + "sessionId: {}, {}", - r, r == null ? null : r.getSessionId(), cause); - setError(new Error(AlluxioStatusException.fromThrowable(cause), false)); - } - - @Override - public void onCompleted() { - setCancel(); - } - - /** - * Validates a read request. - * - * @param request the block read request - * @throws InvalidArgumentException if the request is invalid - */ - private void validateReadRequest(alluxio.grpc.ReadRequest request) - throws InvalidArgumentException { - if (request.getBlockId() < 0) { - throw new InvalidArgumentException( - String.format("Invalid blockId (%d) in read request.", request.getBlockId())); - } - if (request.getOffset() < 0 || request.getLength() <= 0) { - throw new InvalidArgumentException( - String.format("Invalid read bounds in read request %s.", request)); - } - } - - /** - * @param error the error - */ - private void setError(Error error) { - Preconditions.checkNotNull(error, "error"); - try (LockResource lr = new LockResource(mLock)) { - if (mContext == null || mContext.getError() != null || mContext.isDoneUnsafe()) { - // Note, we may reach here via channelUnregistered due to network errors bubbling up before - // mContext is initialized, or channel garbage collection after the request is finished. - return; - } - mContext.setError(error); - if (!mContext.isDataReaderActive()) { - mContext.setDataReaderActive(true); - createDataReader(mContext, mResponseObserver).run(); - } - } - } - - private void setEof() { - try (LockResource lr = new LockResource(mLock)) { - if (mContext == null || mContext.getError() != null || mContext.isCancel() - || mContext.isEof()) { - return; - } - mContext.setEof(true); - if (!mContext.isDataReaderActive()) { - mContext.setDataReaderActive(true); - createDataReader(mContext, mResponseObserver).run(); - } - } - } - - private void setCancel() { - try (LockResource lr = new LockResource(mLock)) { - if (mContext == null || mContext.getError() != null || mContext.isEof() - || mContext.isCancel()) { - return; - } - mContext.setCancel(true); - if (!mContext.isDataReaderActive()) { - mContext.setDataReaderActive(true); - createDataReader(mContext, mResponseObserver).run(); - } - } - } - - /** - * @param request the block read request - * @return an instance of read request based on the request read from channel - */ - protected BlockReadRequestContext createRequestContext(alluxio.grpc.ReadRequest request) { - BlockReadRequestContext context = new BlockReadRequestContext(request); - if (mDomainSocketEnabled) { - context.setCounter(MetricsSystem.counter(MetricKey.WORKER_BYTES_READ_DOMAIN.getName())); - context.setMeter(MetricsSystem - .meter(MetricKey.WORKER_BYTES_READ_DOMAIN_THROUGHPUT.getName())); - } else { - context.setCounter(MetricsSystem.counter(MetricKey.WORKER_BYTES_READ_REMOTE.getName())); - context.setMeter(MetricsSystem - .meter(MetricKey.WORKER_BYTES_READ_REMOTE_THROUGHPUT.getName())); - } - RPC_READ_COUNT.inc(); - return context; - } - - /** - * Creates a read reader. - * - * @param context read request context - * @param response channel - * @return the data reader for this handler - */ - private DataReader createDataReader(BlockReadRequestContext context, - StreamObserver response) { - return new DataReader(context, response); - } - - /** - * Ready to restart data reader. - */ - public void onReady() { - try (LockResource lr = new LockResource(mLock)) { - if (shouldRestartDataReader()) { - try { - mDataReaderExecutor.submit(createDataReader(mContext, mResponseObserver)); - mContext.setDataReaderActive(true); - } catch (RejectedExecutionException e) { - handleStreamEndingException(Status.RESOURCE_EXHAUSTED.withCause(e) - .withDescription("Failed to create a new data reader")); - } - } - } - } - - /** - * @return true if we should restart the data reader - */ - @GuardedBy("mLock") - private boolean shouldRestartDataReader() { - return mContext != null && !mContext.isDataReaderActive() - && mContext.getPosToQueue() < mContext.getRequest().getEnd() - && mContext.getError() == null && !mContext.isCancel() && !mContext.isEof(); - } - - /** - * @param bytesRead bytes read - */ - private void incrementMetrics(long bytesRead) { - Counter counter = mContext.getCounter(); - Meter meter = mContext.getMeter(); - Preconditions.checkState(counter != null); - counter.inc(bytesRead); - meter.mark(bytesRead); - } - - /** - * A runnable that reads data and writes them to the channel. - */ - private class DataReader implements Runnable { - private final CallStreamObserver mResponse; - private final BlockReadRequestContext mContext; - private final BlockReadRequest mRequest; - private final long mChunkSize; - - /** - * Creates an instance of the {@link DataReader}. - * - * @param context context of the request to complete - * @param response the response - */ - DataReader(BlockReadRequestContext context, StreamObserver response) { - mContext = Preconditions.checkNotNull(context); - mRequest = Preconditions.checkNotNull(context.getRequest()); - mChunkSize = Math.min(mRequest.getChunkSize(), MAX_CHUNK_SIZE); - mResponse = (CallStreamObserver) response; - } - - @Override - public void run() { - try { - runInternal(); - } catch (Throwable e) { - LOG.error("Failed to run DataReader.", e); - throw e; - } - } - - private void runInternal() { - boolean eof; // End of file. Everything requested has been read. - boolean cancel; - Error error; // error occurred, abort requested. - while (true) { - final long start; - final int chunkSize; - try (LockResource lr = new LockResource(mLock)) { - if (mContext.isDoneUnsafe()) { - return; - } - start = mContext.getPosToQueue(); - eof = mContext.isEof(); - cancel = mContext.isCancel(); - error = mContext.getError(); - - if (eof || cancel || error != null || (!mResponse.isReady() && tooManyPendingChunks())) { - mContext.setDataReaderActive(false); - break; - } - chunkSize = (int) Math.min(mRequest.getEnd() - mContext.getPosToQueue(), mChunkSize); - - // chunkSize should always be > 0 here when reaches here. - Preconditions.checkState(chunkSize > 0); - } - - DataBuffer chunk; - try { - // Once we get the data buffer, the lock on the block has been acquired. - // If there are any stream errors during this time, we must unlock the block - // before exiting. - chunk = getDataBuffer(mContext, start, chunkSize); - if (chunk != null) { - try (LockResource lr = new LockResource(mLock)) { - mContext.setPosToQueue(mContext.getPosToQueue() + chunk.getLength()); - } - } - if (chunk == null || chunk.getLength() < chunkSize || start + chunkSize == mRequest - .getEnd()) { - // This can happen if the requested read length is greater than the actual length of the - // block or file starting from the given offset. - setEof(); - } - - if (chunk != null) { - DataBuffer finalChunk = chunk; - mSerializingExecutor.execute(() -> { - try { - ReadResponse response = ReadResponse.newBuilder().setChunk(Chunk.newBuilder() - .setData(UnsafeByteOperations.unsafeWrap(finalChunk.getReadOnlyByteBuffer())) - ).build(); - if (mResponse instanceof DataMessageServerStreamObserver) { - ((DataMessageServerStreamObserver) mResponse) - .onNext(new DataMessage<>(response, finalChunk)); - } else { - mResponse.onNext(response); - } - incrementMetrics(finalChunk.getLength()); - } catch (Exception e) { - LogUtils.warnWithException(LOG, - "Exception occurred while sending data for read request {}.", - mContext.getRequest(), e); - setError(new Error(AlluxioStatusException.fromThrowable(e), true)); - } finally { - finalChunk.release(); - } - }); - } - } catch (Throwable e) { - if (isFatalError(e)) { - ProcessUtils.fatalError(LOG, e, "Error while reading"); - } - LogUtils.warnWithException(LOG, - "Exception occurred while reading data for read request {}. session {}", - mContext.getRequest(), mContext.getRequest().getSessionId(), - e); - setError(new Error(AlluxioStatusException.fromThrowable(e), true)); - } - continue; - } - if (error != null) { - try { - completeRequest(mContext); - } catch (Exception e) { - LOG.error("Failed to close the request.", e); - } - replyError(error); - } else if (eof || cancel) { - try { - completeRequest(mContext); - } catch (Exception e) { - LogUtils.warnWithException(LOG, "Exception occurred while completing read request, " - + "EOF/CANCEL sessionId: {}. {}", mContext.getRequest().getSessionId(), - mContext.getRequest(), e); - setError(new Error(AlluxioStatusException.fromThrowable(e), true)); - } - if (eof) { - replyEof(); - } else { - replyCancel(); - } - } - } - - /** - * Completes the read request. When the request is closed, we should clean up any temporary - * state it may have accumulated. - * - * @param context context of the request to complete - */ - private void completeRequest(BlockReadRequestContext context) throws Exception { - BlockReader reader = context.getBlockReader(); - try { - if (reader != null) { - reader.close(); - } - } finally { - context.setBlockReader(null); - RPC_READ_COUNT.dec(); - } - } - - /** - * Returns the appropriate {@link DataBuffer} representing the data to send, depending on the - * configurable transfer type. - * - * @param context context of the request to complete - * @param len The length, in bytes, of the data to read from the block - * @return a {@link DataBuffer} representing the data - */ - protected DataBuffer getDataBuffer(BlockReadRequestContext context, long offset, int len) - throws Exception { - @Nullable - BlockReader blockReader = null; - // timings - long openMs = -1; - long startTransferMs = -1; - long startMs = System.currentTimeMillis(); - try { - openBlock(context); - openMs = System.currentTimeMillis() - startMs; - blockReader = context.getBlockReader(); - Preconditions.checkState(blockReader != null); - startTransferMs = System.currentTimeMillis(); - ByteBuf buf; - switch (mBlockStoreType) { - case PAGE: - if (mIsReaderBufferPooled) { - buf = PooledDirectNioByteBuf.allocate(len); - } else { - buf = Unpooled.directBuffer(len, len); - } - try { - while (buf.writableBytes() > 0 && blockReader.transferTo(buf) != -1) { - } - return new NettyDataBuffer(buf.retain()); - } finally { - buf.release(); - } - case FILE: - //TODO(beinan): change the blockReader interface to accept pre-allocated byte buffer - // or accept a supplier of the bytebuffer. - if (mIsReaderBufferPooled) { - buf = PooledByteBufAllocator.DEFAULT.buffer(len, len); - try { - while (buf.writableBytes() > 0 && blockReader.transferTo(buf) != -1) { - } - return new NettyDataBuffer(buf.retain()); - } finally { - buf.release(); - } - } else { - ByteBuffer buffer = blockReader.read(offset, len); - return new NettyDataBuffer(Unpooled.wrappedBuffer(buffer)); - } - default: - throw new InvalidArgumentException("Unsupported block store type:" + mBlockStoreType); - } - } finally { - long transferMs = System.currentTimeMillis() - startTransferMs; - long durationMs = System.currentTimeMillis() - startMs; - if (durationMs >= SLOW_BUFFER_MS) { - // This buffer took much longer than expected - String prefix = String - .format("Getting buffer for remote read took longer than %s ms. ", SLOW_BUFFER_MS) - + "reader: " + (blockReader == null ? "null" : blockReader.getClass().getName()); - - String location = blockReader == null ? "null" : blockReader.getLocation(); - - // Do not template the reader class, so the sampling log can distinguish between - // different reader types - SLOW_BUFFER_LOG.warn(prefix - + " location: {} bytes: {} openMs: {} transferMs: {} durationMs: {}", - location, len, openMs, transferMs, durationMs); - } - } - } - - /** - * Opens the block if it is not open. - * - * @throws Exception if it fails to open the block - */ - private void openBlock(BlockReadRequestContext context) - throws Exception { - if (context.getBlockReader() != null) { - return; - } - BlockReadRequest request = context.getRequest(); - // TODO(calvin): Update the locking logic so this can be done better - if (request.isPromote()) { - try { - mWorker.getBlockStore() - .moveBlock(request.getSessionId(), request.getId(), - AllocateOptions.forMove(BlockStoreLocation.anyDirInTier( - WORKER_STORAGE_TIER_ASSOC.getAlias(0)))); - } catch (Exception e) { - LOG.warn("Failed to promote block {}: {}", request.getId(), e.toString()); - } - } - BlockReader reader = mWorker.createBlockReader(request.getSessionId(), request.getId(), - request.getStart(), request.isPositionShort(), request.getOpenUfsBlockOptions()); - context.setBlockReader(reader); - } - - /** - * Writes an error read response to the channel and closes the channel after that. - */ - private void replyError(Error error) { - mSerializingExecutor.execute(() -> { - try { - if (!mContext.isDoneUnsafe()) { - mResponse.onError(error.getCause().toGrpcStatusException()); - mContext.setDoneUnsafe(true); - } else { - LOG.debug("Tried to replyError when stream was already completed. context: {}", - mContext); - } - } catch (StatusRuntimeException e) { - // Ignores the error when client already closed the stream. - if (e.getStatus().getCode() != Status.Code.CANCELLED) { - throw e; - } - } - }); - } - - /** - * Writes a success response. - */ - private void replyEof() { - mSerializingExecutor.execute(() -> { - try { - if (!mContext.isDoneUnsafe()) { - mContext.setDoneUnsafe(true); - mResponse.onCompleted(); - } else { - LOG.debug("Tried to replyEof when stream was already finished. context: {}", mContext); - } - } catch (StatusRuntimeException e) { - if (e.getStatus().getCode() != Status.Code.CANCELLED) { - throw e; - } - } - }); - } - - /** - * Writes a cancel response. - */ - private void replyCancel() { - mSerializingExecutor.execute(() -> { - try { - if (!mContext.isDoneUnsafe()) { - mContext.setDoneUnsafe(true); - mResponse.onCompleted(); - } else { - LOG.debug("Tried to replyCancel when stream was already finished. context: {}", - mContext); - } - } catch (StatusRuntimeException e) { - if (e.getStatus().getCode() != Status.Code.CANCELLED) { - throw e; - } - } - }); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockReadRequestContext.java b/core/server/worker/src/main/java/alluxio/worker/grpc/BlockReadRequestContext.java deleted file mode 100644 index 806255476cd0..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockReadRequestContext.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.wire.BlockReadRequest; -import alluxio.worker.block.io.BlockReader; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; - -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Context of {@link BlockReadRequest}. - */ -@NotThreadSafe -public final class BlockReadRequestContext { - /** - * Set to true if the data reader is active. The following invariants must be maintained: - * 1. If true, there will be at least one more message (data, eof or error) to be sent to gRPC. - * 2. If false, there will be no more message sent to gRPC until it is set to true again. - */ - private boolean mDataReaderActive; - /** - * The next pos to queue to the read buffer. - */ - private long mPosToQueue; - - private long mPosReceived; - /** - * mEof, mCancel and mError are the notifications processed by the data reader thread. They can - * be set by either the gRPC I/O thread or the data reader thread. mError overrides mCancel - * and mEof, mEof overrides mCancel. - * - * These notifications determine 3 ways to complete a read request. - * 1. mEof: The read request is fulfilled. All the data requested by the client or all the data in - * the block/file has been read. The data reader replies a SUCCESS response when processing - * mEof. - * 2. mCancel: The read request is cancelled by the client. A cancel request is ignored if mEof - * is set. The data reader replies a CANCEL response when processing mCancel. - * Note: The client can send a cancel request after the server has sent a SUCCESS response. But - * it is not possible for the client to send a CANCEL request after the channel has been - * released. So it is impossible for a CANCEL request from one read request to cancel - * another read request. - * 3. mError: mError is set whenever an error occurs. It can be from an exception when reading - * data, or writing data to stream or the client closes the channel etc. An ERROR response - * is optionally sent to the client when data reader thread process mError. The channel - * is closed after this error response is sent. - * - * Note: it is guaranteed that only one of SUCCESS and CANCEL responses is sent at most once - * because the data reader thread won't be restarted as long as mCancel or mEof is set except - * when error happens (mError overrides mCancel and mEof). - */ - private boolean mEof; - private boolean mCancel; - private Error mError; - private Counter mCounter; - private Meter mMeter; - /** This is set when the SUCCESS or CANCEL response is sent. This is only for sanity check. */ - private volatile boolean mDone; - private BlockReader mBlockReader; - /** The requests of this context. */ - private final BlockReadRequest mRequest; - - /** - * @param request the read request - */ - public BlockReadRequestContext(alluxio.grpc.ReadRequest request) { - mRequest = BlockReadRequest.from(request); - mPosToQueue = 0; - mDataReaderActive = false; - mEof = false; - mCancel = false; - mError = null; - mDone = false; - } - - /** - * @return request received from channel - */ - public BlockReadRequest getRequest() { - return mRequest; - } - - /** - * @return whether the data reader is active - */ - @GuardedBy("BlockReadHandler#mLock") - public boolean isDataReaderActive() { - return mDataReaderActive; - } - - /** - * @return the next position to queue to the read buffer - */ - @GuardedBy("BlockReadHandler#mLock") - public long getPosToQueue() { - return mPosToQueue; - } - - /** - * @return the position before which data are received by the client - */ - @GuardedBy("BlockReadHandler#mLock") - public long getPosReceived() { - return mPosReceived; - } - - /** - * @return true when the data reader replies a SUCCESS response, false otherwise - */ - @GuardedBy("BlockReadHandler#mLock") - public boolean isEof() { - return mEof; - } - - /** - * @return true when a CANCEL request is received by the client, false otherwise - */ - @GuardedBy("BlockReadHandler#mLock") - public boolean isCancel() { - return mCancel; - } - - /** - * @return the error during this read request - */ - @GuardedBy("BlockReadHandler#mLock") - @Nullable - public Error getError() { - return mError; - } - - /** - * @return true when the SUCCESS or CANCEL response is sent, false otherwise - */ - public boolean isDoneUnsafe() { - return mDone; - } - - /** - * @return metrics counter associated with this request - */ - @Nullable - public Counter getCounter() { - return mCounter; - } - - /** - * @return metrics meter associated with this request - */ - @Nullable - public Meter getMeter() { - return mMeter; - } - - /** - * @param packetReaderActive packet reader state to set - */ - @GuardedBy("BlockReadHandler#mLock") - public void setDataReaderActive(boolean packetReaderActive) { - mDataReaderActive = packetReaderActive; - } - - /** - * @param posToQueue the next position to queue to the netty buffer to set - */ - @GuardedBy("BlockReadHandler#mLock") - public void setPosToQueue(long posToQueue) { - mPosToQueue = posToQueue; - } - - /** - * @param posReceived the position before which data are received by the client - */ - @GuardedBy("BlockReadHandler#mLock") - public void setPosReceived(long posReceived) { - mPosReceived = posReceived; - } - - /** - * @param eof whether SUCCESS response is replied - */ - @GuardedBy("BlockReadHandler#mLock") - public void setEof(boolean eof) { - mEof = eof; - } - - /** - * @param cancel whether the CANCEL request is received - */ - @GuardedBy("BlockReadHandler#mLock") - public void setCancel(boolean cancel) { - mCancel = cancel; - } - - /** - * @param error the error - */ - @GuardedBy("BlockReadHandler#mLock") - public void setError(Error error) { - mError = error; - } - - /** - * @param done whether the SUCCESS or CANCEL response is sent - */ - public void setDoneUnsafe(boolean done) { - mDone = done; - } - - /** - * @param counter counter to set - */ - public void setCounter(Counter counter) { - mCounter = counter; - } - - /** - * @param meter meter to set - */ - public void setMeter(Meter meter) { - mMeter = meter; - } - - /** - * @return block reader - */ - @Nullable - public BlockReader getBlockReader() { - return mBlockReader; - } - - /** - * @param blockReader block reader to set - */ - public void setBlockReader(BlockReader blockReader) { - mBlockReader = blockReader; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWorkerClientServiceHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWorkerClientServiceHandler.java deleted file mode 100644 index 5162fbb7c246..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWorkerClientServiceHandler.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.RpcUtils; -import alluxio.annotation.SuppressFBWarnings; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.AsyncCacheRequest; -import alluxio.grpc.AsyncCacheResponse; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.BlockWorkerGrpc; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.CacheResponse; -import alluxio.grpc.ClearMetricsRequest; -import alluxio.grpc.ClearMetricsResponse; -import alluxio.grpc.CreateLocalBlockRequest; -import alluxio.grpc.CreateLocalBlockResponse; -import alluxio.grpc.FreeWorkerRequest; -import alluxio.grpc.FreeWorkerResponse; -import alluxio.grpc.LoadRequest; -import alluxio.grpc.LoadResponse; -import alluxio.grpc.MoveBlockRequest; -import alluxio.grpc.MoveBlockResponse; -import alluxio.grpc.OpenLocalBlockRequest; -import alluxio.grpc.OpenLocalBlockResponse; -import alluxio.grpc.ReadRequest; -import alluxio.grpc.ReadResponse; -import alluxio.grpc.ReadResponseMarshaller; -import alluxio.grpc.RemoveBlockRequest; -import alluxio.grpc.RemoveBlockResponse; -import alluxio.grpc.TaskStatus; -import alluxio.grpc.WriteRequestMarshaller; -import alluxio.grpc.WriteResponse; -import alluxio.security.authentication.AuthenticatedClientUser; -import alluxio.security.authentication.AuthenticatedUserInfo; -import alluxio.underfs.UfsManager; -import alluxio.util.IdUtils; -import alluxio.util.SecurityUtils; -import alluxio.worker.WorkerProcess; -import alluxio.worker.block.AllocateOptions; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.BlockWorker; -import alluxio.worker.block.DefaultBlockWorker; - -import com.google.common.collect.ImmutableMap; -import io.grpc.MethodDescriptor; -import io.grpc.Status; -import io.grpc.stub.CallStreamObserver; -import io.grpc.stub.ServerCallStreamObserver; -import io.grpc.stub.StreamObserver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - -/** - * Server side implementation of the gRPC BlockWorker interface. - */ -@SuppressFBWarnings("BC_UNCONFIRMED_CAST") -public class BlockWorkerClientServiceHandler extends BlockWorkerGrpc.BlockWorkerImplBase { - private static final Logger LOG = LoggerFactory.getLogger(BlockWorkerClientServiceHandler.class); - private static final boolean ZERO_COPY_ENABLED = - Configuration.getBoolean(PropertyKey.WORKER_NETWORK_ZEROCOPY_ENABLED); - private final DefaultBlockWorker mBlockWorker; - private final UfsManager mUfsManager; - private final ReadResponseMarshaller mReadResponseMarshaller = new ReadResponseMarshaller(); - private final WriteRequestMarshaller mWriteRequestMarshaller = new WriteRequestMarshaller(); - private final boolean mDomainSocketEnabled; - - /** - * Creates a new implementation of gRPC BlockWorker interface. - * - * @param workerProcess the worker process - * @param domainSocketEnabled is using domain sockets - */ - public BlockWorkerClientServiceHandler(WorkerProcess workerProcess, - boolean domainSocketEnabled) { - mBlockWorker = (DefaultBlockWorker) workerProcess.getWorker(BlockWorker.class); - mUfsManager = workerProcess.getUfsManager(); - mDomainSocketEnabled = domainSocketEnabled; - } - - /** - * @return a map of gRPC methods with overridden descriptors - */ - public Map getOverriddenMethodDescriptors() { - if (ZERO_COPY_ENABLED) { - return ImmutableMap.of( - BlockWorkerGrpc.getReadBlockMethod(), - BlockWorkerGrpc.getReadBlockMethod().toBuilder() - .setResponseMarshaller(mReadResponseMarshaller).build(), - BlockWorkerGrpc.getWriteBlockMethod(), - BlockWorkerGrpc.getWriteBlockMethod().toBuilder() - .setRequestMarshaller(mWriteRequestMarshaller).build()); - } - return Collections.emptyMap(); - } - - @Override - public StreamObserver readBlock(StreamObserver responseObserver) { - CallStreamObserver callStreamObserver = - (CallStreamObserver) responseObserver; - if (ZERO_COPY_ENABLED) { - callStreamObserver = - new DataMessageServerStreamObserver<>(callStreamObserver, mReadResponseMarshaller); - } - BlockReadHandler readHandler = new BlockReadHandler(GrpcExecutors.BLOCK_READER_EXECUTOR, - mBlockWorker, callStreamObserver, mDomainSocketEnabled); - callStreamObserver.setOnReadyHandler(readHandler::onReady); - return readHandler; - } - - @Override - public StreamObserver writeBlock( - StreamObserver responseObserver) { - ServerCallStreamObserver serverResponseObserver = - (ServerCallStreamObserver) responseObserver; - if (ZERO_COPY_ENABLED) { - responseObserver = - new DataMessageServerRequestObserver<>(responseObserver, mWriteRequestMarshaller, null); - } - DelegationWriteHandler handler = new DelegationWriteHandler(mBlockWorker, mUfsManager, - responseObserver, getAuthenticatedUserInfo(), mDomainSocketEnabled); - serverResponseObserver.setOnCancelHandler(handler::onCancel); - return handler; - } - - @Override - public StreamObserver openLocalBlock( - StreamObserver responseObserver) { - return new ShortCircuitBlockReadHandler( - mBlockWorker.getBlockStore(), responseObserver); - } - - @Override - public StreamObserver createLocalBlock( - StreamObserver responseObserver) { - ShortCircuitBlockWriteHandler handler = new ShortCircuitBlockWriteHandler( - mBlockWorker, responseObserver); - ServerCallStreamObserver serverCallStreamObserver = - (ServerCallStreamObserver) responseObserver; - serverCallStreamObserver.setOnCancelHandler(handler::onCancel); - return handler; - } - - @Override - public void asyncCache(AsyncCacheRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, () -> { - mBlockWorker.asyncCache(request); - return AsyncCacheResponse.getDefaultInstance(); - }, "asyncCache", "request=%s", responseObserver, request); - } - - @Override - public void cache(CacheRequest request, StreamObserver responseObserver) { - RpcUtils.call(LOG, () -> { - mBlockWorker.cache(request); - return CacheResponse.getDefaultInstance(); - }, "cache", "request=%s", responseObserver, request); - } - - @Override - public void load(LoadRequest request, StreamObserver responseObserver) { - CompletableFuture> failures = - mBlockWorker.load(request.getBlocksList(), request.getOptions()); - CompletableFuture future = failures.thenApply(fail -> { - int numBlocks = request.getBlocksCount(); - TaskStatus taskStatus = TaskStatus.SUCCESS; - if (fail.size() > 0) { - taskStatus = numBlocks > fail.size() ? TaskStatus.PARTIAL_FAILURE : TaskStatus.FAILURE; - } - LoadResponse.Builder response = LoadResponse.newBuilder(); - return response.addAllBlockStatus(fail).setStatus(taskStatus).build(); - }); - RpcUtils.invoke(LOG, future, "load", "request=%s", responseObserver, request); - } - - @Override - public void removeBlock(RemoveBlockRequest request, - StreamObserver responseObserver) { - long sessionId = IdUtils.createSessionId(); - RpcUtils.call(LOG, () -> { - mBlockWorker.removeBlock(sessionId, request.getBlockId()); - return RemoveBlockResponse.getDefaultInstance(); - }, "removeBlock", "request=%s", responseObserver, request); - } - - @Override - public void moveBlock(MoveBlockRequest request, - StreamObserver responseObserver) { - long sessionId = IdUtils.createSessionId(); - RpcUtils.call(LOG, () -> { - mBlockWorker.getBlockStore() - .moveBlock(sessionId, request.getBlockId(), - AllocateOptions.forMove( - BlockStoreLocation.anyDirInAnyTierWithMedium(request.getMediumType()))); - return MoveBlockResponse.getDefaultInstance(); - }, "moveBlock", "request=%s", responseObserver, request); - } - - @Override - public void freeWorker(FreeWorkerRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, () -> { - mBlockWorker.freeWorker(); - return FreeWorkerResponse.getDefaultInstance(); - }, "freeWorker", "request=%s", responseObserver, request); - } - - @Override - public void clearMetrics(ClearMetricsRequest request, - StreamObserver responseObserver) { - RpcUtils.call(LOG, () -> { - mBlockWorker.clearMetrics(); - return ClearMetricsResponse.getDefaultInstance(); - }, "clearMetrics", "request=%s", responseObserver, request); - } - - /** - * @return {@link AuthenticatedUserInfo} that defines the user that has been authorized - */ - private AuthenticatedUserInfo getAuthenticatedUserInfo() { - try { - if (SecurityUtils.isAuthenticationEnabled(Configuration.global())) { - return new AuthenticatedUserInfo( - AuthenticatedClientUser.getClientUser(Configuration.global()), - AuthenticatedClientUser.getConnectionUser(Configuration.global()), - AuthenticatedClientUser.getAuthMethod(Configuration.global())); - } else { - return new AuthenticatedUserInfo(); - } - } catch (Exception e) { - throw Status.UNAUTHENTICATED.withDescription(e.toString()).asRuntimeException(); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteHandler.java deleted file mode 100644 index c6d5148d54bf..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteHandler.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.grpc.WriteResponse; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.security.authentication.AuthenticatedUserInfo; -import alluxio.worker.block.BlockWorker; -import alluxio.worker.block.CreateBlockOptions; - -import com.codahale.metrics.Counter; -import com.google.common.base.Preconditions; -import io.grpc.stub.StreamObserver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This handler handles block write request. Check more information in - * {@link AbstractWriteHandler}. - */ -@alluxio.annotation.SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", - justification = "false positive with superclass generics, " - + "see more description in https://sourceforge.net/p/findbugs/bugs/1242/") -@NotThreadSafe -public final class BlockWriteHandler extends AbstractWriteHandler { - private static final Logger LOG = LoggerFactory.getLogger(BlockWriteHandler.class); - /** Metrics. */ - private static final Counter RPC_WRITE_COUNT = - MetricsSystem.counterWithTags(MetricKey.WORKER_ACTIVE_RPC_WRITE_COUNT.getName(), - MetricKey.WORKER_ACTIVE_RPC_WRITE_COUNT.isClusterAggregated()); - - /** The Block Worker which handles blocks stored in the Alluxio storage of the worker. */ - private final BlockWorker mWorker; - - private final boolean mDomainSocketEnabled; - - /** - * Creates an instance of {@link BlockWriteHandler}. - * - * @param blockWorker the block worker - * @param responseObserver the stream observer for the write response - * @param userInfo the authenticated user info - * @param domainSocketEnabled whether reading block over domain socket - */ - BlockWriteHandler(BlockWorker blockWorker, StreamObserver responseObserver, - AuthenticatedUserInfo userInfo, boolean domainSocketEnabled) { - super(responseObserver, userInfo); - mWorker = blockWorker; - mDomainSocketEnabled = domainSocketEnabled; - } - - @Override - protected BlockWriteRequestContext createRequestContext(alluxio.grpc.WriteRequest msg) { - long bytesToReserve = FILE_BUFFER_SIZE; - if (msg.getCommand().hasSpaceToReserve()) { - bytesToReserve = msg.getCommand().getSpaceToReserve(); - } - BlockWriteRequestContext context = new BlockWriteRequestContext(msg, bytesToReserve); - BlockWriteRequest request = context.getRequest(); - mWorker.createBlock(request.getSessionId(), request.getId(), request.getTier(), - new CreateBlockOptions(null, request.getMediumType(), bytesToReserve)); - if (mDomainSocketEnabled) { - context.setCounter(MetricsSystem.counter(MetricKey.WORKER_BYTES_WRITTEN_DOMAIN.getName())); - context.setMeter(MetricsSystem.meter( - MetricKey.WORKER_BYTES_WRITTEN_DOMAIN_THROUGHPUT.getName())); - } else { - context.setCounter(MetricsSystem.counter(MetricKey.WORKER_BYTES_WRITTEN_REMOTE.getName())); - context.setMeter(MetricsSystem.meter( - MetricKey.WORKER_BYTES_WRITTEN_REMOTE_THROUGHPUT.getName())); - } - RPC_WRITE_COUNT.inc(); - return context; - } - - @Override - protected void completeRequest(BlockWriteRequestContext context) throws Exception { - WriteRequest request = context.getRequest(); - if (context.getBlockWriter() != null) { - context.getBlockWriter().close(); - } - mWorker.commitBlock(request.getSessionId(), request.getId(), request.getPinOnCreate()); - RPC_WRITE_COUNT.dec(); - } - - @Override - protected void cancelRequest(BlockWriteRequestContext context) throws Exception { - WriteRequest request = context.getRequest(); - if (context.getBlockWriter() != null) { - context.getBlockWriter().close(); - } - mWorker.abortBlock(request.getSessionId(), request.getId()); - RPC_WRITE_COUNT.dec(); - } - - @Override - protected void cleanupRequest(BlockWriteRequestContext context) throws Exception { - if (context.getBlockWriter() != null) { - context.getBlockWriter().close(); - } - mWorker.cleanupSession(context.getRequest().getSessionId()); - - // Decrement RPC counter only if the request wasn't completed/canceled already - if (!context.isDoneUnsafe()) { - RPC_WRITE_COUNT.dec(); - } - } - - @Override - protected void flushRequest(BlockWriteRequestContext context) { - // This is a no-op because block worker does not support flush currently. - } - - @Override - protected void writeBuf(BlockWriteRequestContext context, - StreamObserver observer, DataBuffer buf, long pos) throws Exception { - Preconditions.checkState(context != null); - WriteRequest request = context.getRequest(); - long bytesReserved = context.getBytesReserved(); - if (bytesReserved < pos) { - long bytesToReserve = Math.max(FILE_BUFFER_SIZE, pos - bytesReserved); - // Allocate enough space in the existing temporary block for the write. - mWorker.requestSpace(request.getSessionId(), request.getId(), bytesToReserve); - context.setBytesReserved(bytesReserved + bytesToReserve); - } - if (context.getBlockWriter() == null) { - context.setBlockWriter( - mWorker.createBlockWriter(request.getSessionId(), request.getId())); - } - Preconditions.checkState(context.getBlockWriter() != null); - int sz = buf.readableBytes(); - Preconditions.checkState(context.getBlockWriter().append(buf) == sz); - } - - @Override - protected String getLocationInternal(BlockWriteRequestContext context) { - return String.format("temp-block-session-%d-id-%d", context.getRequest().getSessionId(), - context.getRequest().getId()); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteRequest.java b/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteRequest.java deleted file mode 100644 index fb863c6c6ad7..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteRequest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.proto.dataserver.Protocol; - -import com.google.common.base.MoreObjects; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * The block write request internal representation. - */ -@ThreadSafe -public final class BlockWriteRequest extends WriteRequest { - /** Which tier this block writes to. */ - private final int mTier; - private final String mMediumType; - private final Protocol.CreateUfsBlockOptions mCreateUfsBlockOptions; - - /** - * @param request block request in proto - */ - BlockWriteRequest(alluxio.grpc.WriteRequest request) { - super(request); - mTier = request.getCommand().getTier(); - mMediumType = request.getCommand().getMediumType(); - if (request.getCommand().hasCreateUfsBlockOptions()) { - mCreateUfsBlockOptions = request.getCommand().getCreateUfsBlockOptions(); - } else { - mCreateUfsBlockOptions = null; - } - } - - /** - * @return the tier this block is writing to - */ - public int getTier() { - return mTier; - } - - /** - * @return the medium type this block is writing to - */ - public String getMediumType() { - return mMediumType; - } - - /** - * @return the options to create blocks in UFS - */ - @javax.annotation.Nullable - public Protocol.CreateUfsBlockOptions getCreateUfsBlockOptions() { - return mCreateUfsBlockOptions; - } - - /** - * @return whether the request has the options to create blocks in UFS - */ - public boolean hasCreateUfsBlockOptions() { - return mCreateUfsBlockOptions != null; - } - - @Override - protected MoreObjects.ToStringHelper toStringHelper() { - return super.toStringHelper().add("tier", mTier) - .add("createUfsBlockOptions", mCreateUfsBlockOptions); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteRequestContext.java b/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteRequestContext.java deleted file mode 100644 index 171ac61eaaef..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/BlockWriteRequestContext.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.worker.block.io.BlockWriter; - -import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * The block write request internal representation. - */ -@NotThreadSafe -public final class BlockWriteRequestContext extends WriteRequestContext { - private BlockWriter mBlockWriter; - private long mBytesReserved; - private boolean mIsWritingToLocal = true; - private alluxio.resource.CloseableResource mUfsResource; - private java.io.OutputStream mOutputStream; - private String mUfsPath; - - BlockWriteRequestContext(alluxio.grpc.WriteRequest request, long bytesReserved) { - super(new BlockWriteRequest(request)); - mBytesReserved = bytesReserved; - } - - /** - * @return the block writer - */ - @Nullable - public BlockWriter getBlockWriter() { - return mBlockWriter; - } - - /** - * @return the bytes reserved - */ - public long getBytesReserved() { - return mBytesReserved; - } - - /** - * @param blockWriter block writer to set - */ - public void setBlockWriter(BlockWriter blockWriter) { - mBlockWriter = blockWriter; - } - - /** - * @param bytesReserved the bytes reserved to set - */ - public void setBytesReserved(long bytesReserved) { - mBytesReserved = bytesReserved; - } - - /** - * @return is the current request writing to UFS - */ - public boolean isWritingToLocal() { - return mIsWritingToLocal; - } - - /** - * @return the UFS path of the block - */ - @Nullable - public String getUfsPath() { - return mUfsPath; - } - - /** - * @return the output stream - */ - @Nullable - public java.io.OutputStream getOutputStream() { - return mOutputStream; - } - - /** - * @return the handler of the UFS - */ - @Nullable - public alluxio.resource.CloseableResource getUfsResource() { - return mUfsResource; - } - - /** - * @param writingToLocal whether the current request is writing to UFS - */ - public void setWritingToLocal(boolean writingToLocal) { - mIsWritingToLocal = writingToLocal; - } - - /** - * @param outputStream output stream to set - */ - public void setOutputStream(java.io.OutputStream outputStream) { - mOutputStream = outputStream; - } - - /** - * @param ufsResource UFS to set - */ - public void setUfsResource( - alluxio.resource.CloseableResource ufsResource) { - mUfsResource = ufsResource; - } - - /** - * @param ufsPath UFS path to set - */ - public void setUfsPath(String ufsPath) { - mUfsPath = ufsPath; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/DelegationWriteHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/DelegationWriteHandler.java deleted file mode 100644 index a1b55168e6db..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/DelegationWriteHandler.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.grpc.DataMessageMarshaller; -import alluxio.grpc.DataMessageMarshallerProvider; -import alluxio.grpc.WriteRequest; -import alluxio.grpc.WriteResponse; -import alluxio.security.authentication.AuthenticatedUserInfo; -import alluxio.underfs.UfsManager; -import alluxio.worker.block.DefaultBlockWorker; - -import io.grpc.stub.StreamObserver; - -/** - * A write request handler that delegate worker write request of different types - * to corresponding write handlers. - */ -public class DelegationWriteHandler implements StreamObserver { - private final StreamObserver mResponseObserver; - private final DefaultBlockWorker mBlockWorker; - private final UfsManager mUfsManager; - private final DataMessageMarshaller mMarshaller; - private AbstractWriteHandler mWriteHandler; - private final AuthenticatedUserInfo mUserInfo; - private final boolean mDomainSocketEnabled; - - /** - * @param blockWorker the block worker instance - * @param ufsManager the UFS manager - * @param responseObserver the response observer of the gRPC stream - * @param userInfo the authenticated user info - * @param domainSocketEnabled whether using a domain socket - */ - public DelegationWriteHandler(DefaultBlockWorker blockWorker, UfsManager ufsManager, - StreamObserver responseObserver, AuthenticatedUserInfo userInfo, - boolean domainSocketEnabled) { - mBlockWorker = blockWorker; - mUfsManager = ufsManager; - mResponseObserver = responseObserver; - mUserInfo = userInfo; - if (mResponseObserver instanceof DataMessageMarshallerProvider) { - mMarshaller = ((DataMessageMarshallerProvider) mResponseObserver) - .getRequestMarshaller().orElse(null); - } else { - mMarshaller = null; - } - mDomainSocketEnabled = domainSocketEnabled; - } - - private AbstractWriteHandler createWriterHandler(alluxio.grpc.WriteRequest request) { - switch (request.getCommand().getType()) { - case ALLUXIO_BLOCK: - return new BlockWriteHandler(mBlockWorker, mResponseObserver, - mUserInfo, mDomainSocketEnabled); - case UFS_FILE: - return new UfsFileWriteHandler(mUfsManager, mResponseObserver, - mUserInfo); - case UFS_FALLBACK_BLOCK: - return new UfsFallbackBlockWriteHandler( - mBlockWorker, mUfsManager, mResponseObserver, mUserInfo, mDomainSocketEnabled); - default: - throw new IllegalArgumentException(String.format("Invalid request type %s", - request.getCommand().getType().name())); - } - } - - @Override - public void onNext(WriteRequest request) { - if (mWriteHandler == null) { - mWriteHandler = createWriterHandler(request); - } - if (mMarshaller != null) { - mWriteHandler.writeDataMessage(request, mMarshaller.pollBuffer(request)); - } else { - mWriteHandler.write(request); - } - } - - @Override - public void onError(Throwable t) { - if (mWriteHandler != null) { - mWriteHandler.onError(t); - } - } - - @Override - public void onCompleted() { - if (mWriteHandler != null) { - mWriteHandler.onCompleted(); - } - } - - /** - * Handles cancel event from the client. - */ - public void onCancel() { - if (mWriteHandler != null) { - mWriteHandler.onCancel(); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/GrpcExecutors.java b/core/server/worker/src/main/java/alluxio/worker/grpc/GrpcExecutors.java deleted file mode 100644 index 83c5ac90c941..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/GrpcExecutors.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.security.User; -import alluxio.security.authentication.AuthenticatedClientUser; -import alluxio.util.ThreadFactoryUtils; -import alluxio.util.executor.UniqueBlockingQueue; - -import java.util.List; -import java.util.concurrent.AbstractExecutorService; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Executors for gRPC block server. - */ -@ThreadSafe -public final class GrpcExecutors { - private static final long THREAD_STOP_MS = Constants.SECOND_MS * 10; - private static final int THREADS_MIN = 4; - - private static final ThreadPoolExecutor CACHE_MANAGER_THREAD_POOL_EXECUTOR = - new ThreadPoolExecutor(THREADS_MIN, - Configuration.getInt(PropertyKey.WORKER_NETWORK_ASYNC_CACHE_MANAGER_THREADS_MAX), - THREAD_STOP_MS, TimeUnit.MILLISECONDS, new UniqueBlockingQueue<>( - Configuration.getInt(PropertyKey.WORKER_NETWORK_ASYNC_CACHE_MANAGER_QUEUE_MAX)), - ThreadFactoryUtils.build("CacheManagerExecutor-%d", true)); - public static final ExecutorService CACHE_MANAGER_EXECUTOR = - new ImpersonateThreadPoolExecutor(CACHE_MANAGER_THREAD_POOL_EXECUTOR); - - private static final ThreadPoolExecutor BLOCK_READER_THREAD_POOL_EXECUTOR = - new ThreadPoolExecutor(THREADS_MIN, Configuration.getInt( - PropertyKey.WORKER_NETWORK_BLOCK_READER_THREADS_MAX), THREAD_STOP_MS, - TimeUnit.MILLISECONDS, new SynchronousQueue<>(), - ThreadFactoryUtils.build("BlockDataReaderExecutor-%d", true)); - public static final ExecutorService BLOCK_READER_EXECUTOR = - new ImpersonateThreadPoolExecutor(BLOCK_READER_THREAD_POOL_EXECUTOR); - - private static final ThreadPoolExecutor BLOCK_SERIALIZED_THREAD_POOL_EXECUTOR = - new ThreadPoolExecutor(THREADS_MIN, - Configuration.getInt(PropertyKey.WORKER_NETWORK_BLOCK_READER_THREADS_MAX), - THREAD_STOP_MS, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(32), - ThreadFactoryUtils.build("BlockDataReaderSerializedExecutor-%d", true), - new ThreadPoolExecutor.CallerRunsPolicy()); - public static final ExecutorService BLOCK_READER_SERIALIZED_RUNNER_EXECUTOR = - new ImpersonateThreadPoolExecutor(BLOCK_SERIALIZED_THREAD_POOL_EXECUTOR); - - private static final ThreadPoolExecutor BLOCK_WRITE_THREAD_POOL_EXECUTOR = - new ThreadPoolExecutor(THREADS_MIN, Configuration.getInt( - PropertyKey.WORKER_NETWORK_BLOCK_WRITER_THREADS_MAX), THREAD_STOP_MS, - TimeUnit.MILLISECONDS, new SynchronousQueue<>(), - ThreadFactoryUtils.build("BlockDataWriterExecutor-%d", true)); - public static final ExecutorService BLOCK_WRITER_EXECUTOR = - new ImpersonateThreadPoolExecutor(BLOCK_WRITE_THREAD_POOL_EXECUTOR); - - static { - MetricsSystem.registerCachedGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_CACHE_MANAGER_THREAD_ACTIVE_COUNT.getName()), - CACHE_MANAGER_THREAD_POOL_EXECUTOR::getActiveCount, 5, TimeUnit.SECONDS); - MetricsSystem.registerCachedGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_CACHE_MANAGER_THREAD_CURRENT_COUNT.getName()), - CACHE_MANAGER_THREAD_POOL_EXECUTOR::getPoolSize, 5, TimeUnit.SECONDS); - MetricsSystem.registerCachedGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_CACHE_MANAGER_THREAD_QUEUE_WAITING_TASK_COUNT.getName()), - CACHE_MANAGER_THREAD_POOL_EXECUTOR.getQueue()::size, 5, TimeUnit.SECONDS); - // This value is not updated after the process starts, - // so it can be cached for a long time to reduce the number of queries - MetricsSystem.registerCachedGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_CACHE_MANAGER_THREAD_MAX_COUNT.getName()), - CACHE_MANAGER_THREAD_POOL_EXECUTOR::getMaximumPoolSize, 30, TimeUnit.MINUTES); - MetricsSystem.registerCachedGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_CACHE_MANAGER_COMPLETED_TASK_COUNT.getName()), - CACHE_MANAGER_THREAD_POOL_EXECUTOR::getCompletedTaskCount, 5, TimeUnit.SECONDS); - - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_READER_THREAD_ACTIVE_COUNT.getName()), - BLOCK_READER_THREAD_POOL_EXECUTOR::getActiveCount); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_READER_THREAD_CURRENT_COUNT.getName()), - BLOCK_READER_THREAD_POOL_EXECUTOR::getPoolSize); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_READER_THREAD_MAX_COUNT.getName()), - BLOCK_READER_THREAD_POOL_EXECUTOR::getMaximumPoolSize); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_READER_COMPLETED_TASK_COUNT.getName()), - BLOCK_READER_THREAD_POOL_EXECUTOR::getCompletedTaskCount); - - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_SERIALIZED_THREAD_ACTIVE_COUNT.getName()), - BLOCK_SERIALIZED_THREAD_POOL_EXECUTOR::getActiveCount); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_SERIALIZED_THREAD_CURRENT_COUNT.getName()), - BLOCK_SERIALIZED_THREAD_POOL_EXECUTOR::getPoolSize); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_SERIALIZED_THREAD_MAX_COUNT.getName()), - BLOCK_SERIALIZED_THREAD_POOL_EXECUTOR::getMaximumPoolSize); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_SERIALIZED_COMPLETED_TASK_COUNT.getName()), - BLOCK_SERIALIZED_THREAD_POOL_EXECUTOR::getCompletedTaskCount); - - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_WRITER_THREAD_ACTIVE_COUNT.getName()), - BLOCK_WRITE_THREAD_POOL_EXECUTOR::getActiveCount); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_WRITER_THREAD_CURRENT_COUNT.getName()), - BLOCK_WRITE_THREAD_POOL_EXECUTOR::getPoolSize); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_WRITER_THREAD_MAX_COUNT.getName()), - BLOCK_WRITE_THREAD_POOL_EXECUTOR::getMaximumPoolSize); - MetricsSystem.registerGaugeIfAbsent(MetricsSystem.getMetricName( - MetricKey.WORKER_BLOCK_WRITER_COMPLETED_TASK_COUNT.getName()), - BLOCK_WRITE_THREAD_POOL_EXECUTOR::getCompletedTaskCount); - } - - /** - * Private constructor. - */ - private GrpcExecutors() {} - - /** - * This executor passes impersonation information to the real worker thread. - * The proxy user is tracked by {@link AuthenticatedClientUser#sUserThreadLocal}. - * This executor delegates operations to the underlying executor while setting the - * ThreadLocal context for execution. - * */ - private static class ImpersonateThreadPoolExecutor extends AbstractExecutorService { - private final ExecutorService mDelegate; - - public ImpersonateThreadPoolExecutor(ExecutorService service) { - mDelegate = service; - } - - @Override - public void execute(final Runnable command) { - // If there's no impersonation, proxyUser is just null - User proxyUser = AuthenticatedClientUser.getOrNull(); - mDelegate.execute(() -> { - try { - AuthenticatedClientUser.set(proxyUser); - command.run(); - } finally { - AuthenticatedClientUser.remove(); - } - }); - } - - @Override - public void shutdown() { - mDelegate.shutdown(); - } - - @Override - public List shutdownNow() { - return mDelegate.shutdownNow(); - } - - @Override - public boolean isShutdown() { - return mDelegate.isShutdown(); - } - - @Override - public boolean isTerminated() { - return mDelegate.isTerminated(); - } - - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - return mDelegate.awaitTermination(timeout, unit); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/ShortCircuitBlockReadHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/ShortCircuitBlockReadHandler.java deleted file mode 100644 index 4b7608fab229..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/ShortCircuitBlockReadHandler.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import static alluxio.worker.block.BlockMetadataManager.WORKER_STORAGE_TIER_ASSOC; -import static com.google.common.base.Preconditions.checkState; - -import alluxio.RpcUtils; -import alluxio.exception.InvalidWorkerStateException; -import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; -import alluxio.grpc.GrpcExceptionUtils; -import alluxio.grpc.OpenLocalBlockRequest; -import alluxio.grpc.OpenLocalBlockResponse; -import alluxio.util.IdUtils; -import alluxio.util.LogUtils; -import alluxio.worker.block.AllocateOptions; -import alluxio.worker.block.BlockLock; -import alluxio.worker.block.BlockStore; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.DefaultBlockWorker; -import alluxio.worker.block.meta.BlockMeta; - -import io.grpc.stub.StreamObserver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.text.MessageFormat; -import java.util.Optional; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * gRPC handler that handles short circuit read requests. - */ -@NotThreadSafe -class ShortCircuitBlockReadHandler implements StreamObserver { - private static final Logger LOG = - LoggerFactory.getLogger(ShortCircuitBlockReadHandler.class); - - private final BlockStore mLocalBlockStore; - private final StreamObserver mResponseObserver; - private OpenLocalBlockRequest mRequest; - /** The lock id of the block being read. */ - private Optional mBlockLock; - private long mSessionId; - - /** - * Creates an instance of {@link ShortCircuitBlockReadHandler}. - * - * @param localBlockStore the local block store - */ - ShortCircuitBlockReadHandler(BlockStore localBlockStore, - StreamObserver responseObserver) { - mLocalBlockStore = localBlockStore; - mBlockLock = Optional.empty(); - mResponseObserver = responseObserver; - } - - /** - * Handles block open request. - */ - @Override - public void onNext(OpenLocalBlockRequest request) { - RpcUtils.streamingRPCAndLog(LOG, new RpcUtils.StreamingRpcCallable() { - @Override - public OpenLocalBlockResponse call() throws Exception { - checkState(mRequest == null); - mRequest = request; - if (mBlockLock.isPresent()) { - LOG.warn("Lock block {} without releasing previous block lock {}.", - mRequest.getBlockId(), mBlockLock); - throw new InvalidWorkerStateException( - MessageFormat.format("session {0,number,#} is not closed.", mBlockLock)); - } - mSessionId = IdUtils.createSessionId(); - // TODO(calvin): Update the locking logic so this can be done better - Optional meta = mLocalBlockStore.getVolatileBlockMeta(mRequest.getBlockId()); - if (!meta.isPresent()) { - throw new BlockDoesNotExistRuntimeException(mRequest.getBlockId()); - } - if (mRequest.getPromote()) { - // TODO(calvin): Move this logic into BlockStore#moveBlockInternal if possible - // Because the move operation is expensive, we first check if the operation is necessary - BlockStoreLocation dst = BlockStoreLocation.anyDirInTier( - WORKER_STORAGE_TIER_ASSOC.getAlias(0)); - if (!meta.get().getBlockLocation().belongsTo(dst)) { - // Execute the block move if necessary - mLocalBlockStore.moveBlock(mSessionId, mRequest.getBlockId(), - AllocateOptions.forMove(dst)); - - // block is moved, get the latest metadata for the block's - // new path - meta = mLocalBlockStore.getVolatileBlockMeta(mRequest.getBlockId()); - } - } - mBlockLock = mLocalBlockStore.pinBlock(mSessionId, mRequest.getBlockId()); - mLocalBlockStore.accessBlock(mSessionId, mRequest.getBlockId()); - DefaultBlockWorker.Metrics.WORKER_ACTIVE_CLIENTS.inc(); - return OpenLocalBlockResponse.newBuilder() - .setPath(meta.get().getPath()) - .build(); - } - - @Override - public void exceptionCaught(Throwable e) { - if (mBlockLock.isPresent()) { - DefaultBlockWorker.Metrics.WORKER_ACTIVE_CLIENTS.dec(); - mLocalBlockStore.unpinBlock(mBlockLock.get()); - mBlockLock = Optional.empty(); - } - mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(e)); - } - }, "OpenBlock", true, false, mResponseObserver, "Session=%d, Request=%s", - mSessionId, request); - // Must use request object directly for this log as mRequest is only set in the Callable - } - - @Override - public void onError(Throwable t) { - LogUtils.warnWithException(LOG, "Exception occurred processing read request {}.", mRequest, t); - if (mBlockLock.isPresent()) { - DefaultBlockWorker.Metrics.WORKER_ACTIVE_CLIENTS.dec(); - mLocalBlockStore.unpinBlock(mBlockLock.get()); - mLocalBlockStore.cleanupSession(mSessionId); - } - mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(t)); - } - - /** - * Handles block close request. No exceptions should be thrown. - */ - @Override - public void onCompleted() { - RpcUtils.streamingRPCAndLog(LOG, new RpcUtils.StreamingRpcCallable() { - @Override - public OpenLocalBlockResponse call() { - if (mBlockLock.isPresent()) { - DefaultBlockWorker.Metrics.WORKER_ACTIVE_CLIENTS.dec(); - mLocalBlockStore.unpinBlock(mBlockLock.get()); - mBlockLock = Optional.empty(); - } else if (mRequest != null) { - LOG.warn("Close a closed block {}.", mRequest.getBlockId()); - } - return null; - } - - @Override - public void exceptionCaught(Throwable e) { - mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(e)); - mBlockLock = Optional.empty(); - } - }, "CloseBlock", false, true, mResponseObserver, "Session=%d, Request=%s", - mSessionId, mRequest); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/ShortCircuitBlockWriteHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/ShortCircuitBlockWriteHandler.java deleted file mode 100644 index e0da6655cdec..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/ShortCircuitBlockWriteHandler.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.RpcUtils; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.grpc.CreateLocalBlockRequest; -import alluxio.grpc.CreateLocalBlockResponse; -import alluxio.grpc.GrpcExceptionUtils; -import alluxio.util.IdUtils; -import alluxio.util.LogUtils; -import alluxio.worker.block.BlockWorker; -import alluxio.worker.block.CreateBlockOptions; - -import com.google.common.base.Preconditions; -import io.grpc.Context; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.text.MessageFormat; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A handler that handles short circuit read requests. - */ -@NotThreadSafe -class ShortCircuitBlockWriteHandler implements StreamObserver { - private static final Logger LOG = - LoggerFactory.getLogger(ShortCircuitBlockWriteHandler.class); - - private static final long INVALID_SESSION_ID = -1; - - /** The block worker. */ - private final BlockWorker mBlockWorker; - /** An object storing the mapping of tier aliases to ordinals. */ - private final StreamObserver mResponseObserver; - private CreateLocalBlockRequest mRequest = null; - private long mSessionId = INVALID_SESSION_ID; - - /** - * Creates an instance of {@link ShortCircuitBlockWriteHandler}. - * - * @param blockWorker the block worker - */ - ShortCircuitBlockWriteHandler(BlockWorker blockWorker, - StreamObserver responseObserver) { - mBlockWorker = blockWorker; - mResponseObserver = responseObserver; - } - - /** - * Handles request to create local block. No exceptions should be thrown. - * - * @param request a create request - */ - @Override - public void onNext(CreateLocalBlockRequest request) { - final String methodName = request.getOnlyReserveSpace() ? "ReserveSpace" : "CreateBlock"; - RpcUtils.streamingRPCAndLog(LOG, new RpcUtils.StreamingRpcCallable() { - @Override - public CreateLocalBlockResponse call() throws Exception { - if (request.getOnlyReserveSpace()) { - mBlockWorker.requestSpace(mSessionId, request.getBlockId(), request.getSpaceToReserve()); - return CreateLocalBlockResponse.newBuilder().build(); - } else { - Preconditions.checkState(mRequest == null); - mRequest = request; - Preconditions.checkState(mSessionId == INVALID_SESSION_ID, - MessageFormat.format("session {0,number,#} is not closed.", mSessionId)); - mSessionId = IdUtils.createSessionId(); - String path = - mBlockWorker.createBlock(mSessionId, request.getBlockId(), request.getTier(), - new CreateBlockOptions(null, request.getMediumType(), - request.getSpaceToReserve())); - return CreateLocalBlockResponse.newBuilder().setPath(path).build(); - } - } - - @Override - public void exceptionCaught(Throwable throwable) { - if (mSessionId != INVALID_SESSION_ID) { - // In case the client is a UfsFallbackDataWriter, DO NOT clean the temp blocks. - if (throwable instanceof ResourceExhaustedRuntimeException - && request.hasCleanupOnFailure() && !request.getCleanupOnFailure()) { - mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(throwable)); - return; - } - mBlockWorker.cleanupSession(mSessionId); - mSessionId = INVALID_SESSION_ID; - } - mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(throwable)); - } - }, methodName, true, false, mResponseObserver, "Session=%d, Request=%s", mSessionId, request); - } - - @Override - public void onCompleted() { - handleBlockCompleteRequest(false); - } - - /** - * Handles cancel event from the client. - */ - public void onCancel() { - handleBlockCompleteRequest(true); - } - - @Override - public void onError(Throwable t) { - LogUtils.warnWithException(LOG, "Exception occurred processing write request {}.", mRequest, t); - if (t instanceof StatusRuntimeException - && ((StatusRuntimeException) t).getStatus().getCode() == Status.Code.CANCELLED) { - // Cancellation is already handled. - return; - } - // The RPC handlers do not throw exceptions. All the exception seen here is either - // network exception or some runtime exception (e.g. NullPointerException). - LOG.error("Failed to handle RPCs.", t); - if (mSessionId != INVALID_SESSION_ID) { - mBlockWorker.cleanupSession(mSessionId); - mSessionId = INVALID_SESSION_ID; - } - mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(t)); - } - - /** - * Handles complete block request. No exceptions should be thrown. - * - */ - public void handleBlockCompleteRequest(boolean isCanceled) { - final String methodName = isCanceled ? "AbortBlock" : "CommitBlock"; - RpcUtils.streamingRPCAndLog(LOG, new RpcUtils.StreamingRpcCallable() { - @Override - public CreateLocalBlockResponse call() throws Exception { - if (mRequest == null) { - return null; - } - Context newContext = Context.current().fork(); - Context previousContext = newContext.attach(); - try { - if (isCanceled) { - mBlockWorker.abortBlock(mSessionId, mRequest.getBlockId()); - } else { - mBlockWorker.commitBlock(mSessionId, mRequest.getBlockId(), mRequest.getPinOnCreate()); - } - } finally { - newContext.detach(previousContext); - } - mSessionId = INVALID_SESSION_ID; - return null; - } - - @Override - public void exceptionCaught(Throwable throwable) { - mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(throwable)); - mSessionId = INVALID_SESSION_ID; - } - }, methodName, false, !isCanceled, mResponseObserver, "Session=%d, Request=%s", mSessionId, - mRequest); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/UfsFallbackBlockWriteHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/UfsFallbackBlockWriteHandler.java deleted file mode 100644 index 1ae728f3084d..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/UfsFallbackBlockWriteHandler.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.conf.Configuration; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.grpc.WriteRequestCommand; -import alluxio.grpc.WriteResponse; -import alluxio.metrics.MetricInfo; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.proto.dataserver.Protocol; -import alluxio.security.authentication.AuthenticatedUserInfo; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.options.CreateOptions; -import alluxio.worker.BlockUtils; -import alluxio.worker.block.CreateBlockOptions; -import alluxio.worker.block.DefaultBlockWorker; -import alluxio.worker.block.meta.TempBlockMeta; - -import com.google.common.base.Preconditions; -import io.grpc.stub.StreamObserver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Optional; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This handler handles UFS block write request. Instead of writing a block to tiered storage, this - * handler writes the block into UFS . - */ -@alluxio.annotation.SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", - justification = "false positive with superclass generics, " - + "see more description in https://sourceforge.net/p/findbugs/bugs/1242/") -@NotThreadSafe -public final class UfsFallbackBlockWriteHandler - extends AbstractWriteHandler { - private static final Logger LOG = LoggerFactory.getLogger(UfsFallbackBlockWriteHandler.class); - - /** The Block Worker which handles blocks stored in the Alluxio storage of the worker. */ - private final DefaultBlockWorker mWorker; - private final UfsManager mUfsManager; - private final BlockWriteHandler mBlockWriteHandler; - private final boolean mDomainSocketEnabled; - - /** - * Creates an instance of {@link UfsFallbackBlockWriteHandler}. - * - * @param blockWorker the block worker - * @param userInfo the authenticated user info - * @param domainSocketEnabled whether using a domain socket - */ - UfsFallbackBlockWriteHandler(DefaultBlockWorker blockWorker, UfsManager ufsManager, - StreamObserver responseObserver, AuthenticatedUserInfo userInfo, - boolean domainSocketEnabled) { - super(responseObserver, userInfo); - mWorker = blockWorker; - mUfsManager = ufsManager; - mBlockWriteHandler = - new BlockWriteHandler(blockWorker, responseObserver, userInfo, domainSocketEnabled); - mDomainSocketEnabled = domainSocketEnabled; - } - - @Override - protected BlockWriteRequestContext createRequestContext(alluxio.grpc.WriteRequest msg) - throws Exception { - BlockWriteRequestContext context = new BlockWriteRequestContext(msg, FILE_BUFFER_SIZE); - if (mDomainSocketEnabled) { - context.setCounter(MetricsSystem.counter(MetricKey.WORKER_BYTES_WRITTEN_DOMAIN.getName())); - context.setMeter(MetricsSystem - .meter(MetricKey.WORKER_BYTES_WRITTEN_DOMAIN_THROUGHPUT.getName())); - } else { - context.setCounter(MetricsSystem.counter(MetricKey.WORKER_BYTES_WRITTEN_REMOTE.getName())); - context.setMeter(MetricsSystem - .meter(MetricKey.WORKER_BYTES_WRITTEN_REMOTE_THROUGHPUT.getName())); - } - BlockWriteRequest request = context.getRequest(); - Preconditions.checkState(request.hasCreateUfsBlockOptions()); - // if it is already a UFS fallback from short-circuit write, avoid writing to local again - context.setWritingToLocal(!request.getCreateUfsBlockOptions().getFallback()); - if (context.isWritingToLocal()) { - mWorker.createBlock(request.getSessionId(), request.getId(), request.getTier(), - new CreateBlockOptions(null, request.getMediumType(), FILE_BUFFER_SIZE)); - } - return context; - } - - @Override - protected void completeRequest(BlockWriteRequestContext context) - throws Exception { - if (context.isWritingToLocal()) { - mBlockWriteHandler.completeRequest(context); - } else { - mWorker.commitBlockInUfs(context.getRequest().getId(), context.getPos()); - if (context.getOutputStream() != null) { - context.getOutputStream().close(); - context.setOutputStream(null); - } - } - if (context.getUfsResource() != null) { - context.getUfsResource().close(); - } - } - - @Override - protected void cancelRequest(BlockWriteRequestContext context) throws Exception { - if (context.isWritingToLocal()) { - mBlockWriteHandler.cancelRequest(context); - } else { - if (context.getOutputStream() != null) { - context.getOutputStream().close(); - context.setOutputStream(null); - } - if (context.getUfsResource() != null) { - context.getUfsResource().get().deleteExistingFile(context.getUfsPath()); - } - } - if (context.getUfsResource() != null) { - context.getUfsResource().close(); - } - } - - @Override - protected void cleanupRequest(BlockWriteRequestContext context) throws Exception { - if (context.isWritingToLocal()) { - mBlockWriteHandler.cleanupRequest(context); - } else { - cancelRequest(context); - } - } - - @Override - protected void flushRequest(BlockWriteRequestContext context) throws Exception { - if (context.isWritingToLocal()) { - mBlockWriteHandler.flushRequest(context); - } else if (context.getOutputStream() != null) { - context.getOutputStream().flush(); - } - } - - @Override - protected void writeBuf(BlockWriteRequestContext context, - StreamObserver responseObserver, DataBuffer buf, long pos) throws Exception { - if (context.isWritingToLocal()) { - // TODO(binfan): change signature of writeBuf to pass current offset and length of buffer. - // Currently pos is the calculated offset after writeBuf succeeds. - long posBeforeWrite = pos - buf.readableBytes(); - try { - mBlockWriteHandler.writeBuf(context, responseObserver, buf, pos); - return; - } catch (ResourceExhaustedRuntimeException e) { - LOG.warn("Not enough space to write block {} to local worker, fallback to UFS. " - + " {} bytes have been written.", - context.getRequest().getId(), posBeforeWrite); - context.setWritingToLocal(false); - } - // close the block writer first - if (context.getBlockWriter() != null) { - context.getBlockWriter().close(); - } - // prepare the UFS block and transfer data from the temp block to UFS - createUfsBlock(context); - if (posBeforeWrite > 0) { - transferToUfsBlock(context, posBeforeWrite); - } - // close the original block writer and remove the temp file - mBlockWriteHandler.cancelRequest(context); - } - if (context.getOutputStream() == null) { - createUfsBlock(context); - } - buf.readBytes(context.getOutputStream(), buf.readableBytes()); - } - - @Override - protected void handleCommand(WriteRequestCommand command, BlockWriteRequestContext context) - throws Exception { - if (command.hasCreateUfsBlockOptions() - && command.getOffset() == 0 - && command.getCreateUfsBlockOptions().hasBytesInBlockStore()) { - long ufsFallbackInitBytes = command.getCreateUfsBlockOptions().getBytesInBlockStore(); - context.setPos(context.getPos() + ufsFallbackInitBytes); - initUfsFallback(context); - } - } - - @Override - protected String getLocationInternal(BlockWriteRequestContext context) { - Protocol.CreateUfsBlockOptions createUfsBlockOptions = - context.getRequest().getCreateUfsBlockOptions(); - - if (createUfsBlockOptions == null) { - return String.format("blockId-%d", context.getRequest().getId()); - } - - UfsManager.UfsClient ufsClient; - try { - ufsClient = mUfsManager.get(createUfsBlockOptions.getMountId()); - } catch (Throwable e) { - return String.format("blockId-%d", context.getRequest().getId()); - } - - return BlockUtils.getUfsBlockPath(ufsClient, context.getRequest().getId()); - } - - private void initUfsFallback(BlockWriteRequestContext context) throws Exception { - Preconditions.checkState(!context.isWritingToLocal()); - if (context.getOutputStream() == null) { - createUfsBlock(context); - } - // transfer data from the temp block to UFS - transferToUfsBlock(context, context.getPos()); - } - - /** - * Creates a UFS block and initialize it with bytes read from block store. - * - * @param context context of this request - */ - private void createUfsBlock(BlockWriteRequestContext context) - throws Exception { - BlockWriteRequest request = context.getRequest(); - Protocol.CreateUfsBlockOptions createUfsBlockOptions = request.getCreateUfsBlockOptions(); - UfsManager.UfsClient ufsClient = mUfsManager.get(createUfsBlockOptions.getMountId()); - alluxio.resource.CloseableResource ufsResource = - ufsClient.acquireUfsResource(); - context.setUfsResource(ufsResource); - String ufsString = MetricsSystem.escape(ufsClient.getUfsMountPointUri()); - String ufsPath = BlockUtils.getUfsBlockPath(ufsClient, request.getId()); - UnderFileSystem ufs = ufsResource.get(); - // Set the atomic flag to be true to ensure only the creation of this file is atomic on close. - OutputStream ufsOutputStream = - ufs.createNonexistingFile(ufsPath, - CreateOptions.defaults(Configuration.global()).setEnsureAtomic(true) - .setCreateParent(true)); - context.setOutputStream(ufsOutputStream); - context.setUfsPath(ufsPath); - - MetricKey counterKey = MetricKey.WORKER_BYTES_WRITTEN_UFS; - MetricKey meterKey = MetricKey.WORKER_BYTES_WRITTEN_UFS_THROUGHPUT; - context.setCounter(MetricsSystem.counterWithTags(counterKey.getName(), - counterKey.isClusterAggregated(), MetricInfo.TAG_UFS, ufsString)); - context.setMeter(MetricsSystem.meterWithTags(meterKey.getName(), - meterKey.isClusterAggregated(), MetricInfo.TAG_UFS, ufsString)); - } - - /** - * Transfers data from block store to UFS. - * - * @param context context of this request - * @param pos number of bytes in block store to write in the UFS block - */ - private void transferToUfsBlock(BlockWriteRequestContext context, long pos) throws IOException { - OutputStream ufsOutputStream = context.getOutputStream(); - long blockId = context.getRequest().getId(); - Optional block = mWorker.getBlockStore().getTempBlockMeta(blockId); - Preconditions.checkState(block.isPresent() - && Files.copy(Paths.get(block.get().getPath()), ufsOutputStream) == pos); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/UfsFileWriteHandler.java b/core/server/worker/src/main/java/alluxio/worker/grpc/UfsFileWriteHandler.java deleted file mode 100644 index 32a574bda853..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/UfsFileWriteHandler.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.conf.Configuration; -import alluxio.grpc.WriteRequest; -import alluxio.grpc.WriteResponse; -import alluxio.metrics.MetricInfo; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.proto.dataserver.Protocol; -import alluxio.resource.CloseableResource; -import alluxio.security.authentication.AuthenticatedUserInfo; -import alluxio.security.authorization.Mode; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.options.CreateOptions; -import alluxio.util.proto.ProtoUtils; - -import com.google.common.base.Preconditions; -import io.grpc.stub.StreamObserver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * This handler handles writes to a file in the under file system. Due to the semantics enforced - * on under file systems, the client must write all the data of a file through the same stream to - * the under file system. This prevents us from handling the writes at a block level. - * - * For more information about the implementation of read/write buffering, see - * {@link AbstractWriteHandler}. - * - * For more information about the implementation of the client side writer, see - * UnderFileSystemFileOutStream. - */ -@alluxio.annotation.SuppressFBWarnings( - value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", - justification = "false positive with superclass generics, " - + "see more description in https://sourceforge.net/p/findbugs/bugs/1242/") -@NotThreadSafe -public final class UfsFileWriteHandler extends AbstractWriteHandler { - private static final Logger LOG = LoggerFactory.getLogger(UfsFileWriteHandler.class); - private final UfsManager mUfsManager; - - /** - * Creates an instance of {@link UfsFileWriteHandler}. - * - * @param ufsManager the file data manager - * @param userInfo the authenticated user info - */ - UfsFileWriteHandler(UfsManager ufsManager, StreamObserver responseObserver, - AuthenticatedUserInfo userInfo) { - super(responseObserver, userInfo); - mUfsManager = ufsManager; - } - - @Override - protected UfsFileWriteRequestContext createRequestContext(WriteRequest msg) { - return new UfsFileWriteRequestContext(msg); - } - - @Override - protected void completeRequest(UfsFileWriteRequestContext context) - throws Exception { - if (context == null) { - return; - } - if (context.getOutputStream() == null) { - createUfsFile(context); - } - Preconditions.checkState(context.getOutputStream() != null); - context.getOutputStream().close(); - CreateOptions createOptions = context.getCreateOptions(); - if (createOptions != null) { - try { - // Set the owner/group of the file to the correct owner. - context.getUfsResource().get() - .setOwner(context.getRequest().getUfsPath(), createOptions.getOwner(), - createOptions.getGroup()); - } catch (IOException e) { - LOG.warn("Failed to update ownership for ufs path: {} owner: {} group: {} error: {}", - context.getRequest().getUfsPath(), createOptions.getOwner(), createOptions.getGroup(), - e.toString()); - } - } - context.setOutputStream(null); - context.setCreateOptions(null); - context.getUfsResource().close(); - } - - @Override - protected void cancelRequest(UfsFileWriteRequestContext context) throws Exception { - if (context == null) { - return; - } - UfsFileWriteRequest request = context.getRequest(); - // TODO(calvin): Consider adding cancel to the ufs stream api. - if (context.getOutputStream() != null && context.getUfsResource() != null) { - context.getOutputStream().close(); - context.getUfsResource().get().deleteExistingFile(request.getUfsPath()); - context.setOutputStream(null); - context.setCreateOptions(null); - context.getUfsResource().close(); - } - } - - @Override - protected void cleanupRequest(UfsFileWriteRequestContext context) throws Exception { - cancelRequest(context); - } - - @Override - protected void flushRequest(UfsFileWriteRequestContext context) - throws Exception { - Preconditions.checkState(context != null); - if (context.getOutputStream() != null) { - context.getOutputStream().flush(); - } - } - - @Override - protected void writeBuf(UfsFileWriteRequestContext context, - StreamObserver observer, DataBuffer buf, long pos) throws Exception { - Preconditions.checkState(context != null); - if (context.getOutputStream() == null) { - createUfsFile(context); - } - buf.readBytes(context.getOutputStream(), buf.readableBytes()); - } - - @Override - protected String getLocationInternal(UfsFileWriteRequestContext context) { - return context.getRequest().getCreateUfsFileOptions().getUfsPath(); - } - - private void createUfsFile(UfsFileWriteRequestContext context) - throws IOException { - UfsFileWriteRequest request = context.getRequest(); - Preconditions.checkState(request != null); - Protocol.CreateUfsFileOptions createUfsFileOptions = request.getCreateUfsFileOptions(); - UfsManager.UfsClient ufsClient = mUfsManager.get(createUfsFileOptions.getMountId()); - CloseableResource ufsResource = ufsClient.acquireUfsResource(); - context.setUfsResource(ufsResource); - UnderFileSystem ufs = ufsResource.get(); - CreateOptions createOptions = CreateOptions.defaults(Configuration.global()) - .setCreateParent(true) - .setOwner(createUfsFileOptions.getOwner()).setGroup(createUfsFileOptions.getGroup()) - .setMode(new Mode((short) createUfsFileOptions.getMode())); - if (createUfsFileOptions.hasAcl()) { - // This acl information will be ignored by all but HDFS implementations - createOptions.setAcl(ProtoUtils.fromProto(createUfsFileOptions.getAcl())); - } - context.setOutputStream(ufs.createNonexistingFile(request.getUfsPath(), createOptions)); - context.setCreateOptions(createOptions); - String ufsString = MetricsSystem.escape(ufsClient.getUfsMountPointUri()); - - MetricKey counterKey = MetricKey.WORKER_BYTES_WRITTEN_UFS; - MetricKey meterKey = MetricKey.WORKER_BYTES_WRITTEN_UFS_THROUGHPUT; - context.setCounter(MetricsSystem.counterWithTags(counterKey.getName(), - counterKey.isClusterAggregated(), MetricInfo.TAG_UFS, ufsString)); - context.setMeter(MetricsSystem.meterWithTags(meterKey.getName(), - meterKey.isClusterAggregated(), MetricInfo.TAG_UFS, ufsString)); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequest.java b/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequest.java deleted file mode 100644 index e9e623d4397d..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import alluxio.util.IdUtils; - -import com.google.common.base.MoreObjects; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Represents a write request received from gRPC channel. - */ -@ThreadSafe -public class WriteRequest { - /** This ID can either be block ID or temp UFS file ID. */ - private final long mId; - - /** Whether to pin block on create. */ - private final boolean mPinOnCreate; - - /** The session id associated with all temporary resources of this request. */ - private final long mSessionId; - - WriteRequest(alluxio.grpc.WriteRequest request) { - mId = request.getCommand().getId(); - mPinOnCreate = request.getCommand().getPinOnCreate(); - mSessionId = IdUtils.createSessionId(); - } - - /** - * @return the block ID or the temp UFS file ID - */ - public long getId() { - return mId; - } - - /** - * @return whether to pin block on create - */ - public boolean getPinOnCreate() { - return mPinOnCreate; - } - - /** - * @return the session ID - */ - public long getSessionId() { - return mSessionId; - } - - /** - * @return a {@link MoreObjects.ToStringHelper}, inheriting classes should call super - */ - protected MoreObjects.ToStringHelper toStringHelper() { - return MoreObjects.toStringHelper(this) - .add("id", mId) - .add("sessionId", mSessionId); - } - - @Override - public final String toString() { - return toStringHelper().toString(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequestContext.java b/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequestContext.java deleted file mode 100644 index f36652daa8a0..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequestContext.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; - -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Represents the context of a write request received from gRPC stream. This class serves the - * shared states of the request and can be accessed concurrently by the gRPC event thread - * and I/O thread. - * - * @param type of the write request - */ -@ThreadSafe -public class WriteRequestContext { - - /** The requests of this context. */ - private final T mRequest; - - /** - * The error seen in either the gRPC event threads (e.g. failed to read from the network) or the - * data writer thread (e.g. failed to write the data). - */ - @GuardedBy("AbstractWriteHandler#mLock") - private alluxio.worker.grpc.Error mError; - - /** - * The next pos to queue to the buffer. This is only updated and used by the gRPC event thread. - */ - private long mPos; - - private Counter mCounter; - private Meter mMeter; - - /** This is set when EOF or CANCEL is received. This is only for sanity check. */ - private volatile boolean mDone; - - /** - * @param request the write request - */ - public WriteRequestContext(T request) { - mRequest = request; - mPos = 0; - mDone = false; - } - - /** - * @return the write request - */ - public T getRequest() { - return mRequest; - } - - /** - * @return the error - */ - @GuardedBy("AbstractWriteHandler#mLock") - @Nullable - public alluxio.worker.grpc.Error getError() { - return mError; - } - - /** - * @return the next position to write to the block worker - */ - @GuardedBy("AbstractWriteHandler#mLock") - public long getPos() { - return mPos; - } - - /** - * @return metrics counter associated with this request - */ - @Nullable - public Counter getCounter() { - return mCounter; - } - - /** - * @return metrics meter associated with this request - */ - @Nullable - public Meter getMeter() { - return mMeter; - } - - /** - * @return true when the EOF or CANCEL is received, false otherwise - */ - public boolean isDoneUnsafe() { - return mDone; - } - - /** - * @param error the error - */ - @GuardedBy("AbstractWriteHandler#mLock") - public void setError(alluxio.worker.grpc.Error error) { - mError = error; - } - - /** - * @param posToWrite the next position to write to the block worker - */ - @GuardedBy("AbstractWriteHandler#mLock") - public void setPos(long posToWrite) { - mPos = posToWrite; - } - - /** - * @param done whether the EOF or CANCEL is received - */ - public void setDoneUnsafe(boolean done) { - mDone = done; - } - - /** - * @param counter counter to set - */ - public void setCounter(Counter counter) { - mCounter = counter; - } - - /** - * @param meter meter to set - */ - public void setMeter(Meter meter) { - mMeter = meter; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/package-info.java b/core/server/worker/src/main/java/alluxio/worker/package-info.java deleted file mode 100644 index b62b25d76318..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/package-info.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -/** - * Worker process and utils for working with the worker remotely. - * - * Main entry point for the worker is {@link alluxio.worker.AlluxioWorker#main} - * which gets started by the alluxio start scripts. The {@link alluxio.worker.WorkerProcess} - * spins up the different RPC services (gRPC, data) which are mostly wrappers around - * {@link alluxio.worker.block.BlockWorker}. - * - *

Services

- * - *

DataServer

- * - * This service is the main interaction between users and worker for reading and writing blocks. - * The {@link alluxio.worker.DataServer} interface defines how to start/stop, and get port - * details; to start, object init is used. The implementation of this interface is in - * {@link alluxio.worker.grpc.GrpcDataServer}. It creates an {@link alluxio.worker.DataServer} - * instance based on gRPC which is a high performance universal remote procedure call framework. - * - * Data server handles the following types of block requests: - *
    - *
  • Read blocks in worker storage from network
    • - *
  • Read blocks in worker storage from local file system: - * short-circuit read for local client
    • - *
  • Read blocks in UFS
    • - *
  • Write blocks in worker storage from network
    • - *
  • Write blocks in worker storage from local file system: - * short-circuit write for local client
    • - *
  • Write blocks in UFS
    • - *
- * - * The current protocol is described in {@link alluxio.proto.dataserver.Protocol}. - * All the requests are generated by Protobuf to keep the protocol extensible but also - * backward-compatible. - * - */ -package alluxio.worker; diff --git a/core/server/worker/src/main/java/alluxio/worker/page/BlockPageEvictor.java b/core/server/worker/src/main/java/alluxio/worker/page/BlockPageEvictor.java deleted file mode 100644 index 722403310891..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/BlockPageEvictor.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.evictor.CacheEvictor; - -import java.util.HashSet; -import java.util.Set; -import java.util.function.Predicate; -import javax.annotation.Nullable; - -/** - * Evictor for pages that belong to a block. - * Blocks that are not eligible for eviction because - * they are pinned, waiting to be persisted, etc., can be temporarily skipped from eviction - * by calling {@link #addPinnedBlock(long)}, and later calling {@link #removePinnedBlock(long)} - * makes them eligible for eviction again. - */ -public class BlockPageEvictor implements CacheEvictor { - private final CacheEvictor mDelegate; - private final Set mPinnedBlocks = new HashSet<>(); - private final Predicate mIsNotPinned = - (pageId) -> !mPinnedBlocks.contains(pageId.getFileId()); - - /** - * Creates a new block page evictor with an underlying evictor. - * - * @param delegate delegated-to cache evictor - */ - BlockPageEvictor(CacheEvictor delegate) { - mDelegate = delegate; - } - - @Override - public void updateOnGet(PageId pageId) { - mDelegate.updateOnGet(pageId); - } - - @Override - public void updateOnPut(PageId pageId) { - mDelegate.updateOnPut(pageId); - } - - @Override - public void updateOnDelete(PageId pageId) { - mDelegate.updateOnDelete(pageId); - } - - @Nullable - @Override - public synchronized PageId evict() { - return mDelegate.evictMatching(mIsNotPinned); - } - - @Nullable - @Override - public synchronized PageId evictMatching(Predicate criterion) { - return mDelegate.evictMatching(pageId -> criterion.test(pageId) && mIsNotPinned.test(pageId)); - } - - @Override - public synchronized void reset() { - mDelegate.reset(); - mPinnedBlocks.clear(); - } - - /** - * Adds a pinned block that is prevented from eviction. All pages belonging to that block - * will not be eligible for eviction until {@link #removePinnedBlock(long)} is called for that - * block. - * - * @param blockId block id - * @return true if the block was not already pinned, false otherwise - */ - public synchronized boolean addPinnedBlock(long blockId) { - return mPinnedBlocks.add(String.valueOf(blockId)); - } - - /** - * Removes a pinned block that is prevented from eviction. - * - * @param blockId block id - * @return true if the block was pinned, false otherwise - */ - public synchronized boolean removePinnedBlock(long blockId) { - return mPinnedBlocks.remove(String.valueOf(blockId)); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/BlockPageId.java b/core/server/worker/src/main/java/alluxio/worker/page/BlockPageId.java deleted file mode 100644 index 33555f2576a7..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/BlockPageId.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.client.file.cache.PageId; - -import com.google.common.base.Preconditions; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Specialized {@link PageId} when it's part of a block. - */ -public final class BlockPageId extends PageId { - // the name pattern of the page files of a block - // the block id and size are encoded as 16-byte hexadecimal - private static final Pattern FILE_ID_PATTERN = - Pattern.compile("paged_block_([0-9a-fA-F]{16})_size_([0-9a-fA-F]{16})"); - private static final String FILE_ID_TEMPLATE = "paged_block_%016x_size_%016x"; - // placeholder for temp blocks - private static final long INVALID_BLOCK_SIZE = -1; - - /** - * this is constructed from {@link PageId#getFileId()} and cached to avoid parsing the string - * multiple times. - */ - private final long mBlockId; - - /** - * The block size. It is here because we need to put this information into the page store - * but it does not currently support page metadata. - * Todo(bowen): make this part of page metadata and remove it - */ - private final long mBlockSize; - - /** - * @param blockId string representation of the block ID in base 10 - * @param pageIndex index of the page in the block - * @param blockSize block size - * @throws NumberFormatException when {@code blockId} cannot be parsed as a {@code long} - */ - public BlockPageId(String blockId, long pageIndex, long blockSize) { - this(Long.parseLong(blockId), pageIndex, blockSize); - } - - /** - * Creates an instance with a block ID as a {@code long}. - * @param blockId the block ID - * @param pageIndex index of the page in the block - * @param blockSize block size - */ - public BlockPageId(long blockId, long pageIndex, long blockSize) { - super(fileIdOf(blockId, blockSize), pageIndex); - mBlockId = blockId; - mBlockSize = blockSize; - } - - /** - * Creates a new page of the temporary block. - * @param blockId - * @param pageIndex - * @return page ID - */ - public static BlockPageId newTempPage(long blockId, long pageIndex) { - return new BlockPageId(blockId, pageIndex, INVALID_BLOCK_SIZE); - } - - /** - * @param blockId - * @param blockSize - * @return file ID - */ - public static String fileIdOf(long blockId, long blockSize) { - return String.format(FILE_ID_TEMPLATE, blockId, blockSize).intern(); - } - - /** - * @param blockId - * @return file ID - */ - public static String tempFileIdOf(long blockId) { - return fileIdOf(blockId, INVALID_BLOCK_SIZE); - } - - /** - * @param fileId - * @return block ID - * @throws IllegalArgumentException when the fileId does not contain a valid block ID - */ - public static long parseBlockId(String fileId) { - Matcher matcher = FILE_ID_PATTERN.matcher(fileId); - if (matcher.matches()) { - try { - // block id can be negative, Long.parseLong will throw in this case - return Long.parseUnsignedLong(matcher.group(1), 16); - } catch (NumberFormatException e) { - throw new IllegalArgumentException( - String.format("fileId %s does not contain a valid block ID", fileId), e); - } - } - throw new IllegalArgumentException( - String.format("fileId %s is not a valid paged block ID", fileId)); - } - - /** - * @param fileId - * @return block size - * @throws IllegalArgumentException when the fileId does not contain a valid block size - */ - public static long parseBlockSize(String fileId) { - Matcher matcher = FILE_ID_PATTERN.matcher(fileId); - if (matcher.matches()) { - try { - String blockSizeString = matcher.group(2); - long size = Long.parseLong(blockSizeString, 16); - Preconditions.checkArgument(size >= 0, "negative block size: %s", blockSizeString); - return size; - } catch (NumberFormatException e) { - throw new IllegalArgumentException( - String.format("fileId %s does not contain a valid block size", fileId), e); - } - } - throw new IllegalArgumentException( - String.format("fileId %s is not a valid paged block ID", fileId)); - } - - /** - * @return the block ID - */ - public long getBlockId() { - return mBlockId; - } - - /** - * @return the block size - */ - public long getBlockSize() { - return mBlockSize; - } - - /** - * Try to downcast a {@link PageId} to a {@link BlockPageId}. If the object is already a block - * page ID, then it is immediately returned. Otherwise, attempt to parse the {@code fileId} of - * the page if the encoded file name matches a block and contains all necessary metadata. - * - * @param pageId the page ID to downcast - * @return the downcast block page ID - * @throws IllegalArgumentException if the page ID cannot be cast to a block page ID - */ - public static BlockPageId downcast(PageId pageId) { - if (pageId instanceof BlockPageId) { - return (BlockPageId) pageId; - } - String fileId = pageId.getFileId(); - long blockId = parseBlockId(fileId); - long blockSize = parseBlockSize(fileId); - return new BlockPageId(blockId, pageId.getPageIndex(), blockSize); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof PageId)) { - return false; - } - // a fast path comparing longs instead of strings when both are BlockPageIds - if (o instanceof BlockPageId) { // we are final so instanceof check is ok - BlockPageId that = (BlockPageId) o; - // block size is metadata so shouldn't be considered for equality - return mBlockId == that.mBlockId && getPageIndex() == that.getPageIndex(); - } - // otherwise o is either the super class PageId or some other subclass of PageId. - // super.equals(o) does not work here because if o is a subclass of PageId, - // it may have its own unique fields, so need to call their equals method - return o.equals(this); - } - - // hashCode impl is intentionally not overridden to preserve compatibility - // with parent class - @Override - public int hashCode() { - return super.hashCode(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/NettyBufTargetBuffer.java b/core/server/worker/src/main/java/alluxio/worker/page/NettyBufTargetBuffer.java deleted file mode 100644 index ed8f2c84292e..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/NettyBufTargetBuffer.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.client.file.cache.store.PageReadTargetBuffer; - -import io.netty.buffer.ByteBuf; - -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.channels.WritableByteChannel; - -/** - * Netty Buf backed target buffer for zero-copy read from page store. - */ -public class NettyBufTargetBuffer implements PageReadTargetBuffer { - private final ByteBuf mTarget; - private long mOffset = 0; - - /** - * @param target target buffer - */ - public NettyBufTargetBuffer(ByteBuf target) { - mTarget = target; - } - - @Override - public byte[] byteArray() { - throw new UnsupportedOperationException(); - } - - @Override - public ByteBuffer byteBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public long offset() { - return mOffset; - } - - @Override - public WritableByteChannel byteChannel() { - return new WritableByteChannel() { - @Override - public int write(ByteBuffer src) throws IOException { - int readableBytes = src.remaining(); - mTarget.writeBytes(src); - return readableBytes - src.remaining(); - } - - @Override - public boolean isOpen() { - return true; - } - - @Override - public void close() throws IOException { - } - }; - } - - @Override - public long remaining() { - return mTarget.writableBytes(); - } - - @Override - public void writeBytes(byte[] srcArray, int srcOffset, int length) { - mTarget.writeBytes(srcArray, srcOffset, length); - } - - @Override - public int readFromFile(RandomAccessFile file, int length) throws IOException { - try (FileChannel channel = file.getChannel()) { - return mTarget.writeBytes(channel, length); - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMeta.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMeta.java deleted file mode 100644 index 28b27afb1253..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMeta.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.StorageDir; - -/** - * Metadata of a paged block. - */ -public class PagedBlockMeta implements BlockMeta { - private final long mBlockId; - private final long mBlockSize; - private final PagedBlockStoreDir mDir; - - /** - * @param blockId - * @param blockSize - * @param dir the directory that stores the pages of this block - */ - public PagedBlockMeta(long blockId, long blockSize, PagedBlockStoreDir dir) { - mBlockId = blockId; - mBlockSize = blockSize; - mDir = dir; - } - - @Override - public long getBlockId() { - return mBlockId; - } - - @Override - public BlockStoreLocation getBlockLocation() { - return mDir.getLocation(); - } - - @Override - public long getBlockSize() { - return mBlockSize; - } - - @Override - public StorageDir getParentDir() { - // todo(bowen): make PagedBlockStoreDir implement StorageDir? - throw new UnsupportedOperationException("getParentDir"); - } - - /** - * @return dir - */ - public PagedBlockStoreDir getDir() { - return mDir; - } - - @Override - public String getPath() { - // todo(bowen): paged block does not have a single file representation, this is most likely - // a directory; in case of memory page store, there is not path at all - return mDir.getRootPath().toString(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java deleted file mode 100644 index 1564190d76b3..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.client.file.cache.DefaultPageMetaStore; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageInfo; -import alluxio.client.file.cache.PageMetaStore; -import alluxio.client.file.cache.allocator.Allocator; -import alluxio.client.file.cache.allocator.HashAllocator; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.client.quota.CacheScope; -import alluxio.collections.IndexDefinition; -import alluxio.collections.IndexedSet; -import alluxio.exception.PageNotFoundException; -import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; -import alluxio.worker.block.BlockStoreEventListener; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.stream.Collectors; -import javax.annotation.concurrent.GuardedBy; - -/** - * This class manages metadata for the paged block store. - */ -public class PagedBlockMetaStore implements PageMetaStore { - private static final Logger LOG = LoggerFactory.getLogger(PagedBlockMetaStore.class); - @GuardedBy("getLock()") - private final PageMetaStore mDelegate; - @GuardedBy("getLock()") - private final IndexedSet mBlocks = - new IndexedSet<>(INDEX_BLOCK_ID, INDEX_STORE_DIR); - private final IndexedSet mTempBlocks = - new IndexedSet<>(INDEX_TEMP_BLOCK_ID, INDEX_TEMP_STORE_DIR); - - private final List mBlockStoreEventListeners = - new CopyOnWriteArrayList<>(); - - private static final IndexDefinition INDEX_BLOCK_ID = - new IndexDefinition(true) { - @Override - public Long getFieldValue(PagedBlockMeta o) { - return o.getBlockId(); - } - }; - private static final IndexDefinition INDEX_STORE_DIR = - new IndexDefinition(false) { - @Override - public PagedBlockStoreDir getFieldValue(PagedBlockMeta o) { - return o.getDir(); - } - }; - - private static final IndexDefinition INDEX_TEMP_BLOCK_ID = - new IndexDefinition(true) { - @Override - public Long getFieldValue(PagedTempBlockMeta o) { - return o.getBlockId(); - } - }; - private static final - IndexDefinition INDEX_TEMP_STORE_DIR = - new IndexDefinition(false) { - @Override - public PagedBlockStoreDir getFieldValue(PagedTempBlockMeta o) { - return o.getDir(); - } - }; - - private class BlockPageAllocator implements Allocator { - private final Allocator mDelegate; - - private BlockPageAllocator(Allocator delegate) { - mDelegate = delegate; - } - - /** - * Allocates a dir for a page of a block. If any other page of the block is being cached in - * a dir, use the same dir for this page. - */ - @Override - public PageStoreDir allocate(String fileId, long fileLength) { - long blockId = BlockPageId.parseBlockId(fileId); - PagedBlockMeta blockMeta = mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId); - if (blockMeta != null) { - return blockMeta.getDir(); - } - return mDelegate.allocate(fileId, fileLength); - } - } - - /** - * @param dirs the storage dirs - */ - public PagedBlockMetaStore(List dirs) { - this(dirs, new HashAllocator(ImmutableList.copyOf(dirs))); - } - - /** - * @param dirs dirs - * @param allocator allocator - */ - public PagedBlockMetaStore(List dirs, Allocator allocator) { - mDelegate = new DefaultPageMetaStore(ImmutableList.copyOf(dirs), - new BlockPageAllocator(allocator)); - } - - /** - * Checks if at least one page of a block is currently being stored in the cache. - * - * @param blockId block ID - * @return true if the block is stored in the cache, false otherwise - */ - @GuardedBy("getLock().readLock()") - public boolean hasBlock(long blockId) { - PagedBlockMeta blockMeta = mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId); - return blockMeta != null && blockMeta.getDir().getBlockCachedPages(blockId) > 0; - } - - /** - * @param blockId - * @return if - */ - @GuardedBy("getLock().readLock()") - public boolean hasTempBlock(long blockId) { - PagedTempBlockMeta blockMeta = mTempBlocks.getFirstByField(INDEX_TEMP_BLOCK_ID, blockId); - return blockMeta != null; - } - - /** - * Checks if the block is fully cached in the block store. - * - * @param blockId block ID - * @return true if the block has been fully cached, false otherwise - */ - public boolean hasFullBlock(long blockId) { - PagedBlockMeta blockMeta = mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId); - return blockMeta != null - && blockMeta.getDir().getBlockCachedBytes(blockId) == blockMeta.getBlockSize(); - } - - /** - * Allocates a page store directory for a page of a block. - * The implementation additionally guarantees that, as long as there is at least one page of the - * block is being stored, all subsequent pages always get allocated to the same directory as the - * firstly allocated page. - * However, it is unspecified whether the same directory will be chosen for a block, when all - * of its pages are removed from the page store and then added back. - * - * @param fileId the block ID - * @param pageSize size of the page - * @return the allocated page store dir - */ - @Override - @GuardedBy("getLock().readLock()") - public PageStoreDir allocate(String fileId, long pageSize) { - return mDelegate.allocate(fileId, pageSize); - } - - @Override - @GuardedBy("getLock().readLock()") - public PageInfo getPageInfo(PageId pageId) throws PageNotFoundException { - return mDelegate.getPageInfo(pageId); - } - - @Override - public ReadWriteLock getLock() { - return mDelegate.getLock(); - } - - @Override - @GuardedBy("getLock().readLock()") - public boolean hasPage(PageId pageId) { - return mDelegate.hasPage(pageId); - } - - @Override - @GuardedBy("getLock().writeLock()") - public void addPage(PageId pageId, PageInfo pageInfo) { - final BlockPageId blockPageId; - if (pageId instanceof BlockPageId) { - // the page is being added from paged block store internally - blockPageId = (BlockPageId) pageId; - } else { - // the page is being added from external code, typically by cache manager - // when the worker restarts - blockPageId = BlockPageId.downcast(pageId); - pageInfo = new PageInfo(blockPageId, pageInfo.getPageSize(), - pageInfo.getScope(), pageInfo.getLocalCacheDir()); - } - - long blockId = blockPageId.getBlockId(); - if (!mBlocks.contains(INDEX_BLOCK_ID, blockId)) { - long blockSize = blockPageId.getBlockSize(); - PagedBlockStoreDir dir = downcast(pageInfo.getLocalCacheDir()); - mBlocks.add(new PagedBlockMeta(blockId, blockSize, dir)); - } - mDelegate.addPage(pageId, pageInfo); - } - - /** - * Gets the block meta for a page of the block. - * @param pageId the page ID - * @return block meta - * @throws BlockDoesNotExistRuntimeException when the block is not being stored in the store - */ - private PagedBlockMeta getBlockMetaOfPage(PageId pageId) { - long blockId = BlockPageId.downcast(pageId).getBlockId(); - PagedBlockMeta blockMeta = mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId); - if (blockMeta == null) { - throw new BlockDoesNotExistRuntimeException(blockId); - } - return blockMeta; - } - - @Override - @GuardedBy("getLock().writeLock()") - public void addTempPage(PageId pageId, PageInfo pageInfo) { - long blockId = BlockPageId.downcast(pageId).getBlockId(); - PagedTempBlockMeta blockMeta = mTempBlocks.getFirstByField(INDEX_TEMP_BLOCK_ID, blockId); - if (blockMeta == null) { - throw new BlockDoesNotExistRuntimeException(blockId); - } - mDelegate.addTempPage(pageId, pageInfo); - blockMeta.setBlockSize(blockMeta.getBlockSize() + pageInfo.getPageSize()); - } - - @Override - public void commitFile(String fileId, String newFileId) throws PageNotFoundException { - mDelegate.commitFile(fileId, newFileId); - } - - /** - * @param blockId - * @return the permanent block meta after committing - */ - public PagedBlockMeta commit(long blockId) { - PagedTempBlockMeta tempBlockMeta = mTempBlocks.getFirstByField(INDEX_TEMP_BLOCK_ID, blockId); - if (tempBlockMeta == null) { - throw new BlockDoesNotExistRuntimeException(blockId); - } - PagedBlockMeta blockMeta = new PagedBlockMeta(tempBlockMeta.getBlockId(), - tempBlockMeta.getBlockSize(), tempBlockMeta.getDir()); - mTempBlocks.remove(tempBlockMeta); - mBlocks.add(blockMeta); - try { - commitFile(BlockPageId.tempFileIdOf(blockId), - BlockPageId.fileIdOf(blockId, blockMeta.getBlockSize())); - } catch (PageNotFoundException e) { - // this should be unreachable, since we have checked the existence of the block - // otherwise it's a bug - LOG.error("Cannot commit block {} as no pages are found", blockId, e); - throw new RuntimeException(e); - } - return blockMeta; - } - - @Override - public List getStoreDirs() { - return mDelegate.getStoreDirs(); - } - - @Override - @GuardedBy("getLock().writeLock()") - public PageInfo removePage(PageId pageId) throws PageNotFoundException { - PagedBlockMeta blockMeta = getBlockMetaOfPage(pageId); - long blockId = blockMeta.getBlockId(); - PagedBlockStoreDir dir = blockMeta.getDir(); - PageInfo pageInfo = mDelegate.removePage(pageId); - if (dir.getBlockCachedPages(blockId) == 0) { // last page of this block has been removed - mBlocks.remove(blockMeta); - BlockStoreLocation location = dir.getLocation(); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onRemoveBlock(blockId, location); - } - } - } - return pageInfo; - } - - @Override - @GuardedBy("getLock().readLock()") - public long bytes() { - return mDelegate.bytes(); - } - - @Override - @GuardedBy("getLock().readLock()") - public long numPages() { - return mDelegate.numPages(); - } - - @Override - @GuardedBy("getLock().writeLock()") - public void reset() { - mDelegate.reset(); - mBlocks.clear(); - } - - @Override - @GuardedBy("getLock().readLock()") - public PageInfo evict(CacheScope cacheScope, PageStoreDir pageStoreDir) { - return mDelegate.evict(cacheScope, pageStoreDir); - } - - /** - * @param blockMeta - */ - public void addBlock(PagedBlockMeta blockMeta) { - mBlocks.add(blockMeta); - } - - /** - * Adds a temp block for writing. The block is always pinned so that its pages don't get - * evicted before the block is committed. - * @param blockMeta the temp block to add - */ - public void addTempBlock(PagedTempBlockMeta blockMeta) { - mTempBlocks.add(blockMeta); - // a temp block always needs to be pinned as a client is actively writing it - blockMeta.getDir().getEvictor().addPinnedBlock(blockMeta.getBlockId()); - } - - @GuardedBy("getLock().readLock()") - Optional getBlock(long blockId) { - return Optional.ofNullable(mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId)); - } - - @GuardedBy("getLock().readLock()") - Optional getTempBlock(long blockId) { - return Optional.ofNullable(mTempBlocks.getFirstByField(INDEX_TEMP_BLOCK_ID, blockId)); - } - - /** - * @param listener listener - */ - public void registerBlockStoreEventListener(BlockStoreEventListener listener) { - mBlockStoreEventListeners.add(listener); - } - - /** - * @return brief store meta - */ - public PagedBlockStoreMeta getStoreMeta() { - final List pageStoreDirs = getStoreDirs(); - ImmutableList.Builder dirPaths = ImmutableList.builder(); - ImmutableList.Builder capacityOnDirs = ImmutableList.builder(); - ImmutableList.Builder usedBytesOnDirs = ImmutableList.builder(); - int numBlocks = 0; - for (PageStoreDir dir : pageStoreDirs) { - final PagedBlockStoreDir pagedBlockStoreDir = downcast(dir); - dirPaths.add(pagedBlockStoreDir.getRootPath().toString()); - capacityOnDirs.add(pagedBlockStoreDir.getCapacityBytes()); - usedBytesOnDirs.add(pagedBlockStoreDir.getCachedBytes()); - numBlocks += pagedBlockStoreDir.getNumBlocks(); - } - return new PagedBlockStoreMeta(dirPaths.build(), capacityOnDirs.build(), - usedBytesOnDirs.build(), numBlocks); - } - - /** - * @return detailed store meta including all block locations - */ - @GuardedBy("getLock().readLock()") - public PagedBlockStoreMeta getStoreMetaFull() { - final List pageStoreDirs = getStoreDirs(); - ImmutableList.Builder dirPaths = ImmutableList.builder(); - ImmutableList.Builder capacityOnDirs = ImmutableList.builder(); - ImmutableList.Builder usedBytesOnDirs = ImmutableList.builder(); - ImmutableMap.Builder> blockOnDirs = ImmutableMap.builder(); - for (PageStoreDir pageStoreDir : pageStoreDirs) { - final PagedBlockStoreDir pagedBlockStoreDir = downcast(pageStoreDir); - Set blocksOfDir = mBlocks.getByField(INDEX_STORE_DIR, pagedBlockStoreDir); - dirPaths.add(pagedBlockStoreDir.getRootPath().toString()); - capacityOnDirs.add(pagedBlockStoreDir.getCapacityBytes()); - usedBytesOnDirs.add(pagedBlockStoreDir.getCachedBytes()); - BlockStoreLocation location = pagedBlockStoreDir.getLocation(); - List blocks = blocksOfDir.stream() - .map(PagedBlockMeta::getBlockId) - .collect(Collectors.toList()); - blockOnDirs.put(location, blocks); - } - return new PagedBlockStoreMeta(dirPaths.build(), capacityOnDirs.build(), - usedBytesOnDirs.build(), blockOnDirs.build()); - } - - private static PagedBlockStoreDir downcast(PageStoreDir pageStoreDir) { - if (pageStoreDir instanceof PagedBlockStoreDir) { - return (PagedBlockStoreDir) pageStoreDir; - } - throw new IllegalArgumentException( - String.format("Unexpected page store dir type %s, for worker page store it should be %s", - pageStoreDir.getClass().getSimpleName(), PagedBlockStoreDir.class.getSimpleName())); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java deleted file mode 100644 index a0a09d2b8b41..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.CacheManager; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.store.PageReadTargetBuffer; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.grpc.ErrorType; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.network.protocol.databuffer.NioDirectBufferPool; -import alluxio.worker.block.io.BlockReader; - -import com.google.common.base.Preconditions; -import io.grpc.Status; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; -import java.util.Optional; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * A paged implementation of BlockReader interface. The read operations will fall back to the - * under storage when the requested data is not in the local storage. - */ -@NotThreadSafe -public class PagedBlockReader extends BlockReader { - - private static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.allocate(0); - private final long mPageSize; - private final CacheManager mCacheManager; - private final Optional mUfsBlockReader; - private final PagedBlockMeta mBlockMeta; - private boolean mClosed = false; - private boolean mReadFromLocalCache = false; - private boolean mReadFromUfs = false; - private long mPosition; - - /** - * Constructor for PagedBlockReader. - * - * @param cacheManager paging cache manager - * @param blockMeta block meta - * @param offset initial offset within the block to begin the read from - * @param ufsBlockReader ufs block reader - * @param pageSize page size - */ - public PagedBlockReader(CacheManager cacheManager, PagedBlockMeta blockMeta, long offset, - Optional ufsBlockReader, long pageSize) { - Preconditions.checkArgument(offset >= 0 && offset <= blockMeta.getBlockSize(), - "Attempt to read block %d which is %d bytes long at invalid byte offset %d", - blockMeta.getBlockId(), blockMeta.getBlockSize(), offset); - mCacheManager = cacheManager; - mUfsBlockReader = ufsBlockReader; - mBlockMeta = blockMeta; - mPageSize = pageSize; - mPosition = offset; - } - - @Override - public ByteBuffer read(long offset, long length) throws IOException { - if (length == 0 || offset >= mBlockMeta.getBlockSize()) { - return EMPTY_BYTE_BUFFER; - } - // cap length to the remaining of block, as the caller may pass in a longer length than what - // is left in the block, but expect as many bytes as there is - length = Math.min(length, mBlockMeta.getBlockSize() - offset); - ensureReadable(offset, length); - - // must not use pooled buffer, see interface implementation note - ByteBuffer buffer = ByteBuffer.allocateDirect((int) length); - ByteBuf buf = Unpooled.wrappedBuffer(buffer); - // Unpooled.wrappedBuffer returns a buffer with writer index set to capacity, so writable - // bytes is 0, needs explicit clear - buf.clear(); - long bytesRead = read(buf, offset, length); - if (bytesRead < 0) { - return EMPTY_BYTE_BUFFER; - } - buffer.position(0); - buffer.limit((int) bytesRead); - return buffer; - } - - /** - * Preconditions: - * 1. reader not closed - * 2. offset and length must be valid, check them with ensureReadable - * 3. enough space left in buffer for the bytes to read - */ - private long read(ByteBuf byteBuf, long offset, long length) throws IOException { - Preconditions.checkArgument(byteBuf.writableBytes() >= length, - "buffer overflow, trying to write %s bytes, only %s writable", - length, byteBuf.writableBytes()); - PageReadTargetBuffer target = new NettyBufTargetBuffer(byteBuf); - long bytesRead = 0; - while (bytesRead < length) { - long pos = offset + bytesRead; - long pageIndex = pos / mPageSize; - PageId pageId = - new BlockPageId(mBlockMeta.getBlockId(), pageIndex, mBlockMeta.getBlockSize()); - int currentPageOffset = (int) (pos % mPageSize); - int bytesLeftInPage = - (int) Math.min(mPageSize - currentPageOffset, length - bytesRead); - int bytesReadFromCache = mCacheManager.get( - pageId, currentPageOffset, bytesLeftInPage, target, CacheContext.defaults()); - if (bytesReadFromCache > 0) { - bytesRead += bytesReadFromCache; - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).mark(bytesRead); - mReadFromLocalCache = true; - } else { - if (!mUfsBlockReader.isPresent()) { - throw new AlluxioRuntimeException( - Status.INTERNAL, - String.format("Block %d cannot be read from UFS as UFS reader is missing, " - + "this is most likely a bug", mBlockMeta.getBlockId()), - null, - ErrorType.Internal, - false - ); - } - PagedUfsBlockReader ufsBlockReader = mUfsBlockReader.get(); - // get the page at pageIndex as a whole from UFS - ByteBuffer ufsBuf = NioDirectBufferPool.acquire((int) mPageSize); - try { - int pageBytesRead = ufsBlockReader.readPageAtIndex(ufsBuf, pageIndex); - if (pageBytesRead > 0) { - ufsBuf.position(currentPageOffset); - ufsBuf.limit(currentPageOffset + bytesLeftInPage); - byteBuf.writeBytes(ufsBuf); - bytesRead += bytesLeftInPage; - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()) - .mark(bytesLeftInPage); - mReadFromUfs = true; - ufsBuf.rewind(); - ufsBuf.limit(pageBytesRead); - if (ufsBlockReader.getUfsReadOptions().isCacheIntoAlluxio()) { - mCacheManager.put(pageId, ufsBuf); - } - } - } finally { - NioDirectBufferPool.release(ufsBuf); - } - } - } - return bytesRead; - } - - @Override - public long getLength() { - return mBlockMeta.getBlockSize(); - } - - @Override - public ReadableByteChannel getChannel() { - throw new UnsupportedOperationException(); - } - - @Override - public int transferTo(ByteBuf buf) throws IOException { - if (mBlockMeta.getBlockSize() <= mPosition) { - return -1; - } - int bytesToTransfer = - (int) Math.min(buf.writableBytes(), mBlockMeta.getBlockSize() - mPosition); - ensureReadable(mPosition, bytesToTransfer); - long bytesRead = read(buf, mPosition, bytesToTransfer); - mPosition += bytesRead; - return (int) bytesRead; - } - - @Override - public boolean isClosed() { - return mClosed; - } - - @Override - public String getLocation() { - return mBlockMeta.getPath(); - } - - @Override - public void close() throws IOException { - if (!isClosed()) { - if (mReadFromLocalCache) { - MetricsSystem.counter(MetricKey.WORKER_BLOCKS_READ_LOCAL.getName()).inc(); - } - if (mReadFromUfs) { - MetricsSystem.counter(MetricKey.WORKER_BLOCKS_READ_UFS.getName()).inc(); - } - } - mClosed = true; - } - - private void ensureReadable(long offset, long length) { - Preconditions.checkState(!mClosed, "reader closed"); - Preconditions.checkArgument(length >= 0, "negative read length %s", length); - Preconditions.checkArgument(offset >= 0, "negative offset %s", offset); - Preconditions.checkArgument(offset <= mBlockMeta.getBlockSize(), - "offset (%s) exceeds block size (%s)", offset, mBlockMeta.getBlockSize()); - Preconditions.checkArgument( - offset + length >= 0 && offset + length <= mBlockMeta.getBlockSize(), - "read end %s exceed block size %s", offset + length, mBlockMeta.getBlockSize()); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStore.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStore.java deleted file mode 100644 index 90837874c864..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStore.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_MEDIUM; -import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_TIER; - -import alluxio.client.file.cache.CacheManager; -import alluxio.client.file.cache.CacheManagerOptions; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.exception.BlockAlreadyExistsException; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.exception.runtime.AlreadyExistsRuntimeException; -import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; -import alluxio.grpc.Block; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.ErrorType; -import alluxio.grpc.UfsReadOptions; -import alluxio.proto.dataserver.Protocol; -import alluxio.resource.LockResource; -import alluxio.underfs.UfsManager; -import alluxio.worker.block.AllocateOptions; -import alluxio.worker.block.BlockLock; -import alluxio.worker.block.BlockLockManager; -import alluxio.worker.block.BlockLockType; -import alluxio.worker.block.BlockMasterClient; -import alluxio.worker.block.BlockMasterClientPool; -import alluxio.worker.block.BlockStore; -import alluxio.worker.block.BlockStoreEventListener; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.BlockStoreMeta; -import alluxio.worker.block.CreateBlockOptions; -import alluxio.worker.block.UfsInputStreamCache; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.io.DelegatingBlockReader; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.TempBlockMeta; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import io.grpc.Status; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicReference; - -/** - * A paged implementation of LocalBlockStore interface. - * Implements the block level operations， but instead of using physical block files, - * we use pages managed by the CacheManager to store the data. - */ -public class PagedBlockStore implements BlockStore { - private static final Logger LOG = LoggerFactory.getLogger(PagedBlockStore.class); - - private final CacheManager mCacheManager; - private final UfsManager mUfsManager; - - private final BlockLockManager mLockManager = new BlockLockManager(); - private final PagedBlockMetaStore mPageMetaStore; - private final BlockMasterClientPool mBlockMasterClientPool; - private final AtomicReference mWorkerId; - /** A set of pinned inodes updated via periodic master-worker sync. */ - private final Set mPinnedInodes = new HashSet<>(); - private final UfsInputStreamCache mUfsInStreamCache = new UfsInputStreamCache(); - private final List mBlockStoreEventListeners = - new CopyOnWriteArrayList<>(); - private final long mPageSize; - - /** - * Create an instance of PagedBlockStore. - * @param ufsManager - * @param pool - * @param workerId - * @return an instance of PagedBlockStore - */ - public static PagedBlockStore create(UfsManager ufsManager, BlockMasterClientPool pool, - AtomicReference workerId) { - try { - AlluxioConfiguration conf = Configuration.global(); - CacheManagerOptions cacheManagerOptions = CacheManagerOptions.createForWorker(conf); - List pageStoreDirs = PageStoreDir.createPageStoreDirs(cacheManagerOptions); - List dirs = PagedBlockStoreDir.fromPageStoreDirs(pageStoreDirs); - PagedBlockMetaStore pageMetaStore = new PagedBlockMetaStore(dirs); - CacheManager cacheManager = - CacheManager.Factory.create(conf, cacheManagerOptions, pageMetaStore); - return new PagedBlockStore(cacheManager, ufsManager, pool, workerId, pageMetaStore, - cacheManagerOptions.getPageSize()); - } catch (IOException e) { - throw new RuntimeException("Failed to create PagedLocalBlockStore", e); - } - } - - /** - * Constructor for PagedLocalBlockStore. - * - * @param cacheManager page cache manager - * @param ufsManager ufs manager - * @param pageMetaStore meta data store for pages and blocks - * @param pageSize page size - */ - PagedBlockStore(CacheManager cacheManager, UfsManager ufsManager, BlockMasterClientPool pool, - AtomicReference workerId, PagedBlockMetaStore pageMetaStore, - long pageSize) { - mCacheManager = cacheManager; - mUfsManager = ufsManager; - mBlockMasterClientPool = pool; - mWorkerId = workerId; - mPageMetaStore = pageMetaStore; - mPageSize = pageSize; - } - - @Override - public Optional pinBlock(long sessionId, long blockId) { - LOG.debug("pinBlock: sessionId={}, blockId={}", sessionId, blockId); - BlockLock lock = mLockManager.acquireBlockLock(sessionId, blockId, BlockLockType.READ); - if (hasBlockMeta(blockId)) { - return Optional.of(lock); - } - lock.close(); - return Optional.empty(); - } - - @Override - public void unpinBlock(BlockLock lock) { - LOG.debug("unpinBlock: id={}", lock.get()); - lock.close(); - } - - @Override - public void commitBlock(long sessionId, long blockId, boolean pinOnCreate) { - LOG.debug("commitBlock: sessionId={}, blockId={}, pinOnCreate={}", - sessionId, blockId, pinOnCreate); - try (BlockLock blockLock = - mLockManager.acquireBlockLock(sessionId, blockId, BlockLockType.WRITE); - LockResource metaLock = new LockResource(mPageMetaStore.getLock().writeLock())) { - PagedBlockMeta blockMeta = mPageMetaStore.getTempBlock(blockId) - .orElseThrow(() -> new BlockDoesNotExistRuntimeException(blockId)); - Preconditions.checkState( - blockMeta.getBlockSize() == blockMeta.getDir().getTempBlockCachedBytes(blockId), - "committing a block which has not been not fully written" - ); - PagedBlockStoreDir pageStoreDir = blockMeta.getDir(); - // unconditionally pin this block until committing is done - boolean isPreviouslyUnpinned = pageStoreDir.getEvictor().addPinnedBlock(blockId); - try { - pageStoreDir.commit(BlockPageId.tempFileIdOf(blockId), - BlockPageId.fileIdOf(blockId, blockMeta.getBlockSize())); - final PagedBlockMeta committed = mPageMetaStore.commit(blockId); - commitBlockToMaster(committed); - } catch (IOException e) { - throw AlluxioRuntimeException.from(e); - } finally { - if (!pinOnCreate && isPreviouslyUnpinned) { - pageStoreDir.getEvictor().removePinnedBlock(blockId); - } - } - } - } - - /** - * Commits a block to master. The block must have been committed in metastore and storage dir. - * Caller must have acquired at least READ lock on the metastore and the block. - * @param blockMeta the block to commit - */ - private void commitBlockToMaster(PagedBlockMeta blockMeta) { - final long blockId = blockMeta.getBlockId(); - BlockMasterClient bmc = mBlockMasterClientPool.acquire(); - try { - bmc.commitBlock(mWorkerId.get(), mPageMetaStore.getStoreMeta().getUsedBytes(), DEFAULT_TIER, - DEFAULT_MEDIUM, blockId, blockMeta.getBlockSize()); - } catch (IOException e) { - throw new AlluxioRuntimeException(Status.UNAVAILABLE, - ExceptionMessage.FAILED_COMMIT_BLOCK_TO_MASTER.getMessage(blockId), e, ErrorType.Internal, - false); - } finally { - mBlockMasterClientPool.release(bmc); - } - BlockStoreLocation blockLocation = - new BlockStoreLocation(DEFAULT_TIER, getDirIndexOfBlock(blockId)); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onCommitBlock(blockId, blockLocation); - } - } - } - - @Override - public String createBlock(long sessionId, long blockId, int tier, - CreateBlockOptions createBlockOptions) { - String fileId = BlockPageId.tempFileIdOf(blockId); - PageStoreDir pageStoreDir = - mPageMetaStore.allocate(fileId, createBlockOptions.getInitialBytes()); - pageStoreDir.putTempFile(fileId); - return "DUMMY_FILE_PATH"; - } - - @Override - public BlockReader createBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException { - BlockLock blockLock = mLockManager.acquireBlockLock(sessionId, blockId, BlockLockType.READ); - - try (LockResource lock = new LockResource(mPageMetaStore.getLock().readLock())) { - Optional blockMeta = mPageMetaStore.getBlock(blockId); - if (blockMeta.isPresent()) { - final BlockPageEvictor evictor = blockMeta.get().getDir().getEvictor(); - evictor.addPinnedBlock(blockId); - return new DelegatingBlockReader(getBlockReader(blockMeta.get(), offset, options), () -> { - evictor.removePinnedBlock(blockId); - unpinBlock(blockLock); - }); - } - } - // this is a block that needs to be read from UFS - try (LockResource lock = new LockResource(mPageMetaStore.getLock().writeLock())) { - // in case someone else has added this block while we wait for the lock, - // just use the block meta; otherwise create a new one and add to the metastore - Optional blockMeta = mPageMetaStore.getBlock(blockId); - if (blockMeta.isPresent()) { - blockMeta.get().getDir().getEvictor().addPinnedBlock(blockId); - return new DelegatingBlockReader(getBlockReader(blockMeta.get(), offset, options), () -> { - blockMeta.get().getDir().getEvictor().removePinnedBlock(blockId); - unpinBlock(blockLock); - }); - } - long blockSize = options.getBlockSize(); - PagedBlockStoreDir dir = - (PagedBlockStoreDir) mPageMetaStore.allocate(BlockPageId.fileIdOf(blockId, blockSize), - blockSize); - PagedBlockMeta newBlockMeta = new PagedBlockMeta(blockId, blockSize, dir); - if (options.getNoCache()) { - // block does not need to be cached in Alluxio, no need to add and commit it - unpinBlock(blockLock); - final UfsBlockReadOptions readOptions; - try { - readOptions = UfsBlockReadOptions.fromProto(options); - } catch (IllegalArgumentException e) { - throw new AlluxioRuntimeException(Status.INTERNAL, - String.format("Block %d may need to be read from UFS, but key UFS read options " - + "is missing in client request", blockId), e, ErrorType.Internal, false); - } - return new PagedUfsBlockReader(mUfsManager, mUfsInStreamCache, newBlockMeta, - offset, readOptions, mPageSize); - } - mPageMetaStore.addBlock(newBlockMeta); - dir.getEvictor().addPinnedBlock(blockId); - return new DelegatingBlockReader(getBlockReader(newBlockMeta, offset, options), () -> { - commitBlockToMaster(newBlockMeta); - newBlockMeta.getDir().getEvictor().removePinnedBlock(blockId); - unpinBlock(blockLock); - }); - } - } - - private BlockReader getBlockReader(PagedBlockMeta blockMeta, long offset, - Protocol.OpenUfsBlockOptions options) { - final long blockId = blockMeta.getBlockId(); - Optional readOptions = Optional.empty(); - try { - readOptions = Optional.of(UfsBlockReadOptions.fromProto(options)); - } catch (IllegalArgumentException e) { - // the client does not provide enough information about how to read this block from UFS - // this is fine for e.g. MUST_CACHE files, so we will simply ignore the error here - // on the other hand, if the block being read should be readable from UFS, but - // somehow client didn't send the read options, we will raise the error in the block reader - // when encountered - LOG.debug("Client did not provide enough info to read block {} from UFS", blockId, e); - } - final Optional ufsBlockReader = - readOptions.map(opt -> new PagedUfsBlockReader( - mUfsManager, mUfsInStreamCache, blockMeta, offset, opt, mPageSize)); - return new PagedBlockReader(mCacheManager, blockMeta, offset, ufsBlockReader, mPageSize); - } - - @Override - public BlockReader createUfsBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, - Protocol.OpenUfsBlockOptions options) throws IOException { - PagedBlockMeta blockMeta = mPageMetaStore - .getBlock(blockId) - .orElseGet(() -> { - long blockSize = options.getBlockSize(); - PagedBlockStoreDir dir = - (PagedBlockStoreDir) mPageMetaStore.allocate(String.valueOf(blockId), blockSize); - // do not add the block to metastore - return new PagedBlockMeta(blockId, blockSize, dir); - }); - UfsBlockReadOptions readOptions = UfsBlockReadOptions.fromProto(options); - return new PagedUfsBlockReader(mUfsManager, mUfsInStreamCache, blockMeta, - offset, readOptions, mPageSize); - } - - @Override - public void abortBlock(long sessionId, long blockId) { - PagedTempBlockMeta blockMeta = mPageMetaStore.getTempBlock(blockId) - .orElseThrow(() -> new BlockDoesNotExistRuntimeException(blockId)); - try { - blockMeta.getDir().abort(BlockPageId.tempFileIdOf(blockId)); - } catch (IOException e) { - throw AlluxioRuntimeException.from(e); - } - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onAbortBlock(blockId); - } - } - } - - @Override - public void requestSpace(long sessionId, long blockId, long additionalBytes) { - // TODO(bowen): implement actual space allocation and replace placeholder values - boolean blockEvicted = false; - if (blockEvicted) { - long evictedBlockId = 0; - BlockStoreLocation evictedBlockLocation = new BlockStoreLocation(DEFAULT_TIER, 1); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onRemoveBlockByWorker(evictedBlockId); - listener.onRemoveBlock(evictedBlockId, evictedBlockLocation); - } - } - } - } - - @Override - public CompletableFuture> load(List fileBlocks, UfsReadOptions options) { - throw new UnsupportedOperationException(); - } - - @Override - public BlockWriter createBlockWriter(long sessionId, long blockId) - throws IOException { - // note: no need to take a write block lock here as the block will not be visible to other - // clients until the block is committed - try (LockResource lock = new LockResource(mPageMetaStore.getLock().writeLock())) { - if (!mPageMetaStore.hasBlock(blockId) && !mPageMetaStore.hasTempBlock(blockId)) { - PagedBlockStoreDir dir = - (PagedBlockStoreDir) mPageMetaStore.allocate(BlockPageId.tempFileIdOf(blockId), 0); - PagedTempBlockMeta blockMeta = new PagedTempBlockMeta(blockId, dir); - mPageMetaStore.addTempBlock(blockMeta); - return new PagedBlockWriter(mCacheManager, blockId, mPageSize); - } - } - throw new AlreadyExistsRuntimeException(new BlockAlreadyExistsException( - String.format("Cannot overwrite an existing block %d", blockId))); - } - - @Override - public void moveBlock(long sessionId, long blockId, AllocateOptions moveOptions) - throws IOException { - // TODO(bowen): implement actual move and replace placeholder values - int dirIndex = getDirIndexOfBlock(blockId); - BlockStoreLocation srcLocation = new BlockStoreLocation(DEFAULT_TIER, dirIndex); - BlockStoreLocation destLocation = moveOptions.getLocation(); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onMoveBlockByClient(blockId, srcLocation, destLocation); - } - } - throw new UnsupportedOperationException(); - } - - @Override - public void removeBlock(long sessionId, long blockId) throws IOException { - LOG.debug("removeBlock: sessionId={}, blockId={}", sessionId, blockId); - // TODO(bowen): implement actual removal and replace placeholder values - boolean removeSuccess = true; - int dirIndex = getDirIndexOfBlock(blockId); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onRemoveBlockByClient(blockId); - if (removeSuccess) { - BlockStoreLocation removedFrom = new BlockStoreLocation(DEFAULT_TIER, dirIndex); - listener.onRemoveBlock(blockId, removedFrom); - } - } - } - } - - @Override - public void accessBlock(long sessionId, long blockId) { - // TODO(bowen): implement actual access and replace placeholder values - boolean blockExists = true; - if (blockExists) { - int dirIndex = getDirIndexOfBlock(blockId); - BlockStoreLocation dummyLoc = new BlockStoreLocation(DEFAULT_TIER, dirIndex); - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - listener.onAccessBlock(blockId); - listener.onAccessBlock(blockId, dummyLoc); - } - } - } - throw new UnsupportedOperationException(); - } - - @Override - public BlockStoreMeta getBlockStoreMeta() { - return mPageMetaStore.getStoreMeta(); - } - - @Override - public BlockStoreMeta getBlockStoreMetaFull() { - return mPageMetaStore.getStoreMetaFull(); - } - - @Override - public Optional getTempBlockMeta(long blockId) { - return Optional.empty(); - } - - @Override - public boolean hasBlockMeta(long blockId) { - return mPageMetaStore.getBlock(blockId).isPresent(); - } - - @Override - public boolean hasTempBlockMeta(long blockId) { - return mPageMetaStore.getTempBlock(blockId).isPresent(); - } - - @Override - public Optional getVolatileBlockMeta(long blockId) { - return Optional.empty(); - } - - @Override - public void cleanupSession(long sessionId) { - // TODO(bowen): session cleaner seems to be defunct, as Sessions are always empty - } - - @Override - public void registerBlockStoreEventListener(BlockStoreEventListener listener) { - mBlockStoreEventListeners.add(listener); - mPageMetaStore.registerBlockStoreEventListener(listener); - } - - @Override - public void updatePinnedInodes(Set inodes) { - // TODO(bowen): this is unused now, make sure to use the pinned inodes when allocating space - LOG.debug("updatePinnedInodes: inodes={}", inodes); - synchronized (mPinnedInodes) { - mPinnedInodes.clear(); - mPinnedInodes.addAll(Preconditions.checkNotNull(inodes)); - } - } - - @Override - public void removeInaccessibleStorage() { - // TODO(bowen): implement actual removal and replace placeholder values - for (BlockStoreEventListener listener : mBlockStoreEventListeners) { - synchronized (listener) { - List lostBlocks = ImmutableList.of(); - // TODO(bowen): lost directories can be obtained by iterating dirs in PageMetaStore - // and check their health - String lostStoragePath = "lostDir"; - BlockStoreLocation lostStoreLocation = new BlockStoreLocation(DEFAULT_TIER, 1); - for (long lostBlock : lostBlocks) { - listener.onBlockLost(lostBlock); - } - listener.onStorageLost(DEFAULT_TIER, lostStoragePath); - listener.onStorageLost(lostStoreLocation); - } - } - } - - @Override - public void close() throws IOException { - } - - private int getDirIndexOfBlock(long blockId) { - return mPageMetaStore.getBlock(blockId) - .orElseThrow(() -> new BlockDoesNotExistRuntimeException(blockId)) - .getDir() - .getDirIndex(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java deleted file mode 100644 index d1ca6ad31dfc..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java +++ /dev/null @@ -1,268 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_MEDIUM; -import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_TIER; - -import alluxio.client.file.cache.PageInfo; -import alluxio.client.file.cache.PageStore; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.base.Preconditions; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -/** - * A directory storing paged blocks. - */ -public class PagedBlockStoreDir implements PageStoreDir { - protected final PageStoreDir mDelegate; - protected final int mIndex; - // block to pages mapping; if no pages have been added for a block, - // the block may not exist in this map, but it can still exist in mBlocks - protected final HashMultimap mBlockToPagesMap = HashMultimap.create(); - - protected final HashMultimap mTempBlockToPagesMap = HashMultimap.create(); - - protected final BlockStoreLocation mLocation; - - protected final BlockPageEvictor mEvictor; - - /** - * Creates a new dir. - * - * @param delegate the page store dir that is delegated to - * @param index the index of this dir in the tier - */ - public PagedBlockStoreDir(PageStoreDir delegate, int index) { - mDelegate = delegate; - mIndex = index; - mLocation = new BlockStoreLocation(DEFAULT_TIER, index, DEFAULT_MEDIUM); - mEvictor = new BlockPageEvictor(delegate.getEvictor()); - } - - /** - * Creates from a list of {@link PageStoreDir}. - * - * @param dirs dirs - * @return a list of {@link PagedBlockStoreDir} - */ - public static List fromPageStoreDirs(List dirs) { - ImmutableList.Builder listBuilder = ImmutableList.builder(); - for (int i = 0; i < dirs.size(); i++) { - listBuilder.add(new PagedBlockStoreDir(dirs.get(i), i)); - } - return listBuilder.build(); - } - - @Override - public Path getRootPath() { - return mDelegate.getRootPath(); - } - - @Override - public PageStore getPageStore() { - return mDelegate.getPageStore(); - } - - /** - * @return index of this directory in the list of all directories - */ - public int getDirIndex() { - return mIndex; - } - - /** - * @return the block storage location of this directory - */ - public BlockStoreLocation getLocation() { - return mLocation; - } - - /** - * @return number of blocks being stored in this dir - */ - public int getNumBlocks() { - return mBlockToPagesMap.keySet().size(); - } - - @Override - public long getCapacityBytes() { - return mDelegate.getCapacityBytes(); - } - - @Override - public void reset() throws IOException { - mDelegate.reset(); - mBlockToPagesMap.clear(); - mTempBlockToPagesMap.clear(); - mEvictor.reset(); - } - - @Override - public void scanPages(Consumer> pageInfoConsumer) throws IOException { - Consumer> wrapper = (optionalPageInfo) -> { - Optional mapped = optionalPageInfo.map(pageInfo -> { - Preconditions.checkArgument(pageInfo.getLocalCacheDir() == mDelegate, - "scanPages should only return pages under the delegated dir"); - BlockPageId blockPageId; - try { - blockPageId = BlockPageId.downcast(pageInfo.getPageId()); - } catch (IllegalArgumentException e) { - // not a paged block id, return as is - return pageInfo; - } - return new PageInfo(blockPageId, pageInfo.getPageSize(), this); - }); - pageInfoConsumer.accept(mapped); - }; - mDelegate.scanPages(wrapper); - } - - @Override - public long getCachedBytes() { - return mDelegate.getCachedBytes(); - } - - @Override - public void putPage(PageInfo pageInfo) { - long blockId = BlockPageId.downcast(pageInfo.getPageId()).getBlockId(); - if (mBlockToPagesMap.put(blockId, pageInfo)) { - mDelegate.putPage(pageInfo); - } - } - - @Override - public void putTempPage(PageInfo pageInfo) { - long blockId = BlockPageId.downcast(pageInfo.getPageId()).getBlockId(); - if (mTempBlockToPagesMap.put(blockId, pageInfo)) { - mDelegate.putTempPage(pageInfo); - } - } - - @Override - public boolean putTempFile(String fileId) { - return mDelegate.putTempFile(fileId); - } - - @Override - public boolean reserve(long bytes) { - // todo(bowen): check constraints and update used bytes, etc. - return mDelegate.reserve(bytes); - } - - @Override - public long deletePage(PageInfo pageInfo) { - long blockId = BlockPageId.downcast(pageInfo.getPageId()).getBlockId(); - if (mBlockToPagesMap.remove(blockId, pageInfo)) { - long used = mDelegate.deletePage(pageInfo); - if (!mBlockToPagesMap.containsKey(blockId)) { - mEvictor.removePinnedBlock(blockId); - } - return used; - } - return getCachedBytes(); - } - - @Override - public long release(long bytes) { - return mDelegate.release(bytes); - } - - @Override - public boolean hasFile(String fileId) { - return mDelegate.hasFile(fileId); - } - - @Override - public boolean hasTempFile(String fileId) { - return mDelegate.hasTempFile(fileId); - } - - @Override - public BlockPageEvictor getEvictor() { - return mEvictor; - } - - @Override - public void close() { - mDelegate.close(); - } - - @Override - public void commit(String fileId, String newFileId) throws IOException { - long blockId = BlockPageId.parseBlockId(fileId); - Preconditions.checkArgument( - BlockPageId.parseBlockId(newFileId) == blockId, - "committing with different block IDs: temp: %s, new: %s", fileId, newFileId); - mDelegate.commit(fileId, newFileId); - Set pages = mTempBlockToPagesMap.removeAll(blockId); - List newPages = pages.stream() - .map(page -> { - BlockPageId newPageId = new BlockPageId(blockId, page.getPageId().getPageIndex(), - BlockPageId.parseBlockSize(newFileId)); - return new PageInfo(newPageId, page.getPageSize(), page.getScope(), - page.getLocalCacheDir()); - }) - .collect(Collectors.toList()); - mBlockToPagesMap.putAll(blockId, newPages); - } - - @Override - public void abort(String fileId) throws IOException { - long blockId = BlockPageId.parseBlockId(fileId); - mDelegate.abort(fileId); - mTempBlockToPagesMap.removeAll(blockId); - mEvictor.removePinnedBlock(blockId); - } - - /** - * Gets how many bytes of a block is being cached by this dir. - * - * @param blockId the block id - * @return total size of pages of this block being cached - */ - public long getBlockCachedBytes(long blockId) { - return mBlockToPagesMap.get(blockId).stream().map(PageInfo::getPageSize).reduce(0L, Long::sum); - } - - /** - * Gets how many bytes of a temp block is being cached by this dir. - * - * @param blockId the block id - * @return total size of pages of this block being cached - */ - public long getTempBlockCachedBytes(long blockId) { - return mTempBlockToPagesMap.get(blockId).stream().map(PageInfo::getPageSize) - .reduce(0L, Long::sum); - } - - /** - * Gets how many pages of a block is being cached by this dir. - * - * @param blockId the block id - * @return total number of pages of this block being cached - */ - public int getBlockCachedPages(long blockId) { - return mBlockToPagesMap.get(blockId).size(); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreMeta.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreMeta.java deleted file mode 100644 index 87dd66b74f12..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreMeta.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.Constants; -import alluxio.DefaultStorageTierAssoc; -import alluxio.StorageTierAssoc; -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.BlockStoreMeta; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Streams; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import javax.annotation.Nullable; - -/** - * Metadata of the paged block store. - */ -public class PagedBlockStoreMeta implements BlockStoreMeta { - // Paged block store does not support multiple tiers, so assume everything is in the top tier - public static final String DEFAULT_TIER = - Configuration.getString(PropertyKey.MASTER_TIERED_STORE_GLOBAL_LEVEL0_ALIAS); - // top tier is conventionally ramdisk - public static final String DEFAULT_MEDIUM = Constants.MEDIUM_MEM; - public static final DefaultStorageTierAssoc DEFAULT_STORAGE_TIER_ASSOC = - new DefaultStorageTierAssoc(ImmutableList.of(DEFAULT_MEDIUM)); - - private final long mCapacity; - private final long mUsed; - private final int mNumBlocks; - private final List mDirPaths; - private final List mCapacityBytesOnDirs; - private final List mUsedBytesOnDirs; - private final Optional>> mBlockLocations; - private final Optional> mBlocks; - - PagedBlockStoreMeta(List storageDirPaths, List capacityBytesOnDirs, - List usedBytesOnDirs, int numBlocks) { - mDirPaths = storageDirPaths; - mCapacityBytesOnDirs = capacityBytesOnDirs; - mUsedBytesOnDirs = usedBytesOnDirs; - mCapacity = capacityBytesOnDirs.stream().reduce(0L, Long::sum); - mUsed = usedBytesOnDirs.stream().reduce(0L, Long::sum); - mNumBlocks = numBlocks; - mBlocks = Optional.empty(); - mBlockLocations = Optional.empty(); - } - - PagedBlockStoreMeta(List storageDirPaths, List capacityBytesOnDirs, - List usedBytesOnDirs, Map> blockLocations) { - mDirPaths = storageDirPaths; - mCapacityBytesOnDirs = capacityBytesOnDirs; - mUsedBytesOnDirs = usedBytesOnDirs; - mCapacity = capacityBytesOnDirs.stream().reduce(0L, Long::sum); - mUsed = usedBytesOnDirs.stream().reduce(0L, Long::sum); - - mBlockLocations = Optional.of(blockLocations); - ImmutableList.Builder blockIdsListBuilder = new ImmutableList.Builder<>(); - blockLocations.values().forEach(blockIdsListBuilder::addAll); - List blockIds = blockIdsListBuilder.build(); - mNumBlocks = blockIds.size(); - mBlocks = Optional.of(blockIds); - } - - @Nullable - @Override - public Map> getBlockList() { - return mBlocks.map(blocks -> ImmutableMap.of(DEFAULT_TIER, blocks)).orElse(null); - } - - @Nullable - @Override - public Map> getBlockListByStorageLocation() { - return mBlockLocations.orElse(null); - } - - @Override - public long getCapacityBytes() { - return mCapacity; - } - - @Override - public Map getCapacityBytesOnTiers() { - return ImmutableMap.of(DEFAULT_TIER, mCapacity); - } - - @Override - public Map, Long> getCapacityBytesOnDirs() { - return Streams.zip(mDirPaths.stream(), mCapacityBytesOnDirs.stream(), Pair::new) - .collect(ImmutableMap.toImmutableMap( - pair -> new Pair<>(DEFAULT_TIER, pair.getFirst()), - Pair::getSecond - )); - } - - @Override - public Map> getDirectoryPathsOnTiers() { - return ImmutableMap.of(DEFAULT_TIER, mDirPaths); - } - - @Override - public Map> getLostStorage() { - return ImmutableMap.of(); - } - - @Override - public int getNumberOfBlocks() { - return mNumBlocks; - } - - @Override - public long getUsedBytes() { - return mUsed; - } - - @Override - public Map getUsedBytesOnTiers() { - return ImmutableMap.of(DEFAULT_TIER, mUsed); - } - - @Override - public Map, Long> getUsedBytesOnDirs() { - return Streams.zip(mDirPaths.stream(), mUsedBytesOnDirs.stream(), Pair::new) - .collect(ImmutableMap.toImmutableMap( - pair -> new Pair<>(DEFAULT_TIER, pair.getFirst()), - Pair::getSecond - )); - } - - @Override - public StorageTierAssoc getStorageTierAssoc() { - return DEFAULT_STORAGE_TIER_ASSOC; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockWriter.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockWriter.java deleted file mode 100644 index 9ba7f2456f12..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockWriter.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.CacheManager; -import alluxio.client.file.cache.PageId; -import alluxio.exception.runtime.InternalRuntimeException; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.worker.block.io.BlockWriter; - -import io.netty.buffer.ByteBuf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; - -/** - * A paged implementation of BlockWriter interface. - */ -public class PagedBlockWriter extends BlockWriter { - private static final Logger LOG = LoggerFactory.getLogger(PagedBlockWriter.class); - private static final CacheContext TEMP_CACHE_CONTEXT = CacheContext.defaults().setTemporary(true); - - private final CacheManager mCacheManager; - private final long mBlockId; - private final long mPageSize; - private long mPosition; - - PagedBlockWriter(CacheManager cacheManager, long blockId, long pageSize) { - mCacheManager = cacheManager; - mBlockId = blockId; - mPageSize = pageSize; - } - - @Override - public long append(ByteBuffer inputBuf) { - long bytesWritten = 0; - while (inputBuf.hasRemaining()) { - PageId pageId = getPageId(bytesWritten); - int currentPageOffset = getCurrentPageOffset(bytesWritten); - int bytesLeftInPage = getBytesLeftInPage(currentPageOffset, inputBuf.remaining()); - byte[] page = new byte[bytesLeftInPage]; - inputBuf.get(page); - if (!mCacheManager.append(pageId, currentPageOffset, page, TEMP_CACHE_CONTEXT)) { - throw new InternalRuntimeException("Append failed for block " + mBlockId); - } - bytesWritten += bytesLeftInPage; - } - mPosition += bytesWritten; - return bytesWritten; - } - - @Override - public long append(ByteBuf buf) throws IOException { - long bytesWritten = 0; - while (buf.readableBytes() > 0) { - PageId pageId = getPageId(bytesWritten); - int currentPageOffset = getCurrentPageOffset(bytesWritten); - int bytesLeftInPage = getBytesLeftInPage(currentPageOffset, buf.readableBytes()); - byte[] page = new byte[bytesLeftInPage]; - buf.readBytes(page); - if (!mCacheManager.append(pageId, currentPageOffset, page, TEMP_CACHE_CONTEXT)) { - throw new IOException("Append failed for block " + mBlockId); - } - bytesWritten += bytesLeftInPage; - } - mPosition += bytesWritten; - return bytesWritten; - } - - @Override - public long append(DataBuffer buffer) throws IOException { - ByteBuf bytebuf = null; - try { - bytebuf = (ByteBuf) buffer.getNettyOutput(); - } catch (Throwable e) { - LOG.debug("Failed to get ByteBuf from DataBuffer, write performance may be degraded."); - } - if (bytebuf != null) { - return append(bytebuf); - } - return append(buffer.getReadOnlyByteBuffer()); - } - - @Override - public long getPosition() { - return mPosition; - } - - @Override - public WritableByteChannel getChannel() { - throw new UnsupportedOperationException(); - } - - private PageId getPageId(long bytesWritten) { - long pageIndex = (mPosition + bytesWritten) / mPageSize; - return BlockPageId.newTempPage(mBlockId, pageIndex); - } - - private int getCurrentPageOffset(long bytesWritten) { - return (int) ((mPosition + bytesWritten) % mPageSize); - } - - private int getBytesLeftInPage(int currentPageOffset, int remaining) { - return (int) Math.min(mPageSize - currentPageOffset, remaining); - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedTempBlockMeta.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedTempBlockMeta.java deleted file mode 100644 index 683aa28f0fcd..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedTempBlockMeta.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import com.google.common.base.Preconditions; - -/** - * Temp block meta for a paged block. - */ -public class PagedTempBlockMeta extends PagedBlockMeta { - private long mBlockSize; - - /** - * @param blockId - * @param dir - */ - public PagedTempBlockMeta(long blockId, PagedBlockStoreDir dir) { - super(blockId, 0, dir); - mBlockSize = 0; - } - - @Override - public long getBlockSize() { - return mBlockSize; - } - - /** - * @param newSize - */ - public void setBlockSize(long newSize) { - Preconditions.checkState(mBlockSize <= newSize, "Shrinking block, not supported!"); - mBlockSize = newSize; - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedUfsBlockReader.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedUfsBlockReader.java deleted file mode 100644 index ef79f18348da..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedUfsBlockReader.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.conf.PropertyKey; -import alluxio.network.protocol.databuffer.NioDirectBufferPool; -import alluxio.resource.CloseableResource; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.options.OpenOptions; -import alluxio.util.IdUtils; -import alluxio.worker.block.UfsInputStreamCache; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.meta.BlockMeta; - -import com.google.common.base.Preconditions; -import io.netty.buffer.ByteBuf; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.ReadableByteChannel; - -/** - * Block reader that reads from UFS. - */ -public class PagedUfsBlockReader extends BlockReader { - private static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.allocate(0); - private final long mPageSize; - private final UfsManager mUfsManager; - private final UfsInputStreamCache mUfsInStreamCache; - private final BlockMeta mBlockMeta; - private final UfsBlockReadOptions mUfsBlockOptions; - private final long mInitialOffset; - private final ByteBuffer mLastPage; - private long mLastPageIndex = -1; - private boolean mClosed = false; - private long mPosition; - - /** - * @param ufsManager - * @param ufsInStreamCache - * @param blockMeta - * @param offset - * @param ufsBlockReadOptions - * @param pageSize - */ - public PagedUfsBlockReader(UfsManager ufsManager, - UfsInputStreamCache ufsInStreamCache, BlockMeta blockMeta, - long offset, UfsBlockReadOptions ufsBlockReadOptions, long pageSize) { - Preconditions.checkArgument(offset >= 0 && offset <= blockMeta.getBlockSize(), - "Attempt to read block %s which is %s bytes long at invalid byte offset %s", - blockMeta.getBlockId(), blockMeta.getBlockSize(), offset); - mUfsManager = ufsManager; - mUfsInStreamCache = ufsInStreamCache; - mBlockMeta = blockMeta; - mUfsBlockOptions = ufsBlockReadOptions; - mPageSize = pageSize; - mInitialOffset = offset; - mLastPage = ByteBuffer.allocateDirect((int) mPageSize); - mPosition = offset; - } - - @Override - public ByteBuffer read(long offset, long length) throws IOException { - Preconditions.checkState(!mClosed); - Preconditions.checkArgument(length >= 0, "length should be non-negative"); - Preconditions.checkArgument(offset >= 0, "offset should be non-negative"); - - if (length == 0 || offset >= mBlockMeta.getBlockSize()) { - return EMPTY_BYTE_BUFFER; - } - - length = Math.min(length, mBlockMeta.getBlockSize() - offset); - // todo(bowen): this pooled buffer will likely not get released, so will still be GCed instead - // of reused. - ByteBuffer buffer = NioDirectBufferPool.acquire((int) length); - int totalBytesRead = fillWithCachedPage(buffer, offset, length); - offset += totalBytesRead; - try (ReadableByteChannel channel = getChannel(offset)) { - while (totalBytesRead < length) { - int bytesRead = channel.read(buffer); - if (bytesRead < 0) { - throw new IOException(String.format( - "Unexpected EOF when reading %d bytes from offset %d of block %d", - length, offset, mBlockMeta.getBlockId())); - } - totalBytesRead += bytesRead; - } - } - buffer.flip(); - return buffer; - } - - /** - * Reads a page from the UFS block at index {@code pageIndex}. This method will try to read as - * many bytes as the page size designated by {@link PropertyKey#USER_CLIENT_CACHE_PAGE_SIZE}, - * and append to {@code buffer}. If {@code pageIndex} points to the last page of the block, - * the size of the data read can be smaller than the page size. - * - * @param buffer writable output buffer, must have enough remaining space for a page - * @param pageIndex the index of the page within the block - * @return number of bytes read, or -1 if end of block is reached - */ - public int readPageAtIndex(ByteBuffer buffer, long pageIndex) throws IOException { - Preconditions.checkState(!mClosed); - Preconditions.checkArgument(!buffer.isReadOnly(), "read-only buffer"); - Preconditions.checkArgument(buffer.remaining() >= mPageSize, - "%s bytes available in buffer, not enough for a page of size %s", - buffer.remaining(), mPageSize); - Preconditions.checkArgument(pageIndex >= 0 && pageIndex * mPageSize < mBlockMeta.getBlockSize(), - "page index (%s) is out of bound", pageIndex); - - if (pageIndex == mLastPageIndex) { - return fillWithCachedPage(buffer, pageIndex * mPageSize, mLastPage.remaining()); - } - int totalBytesRead = 0; - mLastPage.clear(); - mLastPageIndex = -1; - try (ReadableByteChannel channel = getChannel(pageIndex * mPageSize)) { - while (totalBytesRead < mPageSize) { - int bytesRead = channel.read(mLastPage); - if (bytesRead < 0) { - // reached eof - if (totalBytesRead == 0) { - // not a single byte has been read; report this to caller - return bytesRead; - } - break; - } - totalBytesRead += bytesRead; - } - } - mLastPage.flip(); - mLastPageIndex = pageIndex; - fillWithCachedPage(buffer, pageIndex * mPageSize, totalBytesRead); - return totalBytesRead; - } - - /** - * Fills the output buffer with the content from the cached paged. - * @param outBuffer output buffer - * @param offset offset with the block - * @param length how many bytes to read - * @return how many bytes was filled in the output buffer, 0 when the cached page does not - * content of the requested range - */ - private int fillWithCachedPage(ByteBuffer outBuffer, long offset, long length) { - long pageIndex = offset / mPageSize; - if (pageIndex != mLastPageIndex) { - return 0; - } - int pageSize = Math.min(mLastPage.remaining(), (int) length); - ByteBuffer slice = outBuffer.slice(); - slice.limit(pageSize); - slice.put(mLastPage); - mLastPage.rewind(); - outBuffer.position(outBuffer.position() + pageSize); - return pageSize; - } - - @Override - public long getLength() { - return mBlockMeta.getBlockSize(); - } - - @Override - public ReadableByteChannel getChannel() { - return getChannel(mInitialOffset); - } - - /** - * @param offset offset within the block - * @return readable channel - */ - public ReadableByteChannel getChannel(long offset) { - Preconditions.checkState(!mClosed); - return new UfsReadableChannel(offset); - } - - /** - * @return ufs block read options which this read was created with - */ - public UfsBlockReadOptions getUfsReadOptions() { - return mUfsBlockOptions; - } - - @Override - public int transferTo(ByteBuf buf) throws IOException { - Preconditions.checkState(!mClosed); - // todo(bowen): eliminate copy - ByteBuffer buffer = NioDirectBufferPool.acquire(buf.writableBytes()); - int bytesRead = transferTo(buffer); - buffer.flip(); - buf.writeBytes(buffer); - NioDirectBufferPool.release(buffer); - return bytesRead; - } - - int transferTo(ByteBuffer byteBuffer) throws IOException { - Preconditions.checkState(!mClosed); - int bytesRead = fillWithCachedPage(byteBuffer, mPosition, byteBuffer.remaining()); - mPosition += bytesRead; - try (ReadableByteChannel channel = getChannel(mPosition)) { - bytesRead = channel.read(byteBuffer); - if (bytesRead < 0) { // eof - return bytesRead; - } - mPosition += bytesRead; - return bytesRead; - } - } - - @Override - public boolean isClosed() { - return mClosed; - } - - @Override - public String getLocation() { - return mUfsBlockOptions.getUfsPath(); - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - mClosed = true; - } - - private class UfsReadableChannel implements ReadableByteChannel { - private final long mOffset; - private volatile InputStream mUfsInStream; - private volatile ReadableByteChannel mUfsChannel; - private volatile boolean mClosed = false; - - UfsReadableChannel(long offset) { - mOffset = offset; - } - - @Override - public int read(ByteBuffer dst) throws IOException { - if (mClosed) { - throw new ClosedChannelException(); - } - if (mUfsInStream == null) { - synchronized (this) { - if (mUfsInStream == null) { - UfsManager.UfsClient ufsClient = mUfsManager.get(mUfsBlockOptions.getMountId()); - try (CloseableResource ufsResource = ufsClient.acquireUfsResource()) { - mUfsInStream = mUfsInStreamCache.acquire( - ufsResource.get(), - mUfsBlockOptions.getUfsPath(), - IdUtils.fileIdFromBlockId(mBlockMeta.getBlockId()), - OpenOptions.defaults() - .setOffset(mUfsBlockOptions.getOffsetInFile() + mOffset) - .setPositionShort(true)); - mUfsChannel = Channels.newChannel(mUfsInStream); - } - } - } - } - return mUfsChannel.read(dst); - } - - @Override - public boolean isOpen() { - return !mClosed; - } - - @Override - public void close() throws IOException { - if (mClosed) { - return; - } - synchronized (this) { - if (mClosed) { - return; - } - if (mUfsInStream != null) { - // todo(bowen): cannot release the stream if the channel is being concurrently read from. - // needs to interrupt the reader before releasing - // do not close mChannel as it will close the underlying stream transitively - mUfsInStreamCache.release(mUfsInStream); - mUfsInStream = null; - mUfsChannel = null; - } - mClosed = true; - } - } - } -} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/UfsBlockReadOptions.java b/core/server/worker/src/main/java/alluxio/worker/page/UfsBlockReadOptions.java deleted file mode 100644 index 4156c63334cb..000000000000 --- a/core/server/worker/src/main/java/alluxio/worker/page/UfsBlockReadOptions.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.proto.dataserver.Protocol; - -import com.google.common.base.Preconditions; - -import java.util.Objects; - -/** - * Options for reading a block from UFS. - */ -public final class UfsBlockReadOptions { - - private final long mMountId; - private final long mOffsetInFile; - private final String mUfsPath; - private final boolean mCacheIntoAlluxio; - - UfsBlockReadOptions(long mountId, long offsetInFile, String ufsPath, boolean cacheIntoAlluxio) { - mMountId = mountId; - mOffsetInFile = offsetInFile; - mUfsPath = ufsPath; - mCacheIntoAlluxio = cacheIntoAlluxio; - } - - /** - * Creates from wire object. - * - * @param options wire options - * @return the options - * @throws IllegalArgumentException if critical information is missing in the options object - */ - public static UfsBlockReadOptions fromProto(Protocol.OpenUfsBlockOptions options) { - Preconditions.checkArgument(options.hasMountId(), "missing mount ID for UFS block read"); - Preconditions.checkArgument(options.hasOffsetInFile(), - "missing offset in file for UFS block read"); - Preconditions.checkArgument(options.hasUfsPath(), "missing UFS path for UFS block read"); - return new UfsBlockReadOptions(options.getMountId(), - options.getOffsetInFile(), options.getUfsPath(), !options.getNoCache()); - } - - /** - * @return mount ID - */ - public long getMountId() { - return mMountId; - } - - /** - * @return offset in file - */ - public long getOffsetInFile() { - return mOffsetInFile; - } - - /** - * @return ufs path - */ - public String getUfsPath() { - return mUfsPath; - } - - /** - * @return whether the UFS block should be cached into Alluxio - */ - public boolean isCacheIntoAlluxio() { - return mCacheIntoAlluxio; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - UfsBlockReadOptions that = (UfsBlockReadOptions) o; - return mMountId == that.mMountId && mOffsetInFile == that.mOffsetInFile - && mCacheIntoAlluxio == that.mCacheIntoAlluxio - && Objects.equals(mUfsPath, that.mUfsPath); - } - - @Override - public int hashCode() { - return Objects.hash(mMountId, mOffsetInFile, mUfsPath, mCacheIntoAlluxio); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/AsyncBlockRemoverTest.java b/core/server/worker/src/test/java/alluxio/worker/block/AsyncBlockRemoverTest.java deleted file mode 100644 index 57a984e31600..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/AsyncBlockRemoverTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static junit.framework.TestCase.assertEquals; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.doAnswer; - -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; - -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; - -/** - * Tests for AsyncBlockRemover. - */ -public class AsyncBlockRemoverTest { - private AsyncBlockRemover mRemover; - private final BlockingQueue mBlocksToRemove = new LinkedBlockingQueue<>(); - private final Set mBlocksTriedToRemoved = - Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final BlockWorker mMockWorker = Mockito.mock(BlockWorker.class); - - @Test - public void blockRemove() throws Exception { - doAnswer(args -> { - // keeps track the blocks to remove - mBlocksTriedToRemoved.add(args.getArgument(1)); - return null; - }).when(mMockWorker).removeBlock(anyLong(), anyLong()); - mRemover = new AsyncBlockRemover(mMockWorker, 10, mBlocksToRemove, - Collections.newSetFromMap(new ConcurrentHashMap<>())); - List blocks = new ArrayList<>(); - for (long i = 0; i < 100; i++) { - blocks.add(i); - } - mRemover.addBlocksToDelete(blocks); - CommonUtils.waitFor("async block removal completed", - () -> mBlocksTriedToRemoved.size() == blocks.size(), - WaitForOptions.defaults().setTimeoutMs(10000)); - assertEquals(0, mBlocksToRemove.size()); - } - - @Test - public void failedBlockRemove() throws Exception { - doAnswer(args -> { - // keeps track the blocks to remove - mBlocksTriedToRemoved.add(args.getArgument(1)); - // throw exception - throw new IOException("Failed to remove block"); - }).when(mMockWorker).removeBlock(anyLong(), anyLong()); - mRemover = new AsyncBlockRemover(mMockWorker, 10, mBlocksToRemove, - Collections.newSetFromMap(new ConcurrentHashMap<>())); - List blocks = new ArrayList<>(); - for (long i = 0; i < 100; i++) { - blocks.add(i); - } - mRemover.addBlocksToDelete(blocks); - CommonUtils.waitFor("async block removal completed", - () -> mBlocksTriedToRemoved.size() == blocks.size(), - WaitForOptions.defaults().setTimeoutMs(10000)); - assertEquals(0, mBlocksToRemove.size()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/BlockHeartbeatReporterTest.java b/core/server/worker/src/test/java/alluxio/worker/block/BlockHeartbeatReporterTest.java deleted file mode 100644 index a04a7e4042a3..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/BlockHeartbeatReporterTest.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; - -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Map; - -/** - * Unit tests for {@link BlockHeartbeatReporter}. - */ -public final class BlockHeartbeatReporterTest { - private static final BlockStoreLocation MEM_LOC = - new BlockStoreLocation(Constants.MEDIUM_MEM, 0, Constants.MEDIUM_MEM); - private static final BlockStoreLocation SSD_LOC = - new BlockStoreLocation(Constants.MEDIUM_SSD, 0, Constants.MEDIUM_SSD); - private static final BlockStoreLocation HDD_LOC = - new BlockStoreLocation(Constants.MEDIUM_HDD, 0, Constants.MEDIUM_HDD); - BlockHeartbeatReporter mReporter; - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public final void before() { - mReporter = new BlockHeartbeatReporter(); - } - - private void moveBlock(long blockId, BlockStoreLocation newLocation) { - BlockStoreLocation unusedOldLocation = - new BlockStoreLocation(Constants.MEDIUM_MEM, 0, Constants.MEDIUM_MEM); - mReporter.onMoveBlockByWorker(blockId, unusedOldLocation, newLocation); - } - - private void removeBlock(long blockId) { - mReporter.onRemoveBlockByWorker(blockId); - } - - /** - * Tests the {@link BlockHeartbeatReporter#generateReport()} method for an empty report. - */ - @Test - public void generateReportEmpty() { - BlockHeartbeatReport report = mReporter.generateReport(); - assertTrue(report.getAddedBlocks().isEmpty()); - assertTrue(report.getRemovedBlocks().isEmpty()); - } - - /** - * Tests the {@link BlockHeartbeatReporter#generateReport()} method to correctly generate a report - * after moving block. - */ - @Test - public void generateReportMove() { - Long block1 = 1L; - Long block2 = 2L; - Long block3 = 3L; - moveBlock(block1, MEM_LOC); - moveBlock(block2, SSD_LOC); - moveBlock(block3, HDD_LOC); - BlockHeartbeatReport report = mReporter.generateReport(); - Map> addedBlocks = report.getAddedBlocks(); - - // Block1 moved to memory - List addedBlocksMem = addedBlocks.get(MEM_LOC); - assertEquals(1, addedBlocksMem.size()); - assertEquals(block1, addedBlocksMem.get(0)); - - // Block2 moved to ssd - List addedBlocksSsd = addedBlocks.get(SSD_LOC); - assertEquals(1, addedBlocksSsd.size()); - assertEquals(block2, addedBlocksSsd.get(0)); - - // Block3 moved to hdd - List addedBlocksHdd = addedBlocks.get(HDD_LOC); - assertEquals(1, addedBlocksHdd.size()); - assertEquals(block3, addedBlocksHdd.get(0)); - } - - /** - * Tests the {@link BlockHeartbeatReporter#generateReport()} method that generating a report - * clears the state of the reporter. - */ - @Test - public void generateReportStateClear() { - Long block1 = 1L; - moveBlock(block1, MEM_LOC); - - // First report should have updates - BlockHeartbeatReport report = mReporter.generateReport(); - assertFalse(report.getAddedBlocks().isEmpty()); - - // Second report should not have updates - BlockHeartbeatReport nextReport = mReporter.generateReport(); - assertTrue(nextReport.getAddedBlocks().isEmpty()); - assertTrue(nextReport.getRemovedBlocks().isEmpty()); - } - - /** - * Tests the {@link BlockHeartbeatReporter#generateReport()} method to correctly generate a report - * after removing blocks. - */ - @Test - public void generateReportRemove() { - Long block1 = 1L; - Long block2 = 2L; - Long block3 = 3L; - removeBlock(block1); - removeBlock(block2); - removeBlock(block3); - BlockHeartbeatReport report = mReporter.generateReport(); - - // All blocks should be removed - List removedBlocks = report.getRemovedBlocks(); - assertEquals(3, removedBlocks.size()); - assertTrue(removedBlocks.contains(block1)); - assertTrue(removedBlocks.contains(block2)); - assertTrue(removedBlocks.contains(block3)); - - // No blocks should have been added - Map> addedBlocks = report.getAddedBlocks(); - assertTrue(addedBlocks.isEmpty()); - } - - /** - * Tests the {@link BlockHeartbeatReporter#generateReport()} method to correctly generate a report - * after moving a block and the removing it. - */ - @Test - public void generateReportMoveThenRemove() { - Long block1 = 1L; - moveBlock(block1, MEM_LOC); - removeBlock(block1); - - // The block should not be in the added blocks list - BlockHeartbeatReport report = mReporter.generateReport(); - assertEquals(null, report.getAddedBlocks().get(MEM_LOC)); - - // The block should be in the removed blocks list - List removedBlocks = report.getRemovedBlocks(); - assertEquals(1, removedBlocks.size()); - assertTrue(removedBlocks.contains(block1)); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/BlockLockManagerTest.java b/core/server/worker/src/test/java/alluxio/worker/block/BlockLockManagerTest.java deleted file mode 100644 index a7703bfe7d2a..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/BlockLockManagerTest.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -import alluxio.collections.ConcurrentHashSet; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; - -import com.google.common.base.Throwables; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.TimeUnit; - -/** - * Unit tests for {@link BlockLockManager}. - */ -public final class BlockLockManagerTest { - private static final long TEST_SESSION_ID = 2; - private static final long TEST_SESSION_ID2 = 3; - private static final long TEST_BLOCK_ID = 9; - - private BlockLockManager mLockManager; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - /** The exception expected to be thrown. */ - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() { - mLockManager = new BlockLockManager(); - } - - @After - public void after() { - Configuration.reloadProperties(); - } - - /** - * Tests the {@link BlockLockManager#acquireBlockLock(long, long, BlockLockType)} method. - */ - @Test - public void lockBlock() { - // Read-lock on can both get through - BlockLock lockId1 = - mLockManager.acquireBlockLock(TEST_SESSION_ID, TEST_BLOCK_ID, BlockLockType.READ); - BlockLock lockId2 = - mLockManager.acquireBlockLock(TEST_SESSION_ID, TEST_BLOCK_ID, BlockLockType.READ); - assertNotEquals(lockId1.get(), lockId2.get()); - } - - @Test - public void tryLockBlock() { - // Read-lock on can both get through - Optional lock1 = mLockManager.tryAcquireBlockLock(TEST_SESSION_ID, TEST_BLOCK_ID, - BlockLockType.READ, 1, TimeUnit.MINUTES); - Optional lock2 = mLockManager.tryAcquireBlockLock(TEST_SESSION_ID, TEST_BLOCK_ID, - BlockLockType.READ, 1, TimeUnit.MINUTES); - assertNotEquals(lock1.get().get(), lock2.get().get()); - } - - @Test - public void tryLockBlockWithReadLockLocked() { - // Hold a read-lock on test block - try (BlockLock lock = mLockManager.acquireBlockLock(TEST_SESSION_ID, TEST_BLOCK_ID, - BlockLockType.READ)) { - // Read-lock on the same block should get through - Optional lock2 = mLockManager.tryAcquireBlockLock(TEST_SESSION_ID2, TEST_BLOCK_ID, - BlockLockType.READ, 1, TimeUnit.SECONDS); - assertTrue(lock2.isPresent()); - lock2.get().close(); - // Write-lock should fail - Optional lock3 = mLockManager.tryAcquireBlockLock(TEST_SESSION_ID2, TEST_BLOCK_ID, - BlockLockType.WRITE, 1, TimeUnit.SECONDS); - assertEquals(Optional.empty(), lock3); - } - } - - @Test - public void tryLockBlockWithWriteLockLocked() { - // Hold a write-lock on test block - try (BlockLock lock = mLockManager.acquireBlockLock(TEST_SESSION_ID, TEST_BLOCK_ID, - BlockLockType.WRITE)) { - // Read-lock should fail - Optional lockId2 = - mLockManager.tryAcquireBlockLock(TEST_SESSION_ID2, TEST_BLOCK_ID, - BlockLockType.READ, 1, TimeUnit.SECONDS); - assertFalse(lockId2.isPresent()); - // Write-lock should fail - Optional lockId3 = - mLockManager.tryAcquireBlockLock(TEST_SESSION_ID2, TEST_BLOCK_ID, - BlockLockType.WRITE, 1, TimeUnit.SECONDS); - assertFalse(lockId3.isPresent()); - } - } - - /** - * Test when trying to validate a lock of a block via - * {@link BlockLockManager#checkLock(long, long, long)} which is not locked. - */ - @Test - public void validateLockIdWithNoRecord() { - long badLockId = 1; - assertFalse(mLockManager.checkLock(TEST_SESSION_ID, TEST_BLOCK_ID, badLockId)); - } - - /** - * Test when trying to validate a lock of a block via - * {@link BlockLockManager#checkLock(long, long, long)} with an incorrect session ID. - */ - @Test - public void validateLockIdWithWrongSessionId() { - try (BlockLock lock = mLockManager.acquireBlockLock(TEST_SESSION_ID, TEST_BLOCK_ID, - BlockLockType.READ)) { - long wrongSessionId = TEST_SESSION_ID + 1; - // Validate an existing lockId with wrong session id, expect to see IOException - assertFalse(mLockManager.checkLock(wrongSessionId, TEST_BLOCK_ID, lock.get())); - } - } - - /** - * Test when trying to validate a lock of a block via - * {@link BlockLockManager#checkLock(long, long, long)} with an incorrect block ID. - */ - @Test - public void validateLockIdWithWrongBlockId() { - try (BlockLock lock = mLockManager.acquireBlockLock(TEST_SESSION_ID, TEST_BLOCK_ID, - BlockLockType.READ)) { - long wrongBlockId = TEST_BLOCK_ID + 1; - assertFalse(mLockManager.checkLock(TEST_SESSION_ID, wrongBlockId, lock.get())); - } - } - - /** - * Tests when trying to validate a lock of a block via - * {@link BlockLockManager#checkLock(long, long, long)} after the session was cleaned up. - */ - @Test - public void cleanupSession() { - long sessionId1 = TEST_SESSION_ID; - long sessionId2 = TEST_SESSION_ID + 1; - BlockLock lock1 = mLockManager.acquireBlockLock(sessionId1, TEST_BLOCK_ID, BlockLockType.READ); - BlockLock lock2 = mLockManager.acquireBlockLock(sessionId2, TEST_BLOCK_ID, BlockLockType.READ); - mLockManager.cleanupSession(sessionId2); - // Expect validating sessionId1 to get through - assertTrue(mLockManager.checkLock(sessionId1, TEST_BLOCK_ID, lock1.get())); - // Because sessionId2 has been cleaned up, expect validating sessionId2 to return false - assertFalse(mLockManager.checkLock(sessionId2, TEST_BLOCK_ID, lock2.get())); - } - - /** - * Tests that up to WORKER_TIERED_STORE_BLOCK_LOCKS block locks can be grabbed simultaneously. - */ - @Test(timeout = 10000) - public void grabManyLocks() throws Exception { - int maxLocks = 100; - setMaxLocks(maxLocks); - BlockLockManager manager = new BlockLockManager(); - for (int i = 0; i < maxLocks; i++) { - manager.acquireBlockLock(i, i, BlockLockType.WRITE); - } - lockExpectingHang(manager, 101); - } - - /** - * Tests that an exception is thrown when a session tries to acquire a write lock on a block that - * it currently has a read lock on. - */ - @Test - public void lockAlreadyReadLockedBlock() { - BlockLockManager manager = new BlockLockManager(); - try (BlockLock readLock = manager.acquireBlockLock(1, 1, BlockLockType.READ)) { - mThrown.expect(IllegalStateException.class); - manager.acquireBlockLock(1, 1, BlockLockType.WRITE); - } - } - - /** - * Tests that an exception is thrown when a session tries to acquire a write lock on a block that - * it currently has a write lock on. - */ - @Test - public void lockAlreadyWriteLockedBlock() { - BlockLockManager manager = new BlockLockManager(); - try (BlockLock readLock = manager.acquireBlockLock(1, 1, BlockLockType.WRITE)) { - mThrown.expect(IllegalStateException.class); - manager.acquireBlockLock(1, 1, BlockLockType.WRITE); - } - } - - /** - * Tests that two sessions can both take a read lock on the same block. - */ - @Test(timeout = 10000) - public void lockAcrossSessions() { - BlockLockManager manager = new BlockLockManager(); - try (BlockLock lock1 = manager.acquireBlockLock(1, TEST_BLOCK_ID, BlockLockType.READ); - BlockLock lock2 = manager.acquireBlockLock(2, TEST_BLOCK_ID, BlockLockType.READ)) { - //do nothing - } - } - - /** - * Tests that a write lock can't be taken while a read lock is held. - */ - @Test(timeout = 10000) - public void readBlocksWrite() throws Exception { - BlockLockManager manager = new BlockLockManager(); - try (BlockLock lock = manager.acquireBlockLock(1, TEST_BLOCK_ID, BlockLockType.READ)) { - lockExpectingHang(manager, TEST_BLOCK_ID); - } - } - - /** - * Tests that block locks are returned to the pool when they are no longer in use. - */ - @Test(timeout = 10000) - public void reuseLock() { - setMaxLocks(1); - BlockLockManager manager = new BlockLockManager(); - BlockLock lock1 = manager.acquireBlockLock(TEST_SESSION_ID, 1, BlockLockType.WRITE); - lock1.close(); // Without this line the next lock would hang. - manager.acquireBlockLock(TEST_SESSION_ID, 2, BlockLockType.WRITE); - } - - /** - * Tests that block locks are not returned to the pool when they are still in use. - */ - @Test(timeout = 10000) - public void dontReuseLock() throws Exception { - setMaxLocks(1); - final BlockLockManager manager = new BlockLockManager(); - try (BlockLock lock = manager.acquireBlockLock(TEST_SESSION_ID, 1, BlockLockType.READ)) { - manager.acquireBlockLock(TEST_SESSION_ID, 1, BlockLockType.READ); - } - lockExpectingHang(manager, 2); - } - - /** - * Calls {@link BlockLockManager#acquireBlockLock(long, long, BlockLockType)} - * and fails if it doesn't hang. - * - * @param manager the manager to call lock on - * @param blockId block id to try locking - */ - private void lockExpectingHang(final BlockLockManager manager, final long blockId) - throws Exception { - Thread thread = new Thread( - () -> manager.acquireBlockLock(TEST_SESSION_ID, blockId, BlockLockType.WRITE)); - thread.start(); - thread.join(200); - // Locking should not take 200ms unless there is a hang. - assertTrue(thread.isAlive()); - } - - /** - * Tests that taking and releasing many block locks concurrently won't cause a failure. - * - * This is done by creating 200 threads, 100 for each of 2 different block ids. Each thread locks - * and then unlocks its block 50 times. After this, it takes a final lock on its block before - * returning. At the end of the test, the internal state of the lock manager is validated. - */ - @Test(timeout = 10000) - public void stress() throws Throwable { - final int numBlocks = 2; - final int threadsPerBlock = 100; - final int lockUnlocksPerThread = 50; - setMaxLocks(numBlocks); - final BlockLockManager manager = new BlockLockManager(); - final List threads = new ArrayList<>(); - final CyclicBarrier barrier = new CyclicBarrier(numBlocks * threadsPerBlock); - // If there are exceptions, we will store them here. - final ConcurrentHashSet failedThreadThrowables = new ConcurrentHashSet<>(); - Thread.UncaughtExceptionHandler exceptionHandler = (th, ex) -> failedThreadThrowables.add(ex); - for (int blockId = 0; blockId < numBlocks; blockId++) { - final int finalBlockId = blockId; - for (int i = 0; i < threadsPerBlock; i++) { - Thread t = new Thread(() -> { - try { - barrier.await(); - } catch (Exception e) { - throw new RuntimeException(e); - } - // Lock and unlock the block lockUnlocksPerThread times. - for (int j = 0; j < lockUnlocksPerThread; j++) { - BlockLock lock = - manager.acquireBlockLock(TEST_SESSION_ID, finalBlockId, BlockLockType.READ); - lock.close(); - } - // Lock the block one last time. - manager.acquireBlockLock(TEST_SESSION_ID, finalBlockId, BlockLockType.READ); - }); - t.setUncaughtExceptionHandler(exceptionHandler); - threads.add(t); - } - } - Collections.shuffle(threads); - for (Thread t : threads) { - t.start(); - } - for (Thread t : threads) { - t.join(); - } - if (!failedThreadThrowables.isEmpty()) { - StringBuilder sb = new StringBuilder("Failed with the following errors:\n"); - for (Throwable failedThreadThrowable : failedThreadThrowables) { - sb.append(Throwables.getStackTraceAsString(failedThreadThrowable)); - } - Assert.fail(sb.toString()); - } - manager.validate(); - } - - private void setMaxLocks(int maxLocks) { - Configuration.set(PropertyKey.WORKER_TIERED_STORE_BLOCK_LOCKS, - maxLocks); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/BlockMetadataManagerTest.java b/core/server/worker/src/test/java/alluxio/worker/block/BlockMetadataManagerTest.java deleted file mode 100644 index a625958786b2..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/BlockMetadataManagerTest.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.DefaultBlockMeta; -import alluxio.worker.block.meta.DefaultTempBlockMeta; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageTier; -import alluxio.worker.block.meta.TempBlockMeta; - -import com.google.common.collect.ImmutableMap; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.util.List; -import java.util.Map; - -/** - * Unit tests for {@link BlockMetadataManager}. - */ -public final class BlockMetadataManagerTest { - private static final long TEST_SESSION_ID = 2; - private static final long TEST_BLOCK_ID = 9; - private static final long TEST_TEMP_BLOCK_ID = 10; - private static final long TEST_TEMP_BLOCK_ID2 = TEST_TEMP_BLOCK_ID + 1; - private static final long TEST_BLOCK_SIZE = 20; - - private static final int[] TIER_ORDINAL = {0, 1}; - private static final String[] TIER_ALIAS = {Constants.MEDIUM_MEM, Constants.MEDIUM_HDD}; - private static final String[][] TIER_PATH = {{"/ramdisk"}, {"/disk1", "/disk2"}}; - public static final String[][] TIER_MEDIA_TYPE = - {{Constants.MEDIUM_MEM}, {Constants.MEDIUM_HDD, Constants.MEDIUM_HDD}}; - private static final long[][] TIER_CAPACITY_BYTES = {{1000}, {3000, 5000}}; - - private BlockMetadataManager mMetaManager; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - /** The exception expected to be thrown. */ - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - String baseDir = mFolder.newFolder().getAbsolutePath(); - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED, false); - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED, false); - TieredBlockStoreTestUtils.setupConfWithMultiTier(baseDir, TIER_ORDINAL, TIER_ALIAS, - TIER_PATH, TIER_CAPACITY_BYTES, TIER_MEDIA_TYPE, null); - - mMetaManager = BlockMetadataManager.createBlockMetadataManager(); - } - - /** - * Tests the {@link BlockMetadataManager#getTier(String)} method. - */ - @Test - public void getTier() { - StorageTier tier; - tier = mMetaManager.getTier(Constants.MEDIUM_MEM); // MEM - assertEquals(Constants.MEDIUM_MEM, tier.getTierAlias()); - assertEquals(0, tier.getTierOrdinal()); - tier = mMetaManager.getTier(Constants.MEDIUM_HDD); // HDD - assertEquals(Constants.MEDIUM_HDD, tier.getTierAlias()); - assertEquals(1, tier.getTierOrdinal()); - } - - /** - * Tests the {@link BlockMetadataManager#getDir(BlockStoreLocation)} method. - */ - @Test - public void getDir() { - BlockStoreLocation loc; - StorageDir dir; - - loc = new BlockStoreLocation(Constants.MEDIUM_MEM, 0); - dir = mMetaManager.getDir(loc); - assertEquals(loc.tierAlias(), dir.getParentTier().getTierAlias()); - assertEquals(loc.dir(), dir.getDirIndex()); - - loc = new BlockStoreLocation(Constants.MEDIUM_HDD, 1); - dir = mMetaManager.getDir(loc); - assertEquals(loc.tierAlias(), dir.getParentTier().getTierAlias()); - assertEquals(loc.dir(), dir.getDirIndex()); - } - - /** - * Tests that an exception is thrown in the {@link BlockMetadataManager#getTier(String)} method - * when trying to retrieve a tier which does not exist. - */ - @Test - public void getTierNotExisting() { - String badTierAlias = Constants.MEDIUM_SSD; - mThrown.expect(IllegalArgumentException.class); - mThrown.expectMessage(ExceptionMessage.TIER_ALIAS_NOT_FOUND.getMessage(badTierAlias)); - mMetaManager.getTier(badTierAlias); - } - - /** - * Tests the {@link BlockMetadataManager#getTiers()} method. - */ - @Test - public void getTiers() { - List tiers = mMetaManager.getTiers(); - assertEquals(2, tiers.size()); - assertEquals(Constants.MEDIUM_MEM, tiers.get(0).getTierAlias()); - assertEquals(0, tiers.get(0).getTierOrdinal()); - assertEquals(Constants.MEDIUM_HDD, tiers.get(1).getTierAlias()); - assertEquals(1, tiers.get(1).getTierOrdinal()); - } - - /** - * Tests the {@link BlockMetadataManager#getTiersBelow(String)} method. - */ - @Test - public void getTiersBelow() { - List tiersBelow = mMetaManager.getTiersBelow(Constants.MEDIUM_MEM); - assertEquals(1, tiersBelow.size()); - assertEquals(Constants.MEDIUM_HDD, tiersBelow.get(0).getTierAlias()); - assertEquals(1, tiersBelow.get(0).getTierOrdinal()); - - tiersBelow = mMetaManager.getTiersBelow(Constants.MEDIUM_HDD); - assertEquals(0, tiersBelow.size()); - } - - /** - * Tests the {@link BlockMetadataManager#getAvailableBytes(BlockStoreLocation)} method. - */ - @Test - public void getAvailableBytes() { - assertEquals(9000, mMetaManager.getAvailableBytes(BlockStoreLocation.anyTier())); - assertEquals(1000, - mMetaManager.getAvailableBytes(BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM))); - assertEquals(8000, - mMetaManager.getAvailableBytes(BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD))); - assertEquals(1000, - mMetaManager.getAvailableBytes(new BlockStoreLocation(Constants.MEDIUM_MEM, 0))); - assertEquals(3000, - mMetaManager.getAvailableBytes(new BlockStoreLocation(Constants.MEDIUM_HDD, 0))); - assertEquals(5000, - mMetaManager.getAvailableBytes(new BlockStoreLocation(Constants.MEDIUM_HDD, 1))); - } - - /** - * Tests the different operations for metadata of a block, such as adding a temporary block or - * committing a block. - */ - @Test - public void blockMeta() throws Exception { - StorageDir dir = mMetaManager.getTier(Constants.MEDIUM_HDD).getDir(0); - TempBlockMeta tempBlockMeta = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_BLOCK_SIZE, dir); - - // Empty storage - assertFalse(mMetaManager.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertFalse(mMetaManager.hasBlockMeta(TEST_TEMP_BLOCK_ID)); - // Add temp block - mMetaManager.addTempBlockMeta(tempBlockMeta); - assertTrue(mMetaManager.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertFalse(mMetaManager.hasBlockMeta(TEST_TEMP_BLOCK_ID)); - // Get temp block - assertEquals(tempBlockMeta, mMetaManager.getTempBlockMeta(TEST_TEMP_BLOCK_ID).get()); - // Abort temp block - mMetaManager.abortTempBlockMeta(tempBlockMeta); - assertFalse(mMetaManager.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertFalse(mMetaManager.hasBlockMeta(TEST_TEMP_BLOCK_ID)); - // Add temp block with previous block id - mMetaManager.addTempBlockMeta(tempBlockMeta); - assertTrue(mMetaManager.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertFalse(mMetaManager.hasBlockMeta(TEST_TEMP_BLOCK_ID)); - // Commit temp block - mMetaManager.commitTempBlockMeta(tempBlockMeta); - assertFalse(mMetaManager.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertTrue(mMetaManager.hasBlockMeta(TEST_TEMP_BLOCK_ID)); - // Get block - BlockMeta blockMeta = mMetaManager.getBlockMeta(TEST_TEMP_BLOCK_ID).get(); - assertEquals(TEST_TEMP_BLOCK_ID, blockMeta.getBlockId()); - // Remove block - mMetaManager.removeBlockMeta(blockMeta); - assertFalse(mMetaManager.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertFalse(mMetaManager.hasBlockMeta(TEST_TEMP_BLOCK_ID)); - } - - /** - * Tests that the {@link BlockMetadataManager#getBlockMeta(long)} method returns - * Optional.empty() when trying to retrieve metadata of a block which does not exist. - */ - @Test - public void getBlockMetaNotExisting() { - assertFalse(mMetaManager.getBlockMeta(TEST_BLOCK_ID).isPresent()); - } - - /** - * Tests that an exception is thrown in the {@link BlockMetadataManager#getTempBlockMeta(long)} - * method when trying to retrieve metadata of a temporary block which does not exist. - */ - @Test - public void getTempBlockMetaNotExisting() { - assertFalse(mMetaManager.getTempBlockMeta(TEST_TEMP_BLOCK_ID).isPresent()); - } - - /** - * Dummy unit test, actually the case of move block meta to same dir should never happen. - */ - @Test - public void moveBlockMetaSameDir() throws Exception { - // create and add two temp block metas with same tier and dir to the meta manager - StorageDir dir = mMetaManager.getTier(Constants.MEDIUM_MEM).getDir(0); - TempBlockMeta tempBlockMeta1 = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_BLOCK_SIZE, dir); - TempBlockMeta tempBlockMeta2 = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID2, TEST_BLOCK_SIZE, dir); - mMetaManager.addTempBlockMeta(tempBlockMeta1); - mMetaManager.addTempBlockMeta(tempBlockMeta2); - - // commit the first temp block meta - mMetaManager.commitTempBlockMeta(tempBlockMeta1); - BlockMeta blockMeta = mMetaManager.getBlockMeta(TEST_TEMP_BLOCK_ID).get(); - - mMetaManager.moveBlockMeta(blockMeta, tempBlockMeta2); - - // test to make sure that the dst tempBlockMeta has been removed from the dir - assertFalse(mMetaManager.getTempBlockMeta(TEST_TEMP_BLOCK_ID2).isPresent()); - } - - /** - * Tests that an exception is thrown in the - * {@link BlockMetadataManager#moveBlockMeta(BlockMeta, TempBlockMeta)} method when trying to move - * a block to a not committed block meta. - */ - @Test - public void moveBlockMetaDiffDir() throws Exception { - // create and add two temp block metas with different dirs in the same HDD tier - StorageDir dir1 = mMetaManager.getTier(Constants.MEDIUM_HDD).getDir(0); - StorageDir dir2 = mMetaManager.getTier(Constants.MEDIUM_HDD).getDir(1); - TempBlockMeta tempBlockMeta1 = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_BLOCK_SIZE, dir1); - TempBlockMeta tempBlockMeta2 = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID2, TEST_BLOCK_SIZE, dir2); - mMetaManager.addTempBlockMeta(tempBlockMeta1); - mMetaManager.addTempBlockMeta(tempBlockMeta2); - - // commit the first temp block meta - mMetaManager.commitTempBlockMeta(tempBlockMeta1); - BlockMeta blockMeta = mMetaManager.getBlockMeta(TEST_TEMP_BLOCK_ID).get(); - - mMetaManager.moveBlockMeta(blockMeta, tempBlockMeta2); - - // make sure that the dst tempBlockMeta has been removed from the dir2 - assertFalse(mMetaManager.getTempBlockMeta(TEST_TEMP_BLOCK_ID2).isPresent()); - } - - /** - * Tests that an exception is thrown in the - * {@link BlockMetadataManager#moveBlockMeta(BlockMeta, TempBlockMeta)} method when the worker is - * out of space. - */ - @Test - public void moveBlockMetaOutOfSpaceException() throws Exception { - // Create a committed block under dir2 with larger size than the capacity of dir1, - // so that WorkerOutOfSpaceException should be thrown when move this block to dir1. - - StorageDir dir1 = mMetaManager.getTier(Constants.MEDIUM_HDD).getDir(0); - StorageDir dir2 = mMetaManager.getTier(Constants.MEDIUM_HDD).getDir(1); - long maxHddDir1Capacity = TIER_CAPACITY_BYTES[1][0]; - long blockMetaSize = maxHddDir1Capacity + 1; - BlockMeta blockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, blockMetaSize, dir2); - TempBlockMeta tempBlockMeta2 = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID2, TEST_BLOCK_SIZE, dir1); - mMetaManager.addTempBlockMeta(tempBlockMeta2); - dir2.addBlockMeta(blockMeta); - - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_SPACE_FOR_BLOCK_META.getMessage(TEST_BLOCK_ID, - blockMetaSize, maxHddDir1Capacity, TIER_ALIAS[1])); - mMetaManager.moveBlockMeta(blockMeta, tempBlockMeta2); - } - - /** - * Tests the {@link BlockMetadataManager#resizeTempBlockMeta(TempBlockMeta, long)} method. - */ - @Test - public void resizeTempBlockMeta() { - StorageDir dir = mMetaManager.getTier(Constants.MEDIUM_MEM).getDir(0); - TempBlockMeta tempBlockMeta = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_BLOCK_SIZE, dir); - mMetaManager.resizeTempBlockMeta(tempBlockMeta, TEST_BLOCK_SIZE + 1); - assertEquals(TEST_BLOCK_SIZE + 1, tempBlockMeta.getBlockSize()); - } - - /** - * Tests the {@link BlockMetadataManager#getBlockMeta(long)} method. - */ - @Test - public void getBlockStoreMeta() { - BlockStoreMeta meta = mMetaManager.getBlockStoreMeta(); - Assert.assertNotNull(meta); - - // Assert the capacities are at alias level [MEM: 1000][SSD: 0][HDD: 8000] - Map exceptedCapacityBytesOnTiers = - ImmutableMap.of(Constants.MEDIUM_MEM, 1000L, Constants.MEDIUM_HDD, 8000L); - Map exceptedUsedBytesOnTiers = - ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_HDD, 0L); - assertEquals(exceptedCapacityBytesOnTiers, meta.getCapacityBytesOnTiers()); - assertEquals(exceptedUsedBytesOnTiers, meta.getUsedBytesOnTiers()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/BlockMetadataViewTest.java b/core/server/worker/src/test/java/alluxio/worker/block/BlockMetadataViewTest.java deleted file mode 100644 index e448ae84bbdc..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/BlockMetadataViewTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import alluxio.Constants; -import alluxio.exception.ExceptionMessage; -import alluxio.master.block.BlockId; -import alluxio.worker.block.meta.BlockMeta; -import alluxio.worker.block.meta.DefaultBlockMeta; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageDirEvictorView; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTier; -import alluxio.worker.block.meta.StorageTierEvictorView; -import alluxio.worker.block.meta.StorageTierView; - -import com.google.common.collect.ImmutableList; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; - -/** - * Unit tests for {@link BlockMetadataView}. - */ -public final class BlockMetadataViewTest { - private static final int TEST_TIER_ORDINAL = 0; - private static final int TEST_DIR = 0; - private static final long TEST_BLOCK_ID = 9; - private static final long TEST_BLOCK_SIZE = 20; - - private BlockMetadataManager mMetaManager; - private BlockMetadataEvictorView mMetadataView; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - /** The exception expected to be thrown. */ - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - File tempFolder = mTestFolder.newFolder(); - mMetaManager = TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath()); - mMetadataView = spy(new BlockMetadataEvictorView(mMetaManager, - new HashSet(), new HashSet())); - } - - /** - * Tests the {@link BlockMetadataEvictorView#getTierView(String)} method. - */ - @Test - public void getTierView() { - for (StorageTier tier : mMetaManager.getTiers()) { - String tierAlias = tier.getTierAlias(); - StorageTierView tierView = mMetadataView.getTierView(tierAlias); - assertEquals(tier.getTierAlias(), tierView.getTierViewAlias()); - assertEquals(tier.getTierOrdinal(), tierView.getTierViewOrdinal()); - } - } - - /** - * Tests the {@link BlockMetadataEvictorView#getTierViews()} method. - */ - @Test - public void getTierViews() { - assertEquals(mMetaManager.getTiers().size(), mMetadataView.getTierViews().size()); - } - - /** - * Tests the {@link BlockMetadataEvictorView#getTierViewsBelow(String)} method. - */ - @Test - public void getTierViewsBelow() { - for (StorageTier tier : mMetaManager.getTiers()) { - String tierAlias = tier.getTierAlias(); - assertEquals(mMetaManager.getTiersBelow(tierAlias).size(), - mMetadataView.getTierViewsBelow(tierAlias).size()); - } - } - - /** - * Tests the {@link BlockMetadataEvictorView#getAvailableBytes(BlockStoreLocation)} method. - */ - @Test - public void getAvailableBytes() { - BlockStoreLocation location; - // When location represents anyTier - location = BlockStoreLocation.anyTier(); - assertEquals(mMetaManager.getAvailableBytes(location), - mMetadataView.getAvailableBytes(location)); - // When location represents one particular tier - for (StorageTier tier : mMetaManager.getTiers()) { - String tierAlias = tier.getTierAlias(); - location = BlockStoreLocation.anyDirInTier(tierAlias); - assertEquals(mMetaManager.getAvailableBytes(location), - mMetadataView.getAvailableBytes(location)); - for (StorageDir dir : tier.getStorageDirs()) { - // When location represents one particular dir - location = dir.toBlockStoreLocation(); - assertEquals(mMetaManager.getAvailableBytes(location), - mMetadataView.getAvailableBytes(location)); - } - } - } - - /** - * Tests that an exception is thrown in the {@link BlockMetadataEvictorView#getBlockMeta(long)} - * method when the block does not exist. - */ - @Test - public void getBlockMetaNotExisting() { - assertFalse(mMetadataView.getBlockMeta(TEST_BLOCK_ID).isPresent()); - } - - /** - * Tests that an exception is thrown in the {@link BlockMetadataEvictorView#getTierView(String)} - * method when the tier alias does not exist. - */ - @Test - public void getTierNotExisting() { - String badTierAlias = Constants.MEDIUM_HDD; - mThrown.expect(IllegalArgumentException.class); - mThrown.expectMessage(ExceptionMessage.TIER_VIEW_ALIAS_NOT_FOUND.getMessage(badTierAlias)); - mMetadataView.getTierView(badTierAlias); - } - - /** - * Tests the {@link BlockMetadataEvictorView#getBlockMeta(long)} method. - */ - @Test - public void getBlockMeta() throws Exception { - StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getDir(TEST_DIR); - - // Add one block to test dir, expect block meta found - BlockMeta blockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir); - dir.addBlockMeta(blockMeta); - assertEquals(blockMeta, mMetadataView.getBlockMeta(TEST_BLOCK_ID).get()); - assertTrue(mMetadataView.isBlockEvictable(TEST_BLOCK_ID)); - - // Lock this block, expect null result - when(mMetadataView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(false); - when(mMetadataView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(true); - assertFalse(mMetadataView.getBlockMeta(TEST_BLOCK_ID).isPresent()); - assertFalse(mMetadataView.isBlockEvictable(TEST_BLOCK_ID)); - - // Pin this block, expect null result - when(mMetadataView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(true); - when(mMetadataView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(false); - assertFalse(mMetadataView.getBlockMeta(TEST_BLOCK_ID).isPresent()); - assertFalse(mMetadataView.isBlockEvictable(TEST_BLOCK_ID)); - - // No Pin or lock on this block, expect block meta found - when(mMetadataView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(false); - when(mMetadataView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(false); - assertEquals(blockMeta, mMetadataView.getBlockMeta(TEST_BLOCK_ID).get()); - assertTrue(mMetadataView.isBlockEvictable(TEST_BLOCK_ID)); - } - - /** - * Tests the {@link BlockMetadataEvictorView#isBlockLocked(long)} and the - * {@link BlockMetadataEvictorView#isBlockPinned(long)} method. - */ - @Test - public void isBlockPinnedOrLocked() { - long inode = BlockId.getFileId(TEST_BLOCK_ID); - - // With no pinned and locked blocks - assertFalse(mMetadataView.isBlockLocked(TEST_BLOCK_ID)); - assertFalse(mMetadataView.isBlockPinned(TEST_BLOCK_ID)); - - // Pin block by passing its inode to mMetadataView - HashSet pinnedInodes = new HashSet<>(); - Collections.addAll(pinnedInodes, inode); - mMetadataView = - new BlockMetadataEvictorView(mMetaManager, pinnedInodes, new HashSet()); - assertFalse(mMetadataView.isBlockLocked(TEST_BLOCK_ID)); - assertTrue(mMetadataView.isBlockPinned(TEST_BLOCK_ID)); - - // lock block - HashSet testBlockIdSet = new HashSet<>(); - Collections.addAll(testBlockIdSet, TEST_BLOCK_ID); - Collections.addAll(pinnedInodes, TEST_BLOCK_ID); - mMetadataView = new BlockMetadataEvictorView(mMetaManager, new HashSet(), - testBlockIdSet); - assertTrue(mMetadataView.isBlockLocked(TEST_BLOCK_ID)); - assertFalse(mMetadataView.isBlockPinned(TEST_BLOCK_ID)); - - // Pin and lock block - mMetadataView = new BlockMetadataEvictorView(mMetaManager, pinnedInodes, - testBlockIdSet); - assertTrue(mMetadataView.isBlockLocked(TEST_BLOCK_ID)); - assertTrue(mMetadataView.isBlockPinned(TEST_BLOCK_ID)); - } - - /** - * Assert if two TierViews are the same by comparing their contents. - */ - private void assertSameTierView(StorageTierEvictorView tierView1, - StorageTierEvictorView tierView2) { - assertEquals(tierView1.getTierViewAlias(), tierView2.getTierViewAlias()); - assertEquals(tierView1.getTierViewOrdinal(), tierView2.getTierViewOrdinal()); - List dirViews1 = ImmutableList.copyOf(tierView1.getDirViews()); - List dirViews2 = ImmutableList.copyOf(tierView2.getDirViews()); - assertEquals(dirViews1.size(), dirViews2.size()); - for (int i = 0; i < dirViews1.size(); i++) { - StorageDirEvictorView dirView1 = (StorageDirEvictorView) dirViews1.get(i); - StorageDirEvictorView dirView2 = (StorageDirEvictorView) dirViews2.get(i); - assertEquals(dirView1.getAvailableBytes(), dirView2.getAvailableBytes()); - assertEquals(dirView1.getCapacityBytes(), dirView2.getCapacityBytes()); - assertEquals(dirView1.getCommittedBytes(), dirView2.getCommittedBytes()); - assertEquals(dirView1.getDirViewIndex(), dirView2.getDirViewIndex()); - assertEquals(dirView1.getEvictableBlocks(), dirView2.getEvictableBlocks()); - assertEquals(dirView1.getEvitableBytes(), dirView2.getEvitableBytes()); - } - } - - /** - * Tests that {@code BlockMetadataEvictorView.getTierView(tierAlias)} returns the same - * TierView as {@code new StorageTierEvictorView(mMetadataManager.getTier(tierAlias), this)}. - */ - @Test - public void sameTierView() { - String tierAlias = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getTierAlias(); - StorageTierView tierView1 = mMetadataView.getTierView(tierAlias); - - // Do some operations on metadata - StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getDir(TEST_DIR); - BlockMeta blockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir); - try { - dir.addBlockMeta(blockMeta); - } catch (Exception e) { - e.printStackTrace(); - } - StorageTierEvictorView tierView2 = - new StorageTierEvictorView(mMetaManager.getTier(tierAlias), mMetadataView); - assertSameTierView((StorageTierEvictorView) tierView1, tierView2); - } - - /** - * Tests that {@link BlockMetadataEvictorView#getTierViewsBelow(String)} returns the same - * TierViews as constructing by {@link BlockMetadataManager#getTiersBelow(String)}. - */ - @Test - public void sameTierViewsBelow() { - String tierAlias = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getTierAlias(); - List tierViews1 = mMetadataView.getTierViewsBelow(tierAlias); - - // Do some operations on metadata - StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL + 1).getDir(TEST_DIR); - BlockMeta blockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir); - try { - dir.addBlockMeta(blockMeta); - } catch (Exception e) { - e.printStackTrace(); - } - List tiers2 = mMetaManager.getTiersBelow(tierAlias); - assertEquals(tierViews1.size(), tiers2.size()); - for (int i = 0; i < tierViews1.size(); i++) { - assertSameTierView((StorageTierEvictorView) tierViews1.get(i), - new StorageTierEvictorView(tiers2.get(i), mMetadataView)); - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/BlockStoreLocationTest.java b/core/server/worker/src/test/java/alluxio/worker/block/BlockStoreLocationTest.java deleted file mode 100644 index 2c3d1068556f..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/BlockStoreLocationTest.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; - -import org.junit.Assert; -import org.junit.Test; - -/** - * Unit tests for {@link BlockStoreLocation}. - */ -public final class BlockStoreLocationTest { - - /** - * Tests that a new location can be created with the constructor. - */ - @Test - public void newLocation() { - String tierAlias = Constants.MEDIUM_SSD; - int dirIndex = 3; - String mediumType = Constants.MEDIUM_SSD; - BlockStoreLocation loc = new BlockStoreLocation(tierAlias, dirIndex, mediumType); - Assert.assertNotNull(loc); - assertEquals(tierAlias, loc.tierAlias()); - assertEquals(dirIndex, loc.dir()); - assertEquals(mediumType, loc.mediumType()); - } - - /** - * Tests the {@link BlockStoreLocation#belongsTo(BlockStoreLocation)} method. - */ - @Test - public void testBelongTo() { - BlockStoreLocation anyTier = BlockStoreLocation.anyTier(); - BlockStoreLocation anyDirInTierMEM = - BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM); - BlockStoreLocation anyDirInTierHDD = - BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD); - BlockStoreLocation dirInMEM = new BlockStoreLocation(Constants.MEDIUM_MEM, 1); - BlockStoreLocation dirInHDD = new BlockStoreLocation(Constants.MEDIUM_HDD, 2); - BlockStoreLocation dirWithMediumType = - new BlockStoreLocation(Constants.MEDIUM_MEM, 1, Constants.MEDIUM_MEM); - BlockStoreLocation anyTierWithMEM = - BlockStoreLocation.anyDirInAnyTierWithMedium(Constants.MEDIUM_MEM); - - assertTrue(anyTier.belongsTo(anyTier)); - assertFalse(anyTier.belongsTo(anyDirInTierMEM)); - assertFalse(anyTier.belongsTo(anyDirInTierHDD)); - assertFalse(anyTier.belongsTo(dirInMEM)); - assertFalse(anyTier.belongsTo(dirInHDD)); - - assertTrue(anyDirInTierMEM.belongsTo(anyTier)); - assertTrue(anyDirInTierMEM.belongsTo(anyDirInTierMEM)); - assertFalse(anyDirInTierMEM.belongsTo(anyDirInTierHDD)); - assertFalse(anyDirInTierMEM.belongsTo(dirInMEM)); - assertFalse(anyDirInTierMEM.belongsTo(dirInHDD)); - - assertTrue(anyDirInTierHDD.belongsTo(anyTier)); - assertFalse(anyDirInTierHDD.belongsTo(anyDirInTierMEM)); - assertTrue(anyDirInTierHDD.belongsTo(anyDirInTierHDD)); - assertFalse(anyDirInTierHDD.belongsTo(dirInMEM)); - assertFalse(anyDirInTierHDD.belongsTo(dirInHDD)); - - assertTrue(dirInMEM.belongsTo(anyTier)); - assertTrue(dirInMEM.belongsTo(anyDirInTierMEM)); - assertFalse(dirInMEM.belongsTo(anyDirInTierHDD)); - assertTrue(dirInMEM.belongsTo(dirInMEM)); - assertFalse(dirInMEM.belongsTo(dirInHDD)); - - assertTrue(dirInHDD.belongsTo(anyTier)); - assertFalse(dirInHDD.belongsTo(anyDirInTierMEM)); - assertTrue(dirInHDD.belongsTo(anyDirInTierHDD)); - assertFalse(dirInHDD.belongsTo(dirInMEM)); - assertTrue(dirInHDD.belongsTo(dirInHDD)); - - assertTrue(dirWithMediumType.belongsTo(anyTierWithMEM)); - assertTrue(anyTierWithMEM.belongsTo(anyTier)); - } - - /** - * Tests the {@link BlockStoreLocation#equals(Object)} method. - */ - @Test - public void equals() { - BlockStoreLocation anyTier = BlockStoreLocation.anyTier(); - BlockStoreLocation anyDirInTierMEM = - BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM); - BlockStoreLocation anyDirInTierHDD = - BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD); - BlockStoreLocation dirInMEM = new BlockStoreLocation(Constants.MEDIUM_MEM, 1); - BlockStoreLocation dirInHDD = new BlockStoreLocation(Constants.MEDIUM_HDD, 2); - - assertEquals(anyTier, BlockStoreLocation.anyTier()); // Equals - assertNotEquals(anyDirInTierMEM, BlockStoreLocation.anyTier()); - assertNotEquals(anyDirInTierHDD, BlockStoreLocation.anyTier()); - assertNotEquals(dirInMEM, BlockStoreLocation.anyTier()); - assertNotEquals(dirInHDD, BlockStoreLocation.anyTier()); - - assertNotEquals(anyTier, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM)); - assertEquals(anyDirInTierMEM, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM)); // Equals - assertNotEquals(anyDirInTierHDD, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM)); - assertNotEquals(dirInMEM, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM)); - assertNotEquals(dirInHDD, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM)); - - assertNotEquals(anyTier, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD)); - assertNotEquals(anyDirInTierMEM, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD)); - assertEquals(anyDirInTierHDD, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD)); // Equals - assertNotEquals(dirInMEM, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD)); - assertNotEquals(dirInHDD, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD)); - - BlockStoreLocation loc1 = new BlockStoreLocation(Constants.MEDIUM_MEM, 1); - assertNotEquals(anyTier, loc1); - assertNotEquals(anyDirInTierMEM, loc1); - assertNotEquals(anyDirInTierHDD, loc1); - assertEquals(dirInMEM, loc1); // Equals - assertNotEquals(dirInHDD, loc1); - - BlockStoreLocation loc2 = new BlockStoreLocation(Constants.MEDIUM_HDD, 2); - assertNotEquals(anyTier, loc2); - assertNotEquals(anyDirInTierMEM, loc2); - assertNotEquals(anyDirInTierHDD, loc2); - assertNotEquals(dirInMEM, loc2); - assertEquals(dirInHDD, loc2); // Equals - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/BlockStoreMetaTest.java b/core/server/worker/src/test/java/alluxio/worker/block/BlockStoreMetaTest.java deleted file mode 100644 index 3ed5673f8390..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/BlockStoreMetaTest.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.Constants; -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageTier; - -import com.google.common.collect.ImmutableMap; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Unit tests for {@link BlockStoreMeta}. - */ -public final class BlockStoreMetaTest { - private static final long TEST_SESSION_ID = 33L; - /** block size in Bytes for test. */ - private static final long TEST_BLOCK_SIZE = 200L; - /** num of total committed blocks. */ - private static final long COMMITTED_BLOCKS_NUM = 10L; - - private BlockMetadataManager mMetadataManager; - private BlockStoreMeta mBlockStoreMeta; - private BlockStoreMeta mBlockStoreMetaFull; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - String alluxioHome = mTestFolder.newFolder().getAbsolutePath(); - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED, false); - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED, false); - mMetadataManager = TieredBlockStoreTestUtils.defaultMetadataManager(alluxioHome); - - // Add and commit COMMITTED_BLOCKS_NUM temp blocks repeatedly - StorageDir dir = mMetadataManager.getTier(Constants.MEDIUM_MEM).getDir(0); - for (long blockId = 0L; blockId < COMMITTED_BLOCKS_NUM; blockId++) { - TieredBlockStoreTestUtils.cache(TEST_SESSION_ID, blockId, TEST_BLOCK_SIZE, dir, - mMetadataManager, null); - } - mBlockStoreMeta = new DefaultBlockStoreMeta(mMetadataManager, false); - mBlockStoreMetaFull = new DefaultBlockStoreMeta(mMetadataManager, true); - } - - /** - * Tests the {@link BlockStoreMeta#getBlockList()} method. - */ - @Test - public void getBlockList() { - Map> tierAliasToBlockIds = new HashMap<>(); - for (StorageTier tier : mMetadataManager.getTiers()) { - List blockIdsOnTier = new ArrayList<>(); - for (StorageDir dir : tier.getStorageDirs()) { - blockIdsOnTier.addAll(dir.getBlockIds()); - } - tierAliasToBlockIds.put(tier.getTierAlias(), blockIdsOnTier); - } - Map> actual = mBlockStoreMetaFull.getBlockList(); - Assert.assertEquals(TieredBlockStoreTestUtils.TIER_ALIAS.length, actual.keySet().size()); - Assert.assertEquals(tierAliasToBlockIds, actual); - } - - /** - * Tests the {@link BlockStoreMeta#getCapacityBytes()} method. - */ - @Test - public void getCapacityBytes() { - Assert.assertEquals(TieredBlockStoreTestUtils.getDefaultTotalCapacityBytes(), - mBlockStoreMeta.getCapacityBytes()); - } - - /** - * Tests the {@link BlockStoreMeta#getCapacityBytes()} method. - */ - @Test - public void getCapacityBytesOnDirs() { - Map, Long> dirsToCapacityBytes = new HashMap<>(); - for (StorageTier tier : mMetadataManager.getTiers()) { - for (StorageDir dir : tier.getStorageDirs()) { - dirsToCapacityBytes.put(new Pair<>(tier.getTierAlias(), dir.getDirPath()), - dir.getCapacityBytes()); - } - } - Assert.assertEquals(dirsToCapacityBytes, mBlockStoreMeta.getCapacityBytesOnDirs()); - Assert.assertEquals(TieredBlockStoreTestUtils.getDefaultDirNum(), mBlockStoreMeta - .getCapacityBytesOnDirs().values().size()); - } - - /** - * Tests the {@link BlockStoreMeta#getCapacityBytesOnTiers()} method. - */ - @Test - public void getCapacityBytesOnTiers() { - Map expectedCapacityBytesOnTiers = - ImmutableMap.of(Constants.MEDIUM_MEM, 5000L, Constants.MEDIUM_SSD, 60000L); - Assert.assertEquals(expectedCapacityBytesOnTiers, mBlockStoreMeta.getCapacityBytesOnTiers()); - } - - /** - * Tests the {@link BlockStoreMeta#getNumberOfBlocks()} method. - */ - @Test - public void getNumberOfBlocks() { - Assert.assertEquals(COMMITTED_BLOCKS_NUM, mBlockStoreMetaFull.getNumberOfBlocks()); - } - - /** - * Tests the {@link BlockStoreMeta#getUsedBytes()} method. - */ - @Test - public void getUsedBytes() { - long usedBytes = TEST_BLOCK_SIZE * COMMITTED_BLOCKS_NUM; - Assert.assertEquals(usedBytes, mBlockStoreMeta.getUsedBytes()); - } - - /** - * Tests the {@link BlockStoreMeta#getUsedBytesOnDirs()} method. - */ - @Test - public void getUsedBytesOnDirs() { - Map, Long> dirsToUsedBytes = new HashMap<>(); - for (StorageTier tier : mMetadataManager.getTiers()) { - for (StorageDir dir : tier.getStorageDirs()) { - dirsToUsedBytes.put(new Pair<>(tier.getTierAlias(), dir.getDirPath()), - dir.getCapacityBytes() - dir.getAvailableBytes()); - } - } - Assert.assertEquals(dirsToUsedBytes, mBlockStoreMeta.getUsedBytesOnDirs()); - } - - /** - * Tests the {@link BlockStoreMeta#getUsedBytesOnTiers()} method. - */ - @Test - public void getUsedBytesOnTiers() { - long usedBytes = TEST_BLOCK_SIZE * COMMITTED_BLOCKS_NUM; - Map usedBytesOnTiers = - ImmutableMap.of(Constants.MEDIUM_MEM, usedBytes, Constants.MEDIUM_SSD, 0L); - Assert.assertEquals(usedBytesOnTiers, mBlockStoreMeta.getUsedBytesOnTiers()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/BlockWorkerMetricsTest.java b/core/server/worker/src/test/java/alluxio/worker/block/BlockWorkerMetricsTest.java deleted file mode 100644 index 0ca981e5af98..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/BlockWorkerMetricsTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.metrics.MetricInfo; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.worker.block.DefaultBlockWorker.Metrics; - -import com.google.common.collect.ImmutableMap; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * Unit tests for {@link DefaultBlockWorker.Metrics}. - */ -public final class BlockWorkerMetricsTest { - private static final String MEM = Constants.MEDIUM_MEM; - private static final String HDD = Constants.MEDIUM_HDD; - - private BlockWorker mBlockWorker; - private BlockStoreMeta mBlockStoreMeta; - - @Before - public void before() throws Exception { - Configuration.set(PropertyKey.WORKER_TIERED_STORE_LEVELS, 2); - Configuration.set(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS.format(0), MEM); - Configuration.set(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS.format(1), HDD); - MetricsSystem.clearAllMetrics(); - mBlockWorker = mock(BlockWorker.class); - mBlockStoreMeta = mock(BlockStoreMeta.class); - when(mBlockWorker.getStoreMeta()).thenReturn(mBlockStoreMeta); - when(mBlockWorker.getStoreMetaFull()).thenReturn(mBlockStoreMeta); - Metrics.registerGauges(mBlockWorker); - } - - @Test - public void testMetricsCapacity() { - when(mBlockStoreMeta.getCapacityBytes()).thenReturn(1000L); - Assert.assertEquals(1000L, getGauge(MetricKey.WORKER_CAPACITY_TOTAL.getName())); - when(mBlockStoreMeta.getUsedBytes()).thenReturn(200L); - Assert.assertEquals(200L, getGauge(MetricKey.WORKER_CAPACITY_USED.getName())); - Assert.assertEquals(800L, getGauge(MetricKey.WORKER_CAPACITY_FREE.getName())); - } - - @Test - public void testMetricsTierCapacity() { - when(mBlockStoreMeta.getCapacityBytesOnTiers()) - .thenReturn(ImmutableMap.of(MEM, 1000L, HDD, 2000L)); - when(mBlockStoreMeta.getUsedBytesOnTiers()).thenReturn(ImmutableMap.of(MEM, 100L, HDD, 200L)); - assertEquals(1000L, - getGauge(MetricKey.WORKER_CAPACITY_TOTAL.getName() + MetricInfo.TIER + MEM)); - assertEquals(2000L, - getGauge(MetricKey.WORKER_CAPACITY_TOTAL.getName() + MetricInfo.TIER + HDD)); - assertEquals(100L, getGauge(MetricKey.WORKER_CAPACITY_USED.getName() + MetricInfo.TIER + MEM)); - assertEquals(200L, getGauge(MetricKey.WORKER_CAPACITY_USED.getName() + MetricInfo.TIER + HDD)); - assertEquals(900L, getGauge(MetricKey.WORKER_CAPACITY_FREE.getName() + MetricInfo.TIER + MEM)); - assertEquals(1800L, getGauge(MetricKey.WORKER_CAPACITY_FREE.getName() + MetricInfo.TIER + HDD)); - } - - @Test - public void testMetricBocksCached() { - when(mBlockStoreMeta.getNumberOfBlocks()).thenReturn(200); - Assert.assertEquals(200, getGauge(MetricKey.WORKER_BLOCKS_CACHED.getName())); - } - - private Object getGauge(String name) { - return MetricsSystem.METRIC_REGISTRY.getGauges() - .get(MetricsSystem.getMetricName(name)).getValue(); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/CacheRequestManagerTest.java b/core/server/worker/src/test/java/alluxio/worker/block/CacheRequestManagerTest.java deleted file mode 100644 index 1d4c3ebd6b88..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/CacheRequestManagerTest.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import alluxio.AlluxioTestDirectory; -import alluxio.AlluxioURI; -import alluxio.ConfigurationRule; -import alluxio.Constants; -import alluxio.Sessions; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.URIStatus; -import alluxio.client.file.options.InStreamOptions; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.OpenFilePOptions; -import alluxio.proto.dataserver.Protocol; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.UnderFileSystemConfiguration; -import alluxio.util.CommonUtils; -import alluxio.util.FileSystemOptionsUtils; -import alluxio.util.io.BufferUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.FileBlockInfo; -import alluxio.wire.FileInfo; -import alluxio.worker.file.FileSystemMasterClient; -import alluxio.worker.grpc.GrpcExecutors; - -import com.google.common.collect.ImmutableMap; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; -import java.util.Collections; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Unit tests for {@link CacheRequestManager}. - */ -public class CacheRequestManagerTest { - // block id has to be 0 otherwise would causing start position bug - private static final long BLOCK_ID = 0L; - private static final int CHUNK_SIZE = 128; - private static final int PORT = 22; - - private CacheRequestManager mCacheRequestManager; - private DefaultBlockWorker mBlockWorker; - private BlockStore mBlockStore; - private String mRootUfs; - private final String mLocalWorkerHostname = NetworkAddressUtils.getLocalHostName( - (int) Configuration.getMs(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS)); - private Protocol.OpenUfsBlockOptions mOpenUfsBlockOptions; - private final InstancedConfiguration mConf = Configuration.modifiableGlobal(); - private final String mMemDir = - AlluxioTestDirectory.createTemporaryDirectory(Constants.MEDIUM_MEM).getAbsolutePath(); - - @Rule - public ConfigurationRule mConfigurationRule = new ConfigurationRule( - new ImmutableMap.Builder() - .put(PropertyKey.WORKER_TIERED_STORE_LEVELS, 1) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_ALIAS, Constants.MEDIUM_MEM) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_MEDIUMTYPE, Constants.MEDIUM_MEM) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_QUOTA, "1GB") - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, mMemDir) - .put(PropertyKey.WORKER_RPC_PORT, 0) - .put(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, AlluxioTestDirectory - .createTemporaryDirectory("CacheRequestManagerTest").getAbsolutePath()) - .build(), - mConf); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws IOException { - BlockMasterClient blockMasterClient = mock(BlockMasterClient.class); - BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool()); - when(blockMasterClientPool.createNewResource()).thenReturn(blockMasterClient); - FileSystemMasterClient fileSystemMasterClient = mock(FileSystemMasterClient.class); - Sessions sessions = mock(Sessions.class); - // Connect to the real UFS for testing - UfsManager ufsManager = mock(UfsManager.class); - mRootUfs = Configuration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); - UfsManager.UfsClient ufsClient = new UfsManager.UfsClient( - () -> UnderFileSystem.Factory.create(mRootUfs, - UnderFileSystemConfiguration.defaults(Configuration.global())), - new AlluxioURI(mRootUfs)); - when(ufsManager.get(anyLong())).thenReturn(ufsClient); - TieredBlockStore tieredBlockStore = new TieredBlockStore(); - AtomicReference workerId = new AtomicReference<>(-1L); - BlockStore blockStore = - new MonoBlockStore(tieredBlockStore, blockMasterClientPool, ufsManager, workerId); - mBlockWorker = spy(new DefaultBlockWorker(blockMasterClientPool, fileSystemMasterClient, - sessions, blockStore, workerId)); - mBlockStore = mBlockWorker.getBlockStore(); - FileSystemContext context = mock(FileSystemContext.class); - mCacheRequestManager = - spy(new CacheRequestManager(GrpcExecutors.CACHE_MANAGER_EXECUTOR, mBlockWorker, context)); - // Write an actual file to UFS - String testFilePath = File.createTempFile("temp", null, new File(mRootUfs)).getAbsolutePath(); - byte[] buffer = BufferUtils.getIncreasingByteArray(CHUNK_SIZE); - BufferUtils.writeBufferToFile(testFilePath, buffer); - // create options - BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE); - URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setUfsPath(testFilePath) - .setBlockIds(Collections.singletonList(BLOCK_ID)).setLength(CHUNK_SIZE) - .setBlockSizeBytes(CHUNK_SIZE) - .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); - OpenFilePOptions readOptions = FileSystemOptionsUtils.openFileDefaults(mConf); - InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, mConf, context); - mOpenUfsBlockOptions = options.getOpenUfsBlockOptions(BLOCK_ID); - } - - @Test - public void submitRequestCacheBlockFromUfs() throws Exception { - CacheRequest request = CacheRequest.newBuilder().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE) - .setOpenUfsBlockOptions(mOpenUfsBlockOptions).setSourceHost(mLocalWorkerHostname) - .setSourcePort(PORT).build(); - mCacheRequestManager.submitRequest(request); - assertTrue(mBlockStore.hasBlockMeta(BLOCK_ID)); - } - - @Test - public void submitRequestCacheBlockFromRemoteWorker() throws Exception { - String fakeRemoteWorker = mLocalWorkerHostname + "1"; - CacheRequest request = CacheRequest.newBuilder().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE) - .setOpenUfsBlockOptions(mOpenUfsBlockOptions).setSourceHost(fakeRemoteWorker) - .setSourcePort(PORT).build(); - setupMockRemoteReader(fakeRemoteWorker, PORT, BLOCK_ID, CHUNK_SIZE, mOpenUfsBlockOptions); - mCacheRequestManager.submitRequest(request); - assertTrue(mBlockStore.hasBlockMeta(BLOCK_ID)); - } - - @Test - public void submitAsyncRequestCacheBlockFromUfs() throws Exception { - CacheRequest request = CacheRequest.newBuilder().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE) - .setOpenUfsBlockOptions(mOpenUfsBlockOptions).setSourceHost(mLocalWorkerHostname) - .setSourcePort(PORT).setAsync(true).build(); - mCacheRequestManager.submitRequest(request); - CommonUtils.waitFor("wait for async cache", () -> mBlockStore.hasBlockMeta(BLOCK_ID)); - } - - @Test - public void submitAsyncRequestCacheBlockFromRemoteWorker() throws Exception { - String fakeRemoteWorker = mLocalWorkerHostname + "1"; - CacheRequest request = CacheRequest.newBuilder().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE) - .setOpenUfsBlockOptions(mOpenUfsBlockOptions).setSourceHost(fakeRemoteWorker) - .setSourcePort(PORT).setAsync(true).build(); - setupMockRemoteReader(fakeRemoteWorker, PORT, BLOCK_ID, CHUNK_SIZE, mOpenUfsBlockOptions); - mCacheRequestManager.submitRequest(request); - CommonUtils.waitFor("wait for async cache", () -> mBlockStore.hasBlockMeta(BLOCK_ID)); - } - - private void setupMockRemoteReader(String source, int port, long blockId, long blockLength, - Protocol.OpenUfsBlockOptions options) throws IOException { - InetSocketAddress sourceAddress = new InetSocketAddress(source, port); - RemoteBlockReader reader = mock(RemoteBlockReader.class); - when(mCacheRequestManager.getRemoteBlockReader(blockId, blockLength, sourceAddress, options)) - .thenReturn(reader); - ReadableByteChannel readerChannel = mock(ReadableByteChannel.class); - when(reader.getChannel()).thenReturn(readerChannel); - int bufferSize = 4 * Constants.MB; - ByteBuffer buffer = ByteBuffer.allocateDirect(bufferSize); - when(readerChannel.read(buffer)).thenReturn(-1); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/DefaultBlockWorkerExceptionTest.java b/core/server/worker/src/test/java/alluxio/worker/block/DefaultBlockWorkerExceptionTest.java deleted file mode 100644 index 2e0553965b88..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/DefaultBlockWorkerExceptionTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import alluxio.AlluxioTestDirectory; -import alluxio.AlluxioURI; -import alluxio.ConfigurationRule; -import alluxio.Constants; -import alluxio.Sessions; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.Block; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.UfsReadOptions; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.hdfs.AlluxioHdfsException; -import alluxio.underfs.options.OpenOptions; -import alluxio.worker.file.FileSystemMasterClient; - -import com.google.common.collect.ImmutableMap; -import io.grpc.Status; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.mockito.stubbing.Answer; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Test ufs exception handling for {@link DefaultBlockWorker}. - */ -public class DefaultBlockWorkerExceptionTest { - private static final String FILE_NAME = "/test"; - private static final int BLOCK_SIZE = 128; - private UnderFileSystem mUfs; - private DefaultBlockWorker mBlockWorker; - private final String mRootUfs = Configuration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); - private final String mMemDir = - AlluxioTestDirectory.createTemporaryDirectory(Constants.MEDIUM_MEM).getAbsolutePath(); - - @Rule - public ConfigurationRule mConfigurationRule = new ConfigurationRule( - new ImmutableMap.Builder().put(PropertyKey.WORKER_TIERED_STORE_LEVELS, 1) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_ALIAS, Constants.MEDIUM_MEM) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_MEDIUMTYPE, Constants.MEDIUM_MEM) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_QUOTA, "1GB") - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, mMemDir) - .put(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, - AlluxioTestDirectory.createTemporaryDirectory("DefaultBlockWorkerTest") - .getAbsolutePath()).build(), Configuration.modifiableGlobal()); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws IOException { - BlockMasterClient blockMasterClient = mock(BlockMasterClient.class); - BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool()); - when(blockMasterClientPool.createNewResource()).thenReturn(blockMasterClient); - TieredBlockStore tieredBlockStore = spy(new TieredBlockStore()); - UfsManager ufsManager = mock(UfsManager.class); - AtomicReference workerId = new AtomicReference<>(-1L); - BlockStore blockstore = - new MonoBlockStore(tieredBlockStore, blockMasterClientPool, ufsManager, workerId); - FileSystemMasterClient client = mock(FileSystemMasterClient.class); - Sessions sessions = mock(Sessions.class); - mUfs = mock(UnderFileSystem.class); - UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> mUfs, new AlluxioURI(mRootUfs)); - when(ufsManager.get(anyLong())).thenReturn(ufsClient); - mBlockWorker = - new DefaultBlockWorker(blockMasterClientPool, client, sessions, blockstore, workerId); - } - - @Test - public void loadFailure() throws IOException, ExecutionException, InterruptedException { - AlluxioHdfsException exception = AlluxioHdfsException.fromUfsException(new IOException()); - when(mUfs.openExistingFile(eq(FILE_NAME), any(OpenOptions.class))).thenThrow(exception, - new RuntimeException(), new IOException()); - int blockId = 0; - Block blocks = Block.newBuilder().setBlockId(blockId).setLength(BLOCK_SIZE).setMountId(0) - .setOffsetInFile(0).setUfsPath(FILE_NAME).build(); - List failure = - mBlockWorker.load(Collections.singletonList(blocks), UfsReadOptions.getDefaultInstance()) - .get(); - assertEquals(failure.size(), 1); - assertEquals(exception.getStatus().getCode().value(), failure.get(0).getCode()); - failure = - mBlockWorker.load(Collections.singletonList(blocks), UfsReadOptions.getDefaultInstance()) - .get(); - assertEquals(failure.size(), 1); - assertEquals(2, failure.get(0).getCode()); - failure = - mBlockWorker.load(Collections.singletonList(blocks), UfsReadOptions.getDefaultInstance()) - .get(); - assertEquals(failure.size(), 1); - assertEquals(2, failure.get(0).getCode()); - } - - @Test - public void loadTimeout() throws IOException, ExecutionException, InterruptedException { - when(mUfs.openExistingFile(eq(FILE_NAME), any(OpenOptions.class))).thenAnswer( - (Answer) invocationOnMock -> { - Thread.sleep(Configuration.getMs(PropertyKey.USER_NETWORK_RPC_KEEPALIVE_TIMEOUT) * 2); - return null; - }); - int blockId = 0; - Block blocks = Block.newBuilder().setBlockId(blockId).setLength(BLOCK_SIZE).setMountId(0) - .setOffsetInFile(0).setUfsPath(FILE_NAME).build(); - List failure = - mBlockWorker.load(Collections.singletonList(blocks), UfsReadOptions.getDefaultInstance()) - .get(); - assertEquals(failure.size(), 1); - assertEquals(Status.DEADLINE_EXCEEDED.getCode().value(), failure.get(0).getCode()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/DefaultBlockWorkerTest.java b/core/server/worker/src/test/java/alluxio/worker/block/DefaultBlockWorkerTest.java deleted file mode 100644 index 1d4edc018c8a..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/DefaultBlockWorkerTest.java +++ /dev/null @@ -1,738 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static alluxio.util.CommonUtils.waitFor; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import alluxio.AlluxioURI; -import alluxio.ConfigurationRule; -import alluxio.Constants; -import alluxio.Sessions; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.runtime.AlluxioRuntimeException; -import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.exception.status.DeadlineExceededException; -import alluxio.exception.status.NotFoundException; -import alluxio.exception.status.UnavailableException; -import alluxio.grpc.Block; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.Command; -import alluxio.grpc.CommandType; -import alluxio.grpc.GetConfigurationPOptions; -import alluxio.grpc.UfsReadOptions; -import alluxio.master.NoopUfsManager; -import alluxio.proto.dataserver.Protocol; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystemConfiguration; -import alluxio.util.IdUtils; -import alluxio.util.WaitForOptions; -import alluxio.util.io.BufferUtils; -import alluxio.util.network.NetworkAddressUtils; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.meta.TempBlockMeta; -import alluxio.worker.file.FileSystemMasterClient; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Unit tests for {@link DefaultBlockWorker}. - */ -public class DefaultBlockWorkerTest { - private static final int BLOCK_SIZE = 128; - - TieredBlockStore mTieredBlockStore; - // worker configurations - private static final long WORKER_ID = 30L; - // ufs for fallback read - private static final long UFS_MOUNT_ID = 1L; - // ufs for batch load - private static final long UFS_LOAD_MOUNT_ID = 2L; - private static final WorkerNetAddress WORKER_ADDRESS = - new WorkerNetAddress().setHost("localhost").setRpcPort(20001); - - // invalid initial worker id - private static final long INVALID_WORKER_ID = -1L; - - // test subject - private DefaultBlockWorker mBlockWorker; - - // mocked dependencies of DefaultBlockWorker - private BlockMasterClient mBlockMasterClient; - private FileSystemMasterClient mFileSystemMasterClient; - - private final Random mRandom = new Random(); - - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - // worker's local storage directories - private String mMemDir; - private String mHddDir; - // ufs file for fallback read - private File mTestUfsFile; - - // ufs root path for batch load - private String mRootUfs; - // ufs file for batch load - private String mTestLoadFilePath; - - @Rule - public ConfigurationRule mConfigurationRule = - new ConfigurationRule(new ImmutableMap.Builder() - .put(PropertyKey.WORKER_TIERED_STORE_LEVELS, 2) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_ALIAS, Constants.MEDIUM_MEM) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_MEDIUMTYPE, Constants.MEDIUM_MEM) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_QUOTA, "1GB") - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL1_ALIAS, Constants.MEDIUM_HDD) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL1_DIRS_MEDIUMTYPE, Constants.MEDIUM_HDD) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL1_DIRS_QUOTA, "2GB") - .put(PropertyKey.WORKER_RPC_PORT, 0) - .put(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_RESERVED_BYTES, "0") - .put(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS, "10ms") - .build(), Configuration.modifiableGlobal()); - private BlockStore mBlockStore; - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - // set up storage directories - mMemDir = mTestFolder.newFolder().getAbsolutePath(); - mHddDir = mTestFolder.newFolder().getAbsolutePath(); - mConfigurationRule.set(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, mMemDir); - mConfigurationRule.set(PropertyKey.WORKER_TIERED_STORE_LEVEL1_DIRS_PATH, mHddDir); - - // set up BlockMasterClient - mBlockMasterClient = createMockBlockMasterClient(); - BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool()); - doReturn(mBlockMasterClient).when(blockMasterClientPool).createNewResource(); - - mTieredBlockStore = spy(new TieredBlockStore()); - UfsManager ufsManager = new NoopUfsManager(); - AtomicReference workerId = new AtomicReference<>(INVALID_WORKER_ID); - mBlockStore = - spy(new MonoBlockStore(mTieredBlockStore, blockMasterClientPool, ufsManager, workerId)); - - mFileSystemMasterClient = createMockFileSystemMasterClient(); - - Sessions sessions = mock(Sessions.class); - - // set up a ufs directory for batch load jobs - mRootUfs = mTestFolder.newFolder("DefaultBlockWorkerTest").getAbsolutePath(); - mConfigurationRule.set(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, mRootUfs); - ufsManager.addMount(UFS_LOAD_MOUNT_ID, - new AlluxioURI(mRootUfs), - UnderFileSystemConfiguration.defaults(Configuration.global())); - // Write an actual file to UFS - mTestLoadFilePath = mTestFolder.newFile("temp").getAbsolutePath(); - byte[] buffer = BufferUtils.getIncreasingByteArray((int) (BLOCK_SIZE * 1.5)); - BufferUtils.writeBufferToFile(mTestLoadFilePath, buffer); - - // set up ufs directory for fallback reading - mTestUfsFile = mTestFolder.newFile(); - // mount test file to UFS_MOUNT_ID - ufsManager.addMount( - UFS_MOUNT_ID, - new AlluxioURI(mTestUfsFile.getAbsolutePath()), - UnderFileSystemConfiguration.defaults(Configuration.global()) - ); - - mBlockWorker = new DefaultBlockWorker(blockMasterClientPool, mFileSystemMasterClient, - sessions, mBlockStore, workerId); - } - - @Test - public void getWorkerId() throws Exception { - mBlockWorker.askForWorkerId(WORKER_ADDRESS); - assertEquals(WORKER_ID, (long) mBlockWorker.getWorkerId().get()); - } - - @Test - public void getWorkerInfo() { - // block worker has no dependencies - assertEquals(new HashSet<>(), mBlockWorker.getDependencies()); - // block worker does not expose services - assertEquals(ImmutableMap.of(), mBlockWorker.getServices()); - assertEquals(Constants.BLOCK_WORKER_NAME, mBlockWorker.getName()); - // white list should match configuration default - assertEquals(ImmutableList.of("/"), mBlockWorker.getWhiteList()); - } - - @Test - public void abortBlock() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - mBlockWorker.abortBlock(sessionId, blockId); - assertFalse(mBlockWorker.getBlockStore().getTempBlockMeta(blockId).isPresent()); - } - - @Test - public void createBlockAfterAbort() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - mBlockWorker.abortBlock(sessionId, blockId); - assertFalse(mBlockWorker.getBlockStore().getTempBlockMeta(blockId).isPresent()); - sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - } - - @Test - public void commitBlock() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - assertFalse(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - mBlockWorker.commitBlock(sessionId, blockId, true); - assertTrue(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - } - - @Test - public void commitBlockOnRetry() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - mBlockWorker.commitBlock(sessionId, blockId, true); - mBlockWorker.commitBlock(sessionId, blockId, true); - assertTrue(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - } - - @Test - public void commitBlockFailure() throws Exception { - // simulate master failure in committing block - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - - // simulate server failure to commit block - doThrow(new UnavailableException("test")) - .when(mBlockMasterClient) - .commitBlock( - anyLong(), - anyLong(), - anyString(), - anyString(), - anyLong(), - anyLong()); - - mBlockWorker.createBlock( - sessionId, - blockId, - 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - assertThrows(AlluxioRuntimeException.class, - () -> mBlockWorker.commitBlock(sessionId, blockId, false)); - } - - @Test - public void commitBlockInUfs() throws Exception { - long blockId = mRandom.nextLong(); - long ufsBlockLength = 1024; - mBlockWorker.commitBlockInUfs(blockId, ufsBlockLength); - - verify(mBlockMasterClient, times(1)) - .commitBlockInUfs(blockId, ufsBlockLength); - } - - @Test - public void commitBlockInUfsFailure() throws Exception { - long blockId = mRandom.nextLong(); - long ufsBlockLength = 1024; - - // simulate server failure to commit ufs block - doThrow(new UnavailableException("test")) - .when(mBlockMasterClient) - .commitBlockInUfs(anyLong(), anyLong()); - - assertThrows(AlluxioRuntimeException.class, - () -> mBlockWorker.commitBlockInUfs(blockId, ufsBlockLength)); - } - - @Test - public void createBlock() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - long initialBytes = 1; - String path = mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, null, initialBytes)); - assertTrue(path.startsWith(mMemDir)); // tier 0 is mem - } - - @Test - public void createBlockOutOfSpace() { - // simulates worker out of space - doThrow(ResourceExhaustedRuntimeException.class) - .when(mBlockStore) - .createBlock(anyLong(), anyLong(), anyInt(), any(CreateBlockOptions.class)); - - long sessionId = mRandom.nextLong(); - long blockId = mRandom.nextLong(); - - assertThrows( - ResourceExhaustedRuntimeException.class, - () -> mBlockWorker.createBlock( - sessionId, - blockId, - 0, - new CreateBlockOptions(null, null, 1))); - } - - @Test - public void createBlockLowerTier() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - long initialBytes = 1; - String path = mBlockWorker.createBlock(sessionId, blockId, 1, - new CreateBlockOptions(null, null, initialBytes)); - assertTrue(path.startsWith(mHddDir)); - } - - @Test - public void getTempBlockWriter() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - try (BlockWriter blockWriter = mBlockWorker.createBlockWriter(sessionId, blockId)) { - blockWriter.append(BufferUtils.getIncreasingByteBuffer(10)); - TempBlockMeta meta = mBlockWorker.getBlockStore().getTempBlockMeta(blockId).get(); - assertEquals(Constants.MEDIUM_MEM, meta.getBlockLocation().mediumType()); - } - mBlockWorker.abortBlock(sessionId, blockId); - } - - @Test - public void getReport() { - BlockHeartbeatReport report = mBlockWorker.getReport(); - assertEquals(0, report.getAddedBlocks().size()); - assertEquals(0, report.getRemovedBlocks().size()); - } - - @Test - public void getStoreMeta() throws Exception { - long blockId1 = mRandom.nextLong(); - long blockId2 = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId1, 0, new CreateBlockOptions(null, "", 1L)); - mBlockWorker.createBlock(sessionId, blockId2, 1, new CreateBlockOptions(null, "", 1L)); - - BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull(); - assertEquals(2, storeMeta.getBlockList().size()); - assertEquals(2, storeMeta.getBlockListByStorageLocation().size()); - assertEquals(0, storeMeta.getBlockList().get("MEM").size()); - assertEquals(3L * Constants.GB, storeMeta.getCapacityBytes()); - assertEquals(2L, storeMeta.getUsedBytes()); - assertEquals(1L, storeMeta.getUsedBytesOnTiers().get("MEM").longValue()); - assertEquals(1L, storeMeta.getUsedBytesOnTiers().get("HDD").longValue()); - - BlockStoreMeta storeMeta2 = mBlockWorker.getStoreMeta(); - assertEquals(3L * Constants.GB, storeMeta2.getCapacityBytes()); - assertEquals(2L, storeMeta2.getUsedBytes()); - - mBlockWorker.commitBlock(sessionId, blockId1, true); - mBlockWorker.commitBlock(sessionId, blockId2, true); - - storeMeta = mBlockWorker.getStoreMetaFull(); - assertEquals(1, storeMeta.getBlockList().get("MEM").size()); - assertEquals(1, storeMeta.getBlockList().get("HDD").size()); - Map> blockLocations = storeMeta.getBlockListByStorageLocation(); - assertEquals(1, blockLocations.get( - new BlockStoreLocation("MEM", 0, "MEM")).size()); - assertEquals(1, blockLocations.get( - new BlockStoreLocation("HDD", 0, "HDD")).size()); - assertEquals(2, storeMeta.getNumberOfBlocks()); - } - - @Test - public void hasBlockMeta() throws Exception { - long sessionId = mRandom.nextLong(); - long blockId = mRandom.nextLong(); - assertFalse(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - mBlockWorker.commitBlock(sessionId, blockId, true); - assertTrue(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - } - - @Test - public void removeBlock() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId, 1, new CreateBlockOptions(null, "", 1)); - mBlockWorker.commitBlock(sessionId, blockId, true); - mBlockWorker.removeBlock(sessionId, blockId); - assertFalse(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - } - - @Test - public void requestSpace() { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - long initialBytes = 512; - long additionalBytes = 1024; - mBlockWorker.createBlock(sessionId, blockId, 1, new CreateBlockOptions(null, "", initialBytes)); - mBlockWorker.requestSpace(sessionId, blockId, additionalBytes); - assertEquals(initialBytes + additionalBytes, - mBlockWorker.getBlockStore().getTempBlockMeta(blockId).get().getBlockSize()); - } - - @Test - public void requestSpaceNoBlock() { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - long additionalBytes = 1; - assertThrows(IllegalStateException.class, - () -> mBlockWorker.requestSpace(sessionId, blockId, additionalBytes) - ); - } - - @Test - public void requestSpaceNoSpace() { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - long additionalBytes = 2L * Constants.GB + 1; - mBlockWorker.createBlock(sessionId, blockId, 1, new CreateBlockOptions(null, "", 1)); - assertThrows(ResourceExhaustedRuntimeException.class, - () -> mBlockWorker.requestSpace(sessionId, blockId, additionalBytes) - ); - } - - @Test - public void updatePinList() { - Set pinnedInodes = new HashSet<>(); - pinnedInodes.add(mRandom.nextLong()); - - mBlockWorker.updatePinList(pinnedInodes); - verify(mTieredBlockStore).updatePinnedInodes(pinnedInodes); - } - - @Test - public void getFileInfo() throws Exception { - long fileId = mRandom.nextLong(); - mBlockWorker.getFileInfo(fileId); - verify(mFileSystemMasterClient).getFileInfo(fileId); - } - - @Test - public void getBlockReader() throws Exception { - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - mBlockWorker.createBlock(sessionId, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - mBlockWorker.commitBlock(sessionId, blockId, true); - BlockReader reader = mBlockWorker.createBlockReader(IdUtils.createSessionId(), blockId, 0, - false, Protocol.OpenUfsBlockOptions.newBuilder().build()); - // reader will hold the lock - assertThrows(DeadlineExceededException.class, - () -> mTieredBlockStore.removeBlockInternal(sessionId, blockId, 10) - ); - reader.close(); - mTieredBlockStore.removeBlockInternal(sessionId, blockId, 10); - } - - @Test - public void loadMultipleFromUfs() throws IOException, ExecutionException, InterruptedException { - Block block = - Block.newBuilder().setBlockId(0).setLength(BLOCK_SIZE) - .setMountId(UFS_LOAD_MOUNT_ID).setOffsetInFile(0).setUfsPath(mTestLoadFilePath).build(); - Block block2 = Block.newBuilder().setBlockId(1).setLength(BLOCK_SIZE / 2) - .setMountId(UFS_LOAD_MOUNT_ID).setOffsetInFile(BLOCK_SIZE) - .setUfsPath(mTestLoadFilePath).build(); - - List res = - mBlockWorker.load(Arrays.asList(block, block2), UfsReadOptions.getDefaultInstance()).get(); - assertEquals(res.size(), 0); - assertTrue(mBlockStore.hasBlockMeta(0)); - assertTrue(mBlockStore.hasBlockMeta(1)); - BlockReader reader = mBlockWorker.createBlockReader(0, 0, 0, false, - Protocol.OpenUfsBlockOptions.getDefaultInstance()); - // Read entire block by setting the length to be block size. - ByteBuffer buffer = reader.read(0, BLOCK_SIZE); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, BLOCK_SIZE, buffer)); - reader = mBlockWorker.createBlockReader(0, 1, 0, false, - Protocol.OpenUfsBlockOptions.getDefaultInstance()); - buffer = reader.read(0, BLOCK_SIZE / 2); - assertTrue(BufferUtils.equalIncreasingByteBuffer(BLOCK_SIZE, BLOCK_SIZE / 2, buffer)); - } - - @Test - public void loadDuplicateBlock() throws ExecutionException, InterruptedException { - int blockId = 0; - Block blocks = Block.newBuilder().setBlockId(blockId).setLength(BLOCK_SIZE) - .setMountId(UFS_LOAD_MOUNT_ID).setOffsetInFile(0).setUfsPath(mTestLoadFilePath).build(); - List res = - mBlockWorker.load(Collections.singletonList(blocks), UfsReadOptions.getDefaultInstance()) - .get(); - assertEquals(res.size(), 0); - List failure = - mBlockWorker.load(Collections.singletonList(blocks), UfsReadOptions.getDefaultInstance()) - .get(); - assertEquals(failure.size(), 1); - } - - @Test - public void getFallBackUfsReader() throws Exception { - long ufsBlockSize = 1024; - // flush some data to under file system - byte[] data = new byte[(int) ufsBlockSize]; - Arrays.fill(data, (byte) 7); - try (FileOutputStream fileOut = new FileOutputStream(mTestUfsFile); - BufferedOutputStream bufOut = new BufferedOutputStream(fileOut)) { - bufOut.write(data); - bufOut.flush(); - } - - long sessionId = mRandom.nextLong(); - long blockId = mRandom.nextLong(); - Protocol.OpenUfsBlockOptions options = Protocol.OpenUfsBlockOptions - .newBuilder() - .setMountId(UFS_MOUNT_ID) - .setBlockSize(ufsBlockSize) - .setUfsPath(mTestUfsFile.getAbsolutePath()) - .setBlockInUfsTier(true) - .build(); - - // this read should fall back to ufs - BlockReader reader = mBlockWorker.createBlockReader( - sessionId, blockId, 0, false, options); - - // read a whole block - assertArrayEquals(data, reader.read(0, ufsBlockSize).array()); - reader.close(); - - // after closing, the ufs block should be cached locally - assertTrue(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - } - - @Test - public void getUnavailableBlockReader() { - // access a non-existent non-ufs block - long blockId = mRandom.nextLong(); - long sessionId = mRandom.nextLong(); - - assertThrows( - BlockDoesNotExistRuntimeException.class, - () -> mBlockWorker.createBlockReader( - sessionId, blockId, 0, false, Protocol.OpenUfsBlockOptions.newBuilder().build())); - } - - @Test - public void getUnavailableUfsBlockReader() { - // access a non-existent ufs path - long sessionId = mRandom.nextLong(); - long blockId = mRandom.nextLong(); - Protocol.OpenUfsBlockOptions options = Protocol.OpenUfsBlockOptions - .newBuilder() - .setUfsPath("/nonexistent/path") - .setBlockInUfsTier(true) - .setBlockSize(1024) - .setMountId(UFS_MOUNT_ID) - .build(); - - assertThrows(NotFoundException.class, - () -> mBlockWorker.createUfsBlockReader( - sessionId, blockId, 0, false, options)); - } - - @Test - public void cacheBlockSync() throws Exception { - cacheBlock(false); - } - - @Test - public void cacheBlockAsync() throws Exception { - cacheBlock(true); - } - - @Test - public void getConfiguration() { - alluxio.wire.Configuration conf = mBlockWorker.getConfiguration( - GetConfigurationPOptions - .newBuilder() - .setIgnoreClusterConf(false) - .setIgnorePathConf(false) - .setRawValue(true) - .build()); - assertEquals(conf.getClusterConfHash(), Configuration.hash()); - } - - @Test - public void cleanUpSession() throws Exception { - long sessionId = mRandom.nextLong(); - long blockId = mRandom.nextLong(); - - mBlockWorker.createBlock( - sessionId, - blockId, - 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - mBlockWorker.commitBlock(sessionId, blockId, false); - - // just to hold a read lock on the block - BlockReader reader = mBlockWorker.createBlockReader( - sessionId, blockId, 0, false, Protocol.OpenUfsBlockOptions.newBuilder().build()); - - long anotherSessionId = mRandom.nextLong(); - - // clean up the first session - mBlockWorker.cleanupSession(sessionId); - - // now another session should be able to grab write lock on the block - mBlockWorker.removeBlock(anotherSessionId, blockId); - } - - private void cacheBlock(boolean async) throws Exception { - // flush 1MB random data to ufs so that caching will take a while - long ufsBlockSize = 1024 * 1024; - byte[] data = new byte[(int) ufsBlockSize]; - mRandom.nextBytes(data); - - try (FileOutputStream fileOut = new FileOutputStream(mTestUfsFile); - BufferedOutputStream bufOut = new BufferedOutputStream(fileOut)) { - bufOut.write(data); - bufOut.flush(); - } - - // ufs options: delegate to the ufs mounted at UFS_MOUNT_ID - // with path to our test file - long blockId = mRandom.nextLong(); - Protocol.OpenUfsBlockOptions options = Protocol.OpenUfsBlockOptions - .newBuilder() - .setBlockSize(ufsBlockSize) - .setUfsPath(mTestUfsFile.getAbsolutePath()) - .setMountId(UFS_MOUNT_ID) - .setNoCache(false) - .setOffsetInFile(0) - .build(); - - // cache request: - // delegate to local ufs client rather than remote worker - CacheRequest request = CacheRequest - .newBuilder() - .setSourceHost(NetworkAddressUtils.getLocalHostName(500)) - .setBlockId(blockId) - .setLength(ufsBlockSize) - .setAsync(async) - .setOpenUfsBlockOptions(options) - .build(); - - mBlockWorker.cache(request); - - // check that the block metadata is present - if (async) { - assertFalse(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - waitFor( - "Wait for async cache", - () -> mBlockWorker.getBlockStore().hasBlockMeta(blockId), - WaitForOptions.defaults().setInterval(10).setTimeoutMs(2000)); - } else { - assertTrue(mBlockWorker.getBlockStore().hasBlockMeta(blockId)); - } - - long sessionId = mRandom.nextLong(); - // check that we can read the block locally - // note: this time we use an OpenUfsOption without ufsPath and blockInUfsTier so - // that the worker can't fall back to ufs read. - Protocol.OpenUfsBlockOptions noFallbackOptions = Protocol.OpenUfsBlockOptions.newBuilder() - .setBlockInUfsTier(false).build(); - try (BlockReader reader = mBlockWorker.createBlockReader( - sessionId, blockId, 0, false, noFallbackOptions)) { - ByteBuffer buf = reader.read(0, ufsBlockSize); - // alert: LocalFileBlockReader uses a MappedByteBuffer, which does not - // support the array operation. So we need to compare ByteBuffer manually - assertEquals(0, buf.compareTo(ByteBuffer.wrap(data))); - } - } - - // create a BlockMasterClient that simulates reasonable default - // interactions with the block master - private BlockMasterClient createMockBlockMasterClient() throws Exception { - BlockMasterClient client = mock(BlockMasterClient.class); - - // return designated worker id - doReturn(WORKER_ID) - .when(client) - .getId(any(WorkerNetAddress.class)); - - // return Command.Nothing for heartbeat - doReturn(Command.newBuilder().setCommandType(CommandType.Nothing).build()) - .when(client) - .heartbeat( - anyLong(), - anyMap(), - anyMap(), - anyList(), - anyMap(), - anyMap(), - anyList() - ); - return client; - } - - // create a mocked FileSystemMasterClient that simulates reasonable default - // interactions with file system master - private FileSystemMasterClient createMockFileSystemMasterClient() throws Exception { - FileSystemMasterClient client = mock(FileSystemMasterClient.class); - doReturn(ImmutableSet.of()) - .when(client) - .getPinList(); - return client; - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/NoopBlockWorker.java b/core/server/worker/src/test/java/alluxio/worker/block/NoopBlockWorker.java deleted file mode 100644 index 6381349827d5..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/NoopBlockWorker.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import alluxio.Server; -import alluxio.grpc.AsyncCacheRequest; -import alluxio.grpc.Block; -import alluxio.grpc.BlockStatus; -import alluxio.grpc.CacheRequest; -import alluxio.grpc.GetConfigurationPOptions; -import alluxio.grpc.GrpcService; -import alluxio.grpc.ServiceType; -import alluxio.grpc.UfsReadOptions; -import alluxio.proto.dataserver.Protocol; -import alluxio.wire.Configuration; -import alluxio.wire.FileInfo; -import alluxio.wire.WorkerNetAddress; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.BlockWriter; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicReference; - -/** - * A block worker mock for testing. - */ -public class NoopBlockWorker implements BlockWorker { - - @Override - public AtomicReference getWorkerId() { - return null; - } - - @Override - public void abortBlock(long sessionId, long blockId) throws IOException { - // noop - } - - @Override - public void commitBlock(long sessionId, long blockId, boolean pinOnCreate) { - // noop - } - - @Override - public void commitBlockInUfs(long blockId, long length) { - // noop - } - - @Override - public String createBlock(long sessionId, long blockId, int tier, - CreateBlockOptions createBlockOptions) { - return null; - } - - @Override - public BlockWriter createBlockWriter(long sessionId, long blockId) - throws IOException { - return null; - } - - @Override - public BlockHeartbeatReport getReport() { - return null; - } - - @Override - public BlockStoreMeta getStoreMeta() { - return null; - } - - @Override - public BlockStoreMeta getStoreMetaFull() { - return null; - } - - @Override - public BlockReader createUfsBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException { - return null; - } - - @Override - public void removeBlock(long sessionId, long blockId) - throws IOException { - // noop - } - - @Override - public void freeWorker() - throws IOException { - // noop - } - - @Override - public void requestSpace(long sessionId, long blockId, long additionalBytes) { - // noop - } - - @Override - public void asyncCache(AsyncCacheRequest request) { - // noop - } - - @Override - public void cache(CacheRequest request) { - // noop - } - - @Override - public CompletableFuture> load(List blocks, UfsReadOptions options) { - return null; - } - - @Override - public void updatePinList(Set pinnedInodes) { - // noop - } - - @Override - public FileInfo getFileInfo(long fileId) throws IOException { - return null; - } - - @Override - public BlockReader createBlockReader(long sessionId, long blockId, long offset, - boolean positionShort, Protocol.OpenUfsBlockOptions options) - throws IOException { - return null; - } - - @Override - public void clearMetrics() { - // noop - } - - @Override - public Configuration getConfiguration(GetConfigurationPOptions options) { - return null; - } - - @Override - public List getWhiteList() { - return null; - } - - @Override - public BlockStore getBlockStore() { - throw new UnsupportedOperationException(); - } - - @Override - public Set> getDependencies() { - return null; - } - - @Override - public String getName() { - return null; - } - - @Override - public Map getServices() { - return null; - } - - @Override - public void start(WorkerNetAddress options) throws IOException { - // noop - } - - @Override - public void stop() throws IOException { - // noop - } - - @Override - public void close() throws IOException { - // noop - } - - @Override - public void cleanupSession(long sessionId) { - // noop - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/PinListSyncTest.java b/core/server/worker/src/test/java/alluxio/worker/block/PinListSyncTest.java deleted file mode 100644 index 2e8b44920ef6..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/PinListSyncTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; - -import alluxio.ClientContext; -import alluxio.conf.Configuration; -import alluxio.master.MasterClientContext; -import alluxio.worker.file.FileSystemMasterClient; - -import com.google.common.collect.ImmutableSet; -import org.junit.Test; - -import java.io.IOException; -import java.util.Set; - -public class PinListSyncTest { - - // context for creating FileSystemMasterClient - private final MasterClientContext mContext = MasterClientContext.newBuilder(ClientContext.create( - Configuration.global())).build(); - - private final TestBlockWorker mBlockWorker = new TestBlockWorker(); - - @Test - public void syncPinList() { - Set testPinLists = ImmutableSet.of(1L, 2L, 3L); - // simulates pin list update returned by master - FileSystemMasterClient client = new FileSystemMasterClient(mContext) { - @Override - public Set getPinList() { - return ImmutableSet.copyOf(testPinLists); - } - }; - - PinListSync sync = new PinListSync(mBlockWorker, client); - sync.heartbeat(); - - // should receive the latest pin list - assertEquals(testPinLists, mBlockWorker.getPinList()); - } - - @Test - public void syncPinListFailure() { - // simulates get pin list failure - FileSystemMasterClient client = new FileSystemMasterClient(mContext) { - @Override - public Set getPinList() throws IOException { - throw new IOException(); - } - }; - - PinListSync sync = new PinListSync(mBlockWorker, client); - // should fail - sync.heartbeat(); - - // should not get any pin list update - assertEquals(ImmutableSet.of(), mBlockWorker.getPinList()); - } - - private static final class TestBlockWorker extends NoopBlockWorker { - private Set mLatestPinList = ImmutableSet.of(); - - @Override - public void updatePinList(Set pinList) { - mLatestPinList = ImmutableSet.copyOf(pinList); - } - - public Set getPinList() { - return mLatestPinList; - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/TieredBlockStoreTest.java b/core/server/worker/src/test/java/alluxio/worker/block/TieredBlockStoreTest.java deleted file mode 100644 index 5676b11801fd..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/TieredBlockStoreTest.java +++ /dev/null @@ -1,687 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.retry.CountingRetry; -import alluxio.retry.RetryPolicy; -import alluxio.test.util.ConcurrencyUtils; -import alluxio.util.CommonUtils; -import alluxio.util.io.FileUtils; -import alluxio.worker.block.annotator.BlockIterator; -import alluxio.worker.block.evictor.EvictionPlan; -import alluxio.worker.block.evictor.Evictor; -import alluxio.worker.block.evictor.Evictor.Mode; -import alluxio.worker.block.meta.DefaultBlockMeta; -import alluxio.worker.block.meta.DefaultTempBlockMeta; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.TempBlockMeta; - -import com.google.common.collect.Sets; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; -import org.mockito.invocation.InvocationOnMock; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * Unit tests for {@link TieredBlockStore}. - */ -public final class TieredBlockStoreTest { - private static final long SESSION_ID1 = 2; - private static final long SESSION_ID2 = 3; - private static final long BLOCK_ID1 = 1000; - private static final long BLOCK_ID2 = 1001; - private static final long TEMP_BLOCK_ID = 1003; - private static final long TEMP_BLOCK_ID2 = 1004; - private static final long BLOCK_SIZE = 512; - private static final String FIRST_TIER_ALIAS = TieredBlockStoreTestUtils.TIER_ALIAS[0]; - private static final String SECOND_TIER_ALIAS = TieredBlockStoreTestUtils.TIER_ALIAS[1]; - private TieredBlockStore mBlockStore; - private BlockMetadataManager mMetaManager; - private BlockLockManager mLockManager; - private StorageDir mTestDir1; - private StorageDir mTestDir2; - private StorageDir mTestDir3; - private StorageDir mTestDir4; - private BlockIterator mBlockIterator; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - /** The exception expected to be thrown. */ - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - @Before - public void before() throws Exception { - Configuration.reloadProperties(); - init(0); - } - - private void init(long reservedBytes) throws Exception { - Configuration.set(PropertyKey.WORKER_REVIEWER_CLASS, - "alluxio.worker.block.reviewer.AcceptingReviewer"); - // No reserved space for tests that are not swap related. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_RESERVED_BYTES, reservedBytes); - - File tempFolder = mTestFolder.newFolder(); - TieredBlockStoreTestUtils.setupDefaultConf(tempFolder.getAbsolutePath()); - mMetaManager = BlockMetadataManager.createBlockMetadataManager(); - mLockManager = new BlockLockManager(); - mBlockStore = new TieredBlockStore(mMetaManager, mLockManager); - mBlockIterator = mMetaManager.getBlockIterator(); - - mTestDir1 = mMetaManager.getTier(FIRST_TIER_ALIAS).getDir(0); - mTestDir2 = mMetaManager.getTier(FIRST_TIER_ALIAS).getDir(1); - mTestDir3 = mMetaManager.getTier(SECOND_TIER_ALIAS).getDir(1); - mTestDir4 = mMetaManager.getTier(SECOND_TIER_ALIAS).getDir(2); - } - - @Test - public void differentSessionLockDifferentBlocks() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - TieredBlockStoreTestUtils.cache2(SESSION_ID2, BLOCK_ID2, BLOCK_SIZE, mTestDir2, mMetaManager, - mBlockIterator); - - Optional lock1 = mBlockStore.pinBlock(SESSION_ID1, BLOCK_ID1); - assertTrue( - Sets.difference(mLockManager.getLockedBlocks(), Sets.newHashSet(BLOCK_ID1)).isEmpty()); - - Optional lock2 = mBlockStore.pinBlock(SESSION_ID2, BLOCK_ID2); - assertNotEquals(lock1, lock2); - assertTrue( - Sets.difference(mLockManager.getLockedBlocks(), Sets.newHashSet(BLOCK_ID1, BLOCK_ID2)) - .isEmpty()); - - lock2.get().close(); - assertTrue( - Sets.difference(mLockManager.getLockedBlocks(), Sets.newHashSet(BLOCK_ID1)).isEmpty()); - - lock1.get().close(); - assertTrue(mLockManager.getLockedBlocks().isEmpty()); - } - - @Test - public void sameSessionLockDifferentBlocks() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID2, BLOCK_SIZE, mTestDir2, mMetaManager, - mBlockIterator); - - BlockLock lockId1 = mBlockStore.pinBlock(SESSION_ID1, BLOCK_ID1).get(); - assertTrue( - Sets.difference(mLockManager.getLockedBlocks(), Sets.newHashSet(BLOCK_ID1)).isEmpty()); - - BlockLock lockId2 = mBlockStore.pinBlock(SESSION_ID1, BLOCK_ID2).get(); - assertNotEquals(lockId1.get(), lockId2.get()); - } - - @Test - public void pinNonExistingBlock() { - assertFalse(mBlockStore.pinBlock(SESSION_ID1, BLOCK_ID1).isPresent()); - } - - @Test - public void commitBlock() throws Exception { - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE, mTestDir1); - assertFalse(mBlockStore.hasBlockMeta(TEMP_BLOCK_ID)); - mBlockStore.commitBlock(SESSION_ID1, TEMP_BLOCK_ID, false); - assertTrue(mBlockStore.hasBlockMeta(TEMP_BLOCK_ID)); - assertFalse( - FileUtils.exists(DefaultTempBlockMeta.tempPath(mTestDir1, SESSION_ID1, TEMP_BLOCK_ID))); - assertTrue(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, TEMP_BLOCK_ID))); - } - - @Test - public void abortBlock() throws Exception { - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE, mTestDir1); - mBlockStore.abortBlock(SESSION_ID1, TEMP_BLOCK_ID); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID1)); - assertFalse(mBlockStore.hasBlockMeta(TEMP_BLOCK_ID)); - assertFalse( - FileUtils.exists(DefaultTempBlockMeta.tempPath(mTestDir1, SESSION_ID1, TEMP_BLOCK_ID))); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, TEMP_BLOCK_ID))); - } - - @Test - public void moveBlock() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - mBlockStore.moveBlock(SESSION_ID1, BLOCK_ID1, - AllocateOptions.forMove(mTestDir2.toBlockStoreLocation())); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID1)); - assertTrue(mTestDir2.hasBlockMeta(BLOCK_ID1)); - assertTrue(mBlockStore.hasBlockMeta(BLOCK_ID1)); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID1))); - assertTrue(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir2, BLOCK_ID1))); - - // Move block from the specific Dir - TieredBlockStoreTestUtils.cache2(SESSION_ID2, BLOCK_ID2, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - // Move block from right Dir - mBlockStore.moveBlock(SESSION_ID2, BLOCK_ID2, - AllocateOptions.forMove(mTestDir3.toBlockStoreLocation())); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID2)); - assertTrue(mTestDir3.hasBlockMeta(BLOCK_ID2)); - assertTrue(mBlockStore.hasBlockMeta(BLOCK_ID2)); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID2))); - assertTrue(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir3, BLOCK_ID2))); - } - - @Test - public void tierMoveTargetIsFull() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - // Fill up the target dir - TieredBlockStoreTestUtils.cache2(SESSION_ID2, BLOCK_ID2, mTestDir2.getCapacityBytes(), - mTestDir2, mMetaManager, mBlockIterator); - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_SPACE_FOR_BLOCK_MOVE.getMessage( - mTestDir2.toBlockStoreLocation(), BLOCK_ID1)); - mBlockStore.moveBlock(SESSION_ID1, BLOCK_ID1, - AllocateOptions.forTierMove(mTestDir2.toBlockStoreLocation())); - // Consistency check: the block is still in its original location - assertTrue(mTestDir1.hasBlockMeta(BLOCK_ID1)); - assertFalse(mTestDir2.hasBlockMeta(BLOCK_ID1)); - assertTrue(mBlockStore.hasBlockMeta(BLOCK_ID1)); - assertTrue(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID1))); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir2, BLOCK_ID1))); - } - - @Test - public void moveBlockToSameLocationDoesNothing() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - // Move block to same location will simply do nothing, so the src block keeps where it was, - // and the available space should also remain unchanged. - long availableBytesBefore = mMetaManager.getAvailableBytes(mTestDir1.toBlockStoreLocation()); - mBlockStore.moveBlock(SESSION_ID1, BLOCK_ID1, - AllocateOptions.forMove(mTestDir1.toBlockStoreLocation())); - long availableBytesAfter = mMetaManager.getAvailableBytes(mTestDir1.toBlockStoreLocation()); - - assertEquals(availableBytesBefore, availableBytesAfter); - assertTrue(mTestDir1.hasBlockMeta(BLOCK_ID1)); - assertFalse(mMetaManager.hasTempBlockMeta(BLOCK_ID1)); - assertTrue(mBlockStore.hasBlockMeta(BLOCK_ID1)); - assertTrue(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID1))); - } - - /** - * Tests the {@link TieredBlockStore#removeBlock(long, long)} method. - */ - @Test - public void removeBlock() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - mBlockStore.removeBlock(SESSION_ID1, BLOCK_ID1); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID1)); - assertFalse(mBlockStore.hasBlockMeta(BLOCK_ID1)); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID1))); - - // Remove block from specific Dir - TieredBlockStoreTestUtils.cache2(SESSION_ID2, BLOCK_ID2, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - // Remove block from right Dir - mBlockStore.removeBlock(SESSION_ID2, BLOCK_ID2); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID2)); - assertFalse(mBlockStore.hasBlockMeta(BLOCK_ID2)); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID2))); - - // Remove block from the specific tier - TieredBlockStoreTestUtils.cache2(SESSION_ID2, BLOCK_ID2, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - mBlockStore.removeBlock(SESSION_ID2, BLOCK_ID2); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID2)); - assertFalse(mBlockStore.hasBlockMeta(BLOCK_ID2)); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID2))); - } - - @Test - public void freeSpace() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - mBlockStore.freeSpace(SESSION_ID1, mTestDir1.getCapacityBytes(), mTestDir1.getCapacityBytes(), - mTestDir1.toBlockStoreLocation()); - // Expect BLOCK_ID1 to be moved out of mTestDir1 - assertEquals(mTestDir1.getCapacityBytes(), mTestDir1.getAvailableBytes()); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID1)); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID1))); - } - - @Test - public void freeSpaceAhead() throws Exception { - long halfCapacityBytes = mTestDir1.getCapacityBytes() / 2; - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, halfCapacityBytes, mTestDir1, - mMetaManager, mBlockIterator); - TieredBlockStoreTestUtils.cache2(SESSION_ID2, BLOCK_ID2, halfCapacityBytes, mTestDir1, - mMetaManager, mBlockIterator); - mBlockStore.freeSpace(SESSION_ID1, halfCapacityBytes, 2 * halfCapacityBytes, - mTestDir1.toBlockStoreLocation()); - // Expect BLOCK_ID1 and BLOCK_ID2 to be moved out of mTestDir1 - assertEquals(mTestDir1.getCapacityBytes(), mTestDir1.getAvailableBytes()); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID1)); - assertFalse(mTestDir1.hasBlockMeta(BLOCK_ID2)); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID1))); - assertFalse(FileUtils.exists(DefaultBlockMeta.commitPath(mTestDir1, BLOCK_ID2))); - } - - @Test - public void freeSpaceThreadSafe() throws Exception { - int threadAmount = 10; - List runnables = new ArrayList<>(); - Evictor evictor = mock(Evictor.class); - when( - evictor.freeSpaceWithView(any(Long.class), any(BlockStoreLocation.class), - any(BlockMetadataEvictorView.class), any(Mode.class))) - .thenAnswer((InvocationOnMock invocation) -> { - CommonUtils.sleepMs(20); - return new EvictionPlan(new ArrayList<>(), new ArrayList<>()); - } - ); - for (int i = 0; i < threadAmount; i++) { - runnables.add(() -> { - try { - mBlockStore.freeSpace(SESSION_ID1, 0, 0, - new BlockStoreLocation(Constants.MEDIUM_MEM, 0)); - } catch (Exception e) { - fail(); - } - }); - } - RetryPolicy retry = new CountingRetry(threadAmount); - while (retry.attempt()) { - ConcurrencyUtils.assertConcurrent(runnables, threadAmount); - } - } - - @Test - public void freeSpaceWithPinnedBlocks() throws Exception { - // create a pinned block - TieredBlockStoreTestUtils.cache(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mBlockStore, - mTestDir1.toBlockStoreLocation(), true); - - // Expect an empty eviction plan is feasible - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_EVICTION_PLAN_TO_FREE_SPACE.getMessage( - mTestDir1.getCapacityBytes(), mTestDir1.toBlockStoreLocation().tierAlias())); - mBlockStore.freeSpace(SESSION_ID1, mTestDir1.getCapacityBytes(), mTestDir1.getCapacityBytes(), - mTestDir1.toBlockStoreLocation()); - } - - @Test - public void requestSpace() throws Exception { - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, 1, mTestDir1); - mBlockStore.requestSpace(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE - 1); - assertTrue(mTestDir1.hasTempBlockMeta(TEMP_BLOCK_ID)); - assertEquals(BLOCK_SIZE, mTestDir1.getTempBlockMeta(TEMP_BLOCK_ID).get().getBlockSize()); - assertEquals(mTestDir1.getCapacityBytes() - BLOCK_SIZE, mTestDir1.getAvailableBytes()); - } - - @Test - public void requestSpaceFailedWhenStorageFull() throws Exception { - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, 1, mTestDir1); - // Fill up the target dir - TieredBlockStoreTestUtils.cache(SESSION_ID1, BLOCK_ID1, mTestDir1.getAvailableBytes(), - mBlockStore, mTestDir1.toBlockStoreLocation(), true); - TieredBlockStoreTestUtils.cache(SESSION_ID2, BLOCK_ID2, mTestDir2.getAvailableBytes(), - mBlockStore, mTestDir2.toBlockStoreLocation(), true); - Assert.assertThrows(ResourceExhaustedRuntimeException.class, - () -> mBlockStore.requestSpace(SESSION_ID1, TEMP_BLOCK_ID, 1)); - } - - @Test - public void requestSpaceFailed() throws Exception { - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, - mTestDir1.getAvailableBytes(), mTestDir1); - Assert.assertThrows(ResourceExhaustedRuntimeException.class, - () -> mBlockStore.requestSpace(SESSION_ID1, TEMP_BLOCK_ID, 1)); - } - - @Test - public void createBlockMetaWithoutEviction() { - TempBlockMeta tempBlockMeta = mBlockStore.createBlock(SESSION_ID1, TEMP_BLOCK_ID, - AllocateOptions.forCreate(1, mTestDir1.toBlockStoreLocation())); - assertEquals(1, tempBlockMeta.getBlockSize()); - assertEquals(mTestDir1, tempBlockMeta.getParentDir()); - } - - @Test - public void createBlockMetaWithMediumType() { - BlockStoreLocation loc = BlockStoreLocation.anyDirInAnyTierWithMedium(Constants.MEDIUM_MEM); - TempBlockMeta tempBlockMeta = mBlockStore.createBlock(SESSION_ID1, TEMP_BLOCK_ID, - AllocateOptions.forCreate(1, loc)); - assertEquals(1, tempBlockMeta.getBlockSize()); - assertEquals(mTestDir2, tempBlockMeta.getParentDir()); - - BlockStoreLocation loc2 = BlockStoreLocation.anyDirInAnyTierWithMedium(Constants.MEDIUM_SSD); - TempBlockMeta tempBlockMeta2 = mBlockStore.createBlock(SESSION_ID1, TEMP_BLOCK_ID2, - AllocateOptions.forCreate(1, loc2)); - assertEquals(1, tempBlockMeta2.getBlockSize()); - assertEquals(mTestDir4, tempBlockMeta2.getParentDir()); - } - - /** - * Tests that when creating a block, if the space of the target location is currently taken by - * another block being locked, this creation will happen on the next available dir. - */ - @Test - public void createBlockMetaWithBlockLocked() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - - // session1 locks a block first - BlockLock lock = mBlockStore.pinBlock(SESSION_ID1, BLOCK_ID1).get(); - - // Create file that in dir1 that won't fit. - TieredBlockStoreTestUtils.cache(SESSION_ID2, TEMP_BLOCK_ID, mTestDir1.getCapacityBytes(), - mBlockStore, mTestDir1.toBlockStoreLocation(), false); - - // unlock the original block. - lock.close(); - - assertEquals(mTestDir1.getCapacityBytes(), mTestDir2.getCommittedBytes()); - } - - @Test - public void relaxLocationWrites() throws Exception { - long totalCapacityBytes = 0; - for (StorageDir dir : new StorageDir[] {mTestDir1, mTestDir2, mTestDir3, mTestDir4}) { - totalCapacityBytes += dir.getCapacityBytes(); - } - - long fileSizeBytes = 1000; - int maxFileCount = (int) (totalCapacityBytes / fileSizeBytes); - - // Write to first dir, 2 times of the whole store's peak capacity. - for (int i = 0; i <= maxFileCount * 2; i++) { - TieredBlockStoreTestUtils.cache(i, i, fileSizeBytes, mBlockStore, - mTestDir1.toBlockStoreLocation(), false); - } - } - - @Test - public void stickyLocationWrites() throws Exception { - long fileSizeBytes = 100; - int maxFileCount = (int) (mTestDir1.getCapacityBytes() / fileSizeBytes); - AllocateOptions remainOnLocOptions = AllocateOptions - .forCreate(fileSizeBytes, mTestDir1.toBlockStoreLocation()).setForceLocation(true); - - // Write to first dir, 2 times of its peak capacity. - for (int i = 0; i <= maxFileCount * 2; i++) { - TieredBlockStoreTestUtils.cache(i, i, remainOnLocOptions, mBlockStore, false); - } - - assertEquals(0, mTestDir1.getAvailableBytes()); - assertEquals(0, mTestDir2.getCommittedBytes()); - assertEquals(0, mTestDir3.getCommittedBytes()); - assertEquals(0, mTestDir4.getCommittedBytes()); - } - - /** - * Tests that when moving a block from src location to dst, if the space of the dst location is - * currently taken by another block being locked, this move operation will fail until the lock - * released. - */ - @Test - public void moveBlockMetaWithBlockLocked() throws Exception { - // Setup the src dir containing the block to move - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - // Setup the dst dir whose space is totally taken by another block - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID2, mTestDir2.getCapacityBytes(), - mTestDir2, mMetaManager, mBlockIterator); - - // session1 locks block2 first - BlockLock lock = mBlockStore.pinBlock(SESSION_ID1, BLOCK_ID2).get(); - - // Expect an exception because no eviction plan is feasible - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_SPACE_FOR_BLOCK_MOVE - .getMessage(mTestDir2.toBlockStoreLocation(), BLOCK_ID1)); - - mBlockStore.moveBlock(SESSION_ID1, BLOCK_ID1, - AllocateOptions.forMove(mTestDir2.toBlockStoreLocation())); - - // Expect createBlockMeta to succeed after unlocking this block. - lock.close(); - mBlockStore.moveBlock(SESSION_ID1, BLOCK_ID1, - AllocateOptions.forMove(mTestDir2.toBlockStoreLocation())); - - assertEquals(mTestDir1.getCapacityBytes(), mTestDir1.getAvailableBytes()); - assertEquals(mTestDir2.getCapacityBytes() - BLOCK_SIZE, mTestDir2.getAvailableBytes()); - } - - @Test - public void moveBlockWithReservedSpace() throws Exception { - Configuration.reloadProperties(); - final long RESERVE_SIZE = BLOCK_SIZE; - init(RESERVE_SIZE); - - // Setup the src dir containing the block to move - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - // Setup the dst dir whose space is totally taken by another block - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID2, - mTestDir2.getCapacityBytes() - BLOCK_SIZE, mTestDir2, mMetaManager, mBlockIterator); - assertEquals(0, mTestDir2.getAvailableBytes()); - assertTrue(mTestDir2.getCapacityBytes() > mTestDir2.getCommittedBytes()); - - // Expect an exception because destination is full. - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_SPACE_FOR_BLOCK_MOVE - .getMessage(mTestDir2.toBlockStoreLocation(), BLOCK_ID1)); - - AllocateOptions moveOptions = AllocateOptions.forMove(mTestDir2.toBlockStoreLocation()) - .setForceLocation(true) // Stay in destination. - .setEvictionAllowed(false); // No eviction. - - mBlockStore.moveBlock(SESSION_ID1, BLOCK_ID1, moveOptions); - - // Expect createBlockMeta to succeed after setting 'useReservedSpace' flag for allocation. - mBlockStore.moveBlock(SESSION_ID1, BLOCK_ID1, moveOptions.setUseReservedSpace(true)); - - // Validate source is empty. - assertEquals(mTestDir1.getCapacityBytes(), - mTestDir1.getAvailableBytes() + mTestDir1.getReservedBytes()); - // Validate destination is full. - assertEquals(mTestDir2.getCapacityBytes(), mTestDir2.getCommittedBytes()); - } - - /** - * Tests that when free the space of a location, if the space of the target location is currently - * taken by another block being locked, this freeSpace operation will fail until the lock - * released. - */ - @Test - public void freeSpaceWithBlockLocked() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - - // session1 locks a block first - BlockLock lock = mBlockStore.pinBlock(SESSION_ID1, BLOCK_ID1).get(); - - // Expect an empty eviction plan is feasible - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_EVICTION_PLAN_TO_FREE_SPACE.getMessage( - mTestDir1.getCapacityBytes(), mTestDir1.toBlockStoreLocation().tierAlias())); - mBlockStore.freeSpace(SESSION_ID1, mTestDir1.getCapacityBytes(), mTestDir1.getCapacityBytes(), - mTestDir1.toBlockStoreLocation()); - - // Expect freeSpace to succeed after unlock this block. - lock.close(); - mBlockStore.freeSpace(SESSION_ID1, mTestDir1.getCapacityBytes(), mTestDir1.getCapacityBytes(), - mTestDir1.toBlockStoreLocation()); - assertEquals(mTestDir1.getCapacityBytes(), mTestDir1.getAvailableBytes()); - } - - @Test - public void getBlockWriterForNonExistingBlock() { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.TEMP_BLOCK_META_NOT_FOUND.getMessage(BLOCK_ID1)); - mBlockStore.createBlockWriter(SESSION_ID1, BLOCK_ID1); - } - - @Test - public void abortNonExistingBlock() { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.TEMP_BLOCK_META_NOT_FOUND.getMessage(BLOCK_ID1)); - mBlockStore.abortBlock(SESSION_ID1, BLOCK_ID1); - } - - @Test - public void abortBlockNotOwnedBySessionId() throws Exception { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.BLOCK_ID_FOR_DIFFERENT_SESSION.getMessage(TEMP_BLOCK_ID, - SESSION_ID1, SESSION_ID2)); - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE, mTestDir1); - mBlockStore.abortBlock(SESSION_ID2, TEMP_BLOCK_ID); - } - - @Test - public void abortCommittedBlock() throws Exception { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.TEMP_BLOCK_ID_COMMITTED.getMessage(TEMP_BLOCK_ID)); - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE, mTestDir1); - mBlockStore.commitBlock(SESSION_ID1, TEMP_BLOCK_ID, false); - mBlockStore.abortBlock(SESSION_ID1, TEMP_BLOCK_ID); - } - - @Test - public void moveNonExistingBlock() throws Exception { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.BLOCK_META_NOT_FOUND.getMessage(BLOCK_ID1)); - mBlockStore.moveBlock(SESSION_ID1, BLOCK_ID1, - AllocateOptions.forMove(mTestDir1.toBlockStoreLocation())); - } - - @Test - public void moveTempBlock() throws Exception { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.BLOCK_META_NOT_FOUND.getMessage(TEMP_BLOCK_ID)); - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE, mTestDir1); - mBlockStore.moveBlock(SESSION_ID1, TEMP_BLOCK_ID, - AllocateOptions.forMove(mTestDir2.toBlockStoreLocation())); - } - - @Test - public void cacheSameBlockInDifferentDirs() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.ADD_EXISTING_BLOCK.getMessage(BLOCK_ID1, - FIRST_TIER_ALIAS)); - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir2, mMetaManager, - mBlockIterator); - } - - @Test - public void cacheSameBlockInDifferentTiers() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.ADD_EXISTING_BLOCK.getMessage(BLOCK_ID1, - FIRST_TIER_ALIAS)); - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir3, mMetaManager, - mBlockIterator); - } - - @Test - public void commitBlockTwice() throws Exception { - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE, mTestDir1); - assertFalse(mBlockStore.hasBlockMeta(TEMP_BLOCK_ID)); - assertTrue(mBlockStore.hasTempBlockMeta(TEMP_BLOCK_ID)); - mBlockStore.commitBlock(SESSION_ID1, TEMP_BLOCK_ID , false); - assertTrue(mBlockStore.hasBlockMeta(TEMP_BLOCK_ID)); - assertFalse(mBlockStore.hasTempBlockMeta(TEMP_BLOCK_ID)); - mBlockStore.commitBlock(SESSION_ID1, TEMP_BLOCK_ID, false); - assertTrue(mBlockStore.hasBlockMeta(TEMP_BLOCK_ID)); - assertFalse(mBlockStore.hasTempBlockMeta(TEMP_BLOCK_ID)); - } - - @Test - public void commitNonExistingBlock() throws Exception { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.TEMP_BLOCK_META_NOT_FOUND.getMessage(BLOCK_ID1)); - - mBlockStore.commitBlock(SESSION_ID1, BLOCK_ID1, false); - } - - @Test - public void commitBlockNotOwnedBySessionId() throws Exception { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.BLOCK_ID_FOR_DIFFERENT_SESSION.getMessage(TEMP_BLOCK_ID, - SESSION_ID1, SESSION_ID2)); - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE, mTestDir1); - mBlockStore.commitBlock(SESSION_ID2, TEMP_BLOCK_ID, false); - } - - @Test - public void removeTempBlock() throws Exception { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.REMOVE_UNCOMMITTED_BLOCK.getMessage(TEMP_BLOCK_ID)); - TieredBlockStoreTestUtils.createTempBlock(SESSION_ID1, TEMP_BLOCK_ID, BLOCK_SIZE, mTestDir1); - mBlockStore.removeBlock(SESSION_ID1, TEMP_BLOCK_ID); - } - - @Test - public void checkStorageFailedWhenInaccessible() throws Exception { - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID1, BLOCK_SIZE, mTestDir1, mMetaManager, - mBlockIterator); - TieredBlockStoreTestUtils.cache2(SESSION_ID1, BLOCK_ID2, BLOCK_SIZE, mTestDir2, mMetaManager, - mBlockIterator); - BlockStoreMeta oldMeta = mBlockStore.getBlockStoreMeta(); - FileUtils.deletePathRecursively(mTestDir2.getDirPath()); - mBlockStore.removeInaccessibleStorage(); - BlockStoreMeta meta = mBlockStore.getBlockStoreMetaFull(); - long usedByteInDir = mTestDir2.getCapacityBytes() - mTestDir2.getAvailableBytes(); - assertFalse("failed storage path should be removed", - meta.getDirectoryPathsOnTiers().get(FIRST_TIER_ALIAS).contains(mTestDir2.getDirPath())); - assertEquals("failed storage path quota should be deducted from store capacity", - oldMeta.getCapacityBytes() - mTestDir2.getCapacityBytes(), meta.getCapacityBytes()); - assertEquals("failed storage path used bytes should be deducted from store used bytes", - oldMeta.getUsedBytes() - usedByteInDir, - meta.getUsedBytes()); - assertEquals("failed storage path quota should be deducted from tier capacity", - oldMeta.getCapacityBytesOnTiers().get(FIRST_TIER_ALIAS) - mTestDir2.getCapacityBytes(), - (long) meta.getCapacityBytesOnTiers().get(FIRST_TIER_ALIAS)); - assertEquals("failed storage path used bytes should be deducted from tier used bytes", - oldMeta.getUsedBytesOnTiers().get(FIRST_TIER_ALIAS) - usedByteInDir, - (long) meta.getUsedBytesOnTiers().get(FIRST_TIER_ALIAS)); - assertFalse("blocks in failed storage path should be removed", - meta.getBlockList().get(FIRST_TIER_ALIAS).contains(BLOCK_ID2)); - assertTrue("blocks in working storage path should be retained", - meta.getBlockList().get(FIRST_TIER_ALIAS).contains(BLOCK_ID1)); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/TieredBlockStoreTestUtils.java b/core/server/worker/src/test/java/alluxio/worker/block/TieredBlockStoreTestUtils.java deleted file mode 100644 index 15ef31e70f3b..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/TieredBlockStoreTestUtils.java +++ /dev/null @@ -1,416 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static com.google.common.collect.ImmutableList.toImmutableList; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.io.BufferUtils; -import alluxio.util.io.FileUtils; -import alluxio.util.io.PathUtils; -import alluxio.worker.block.annotator.BlockIterator; -import alluxio.worker.block.evictor.Evictor; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.io.LocalFileBlockWriter; -import alluxio.worker.block.meta.DefaultTempBlockMeta; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.TempBlockMeta; - -import com.google.common.base.Preconditions; -import com.google.common.primitives.Ints; - -import java.util.Arrays; -import java.util.Collections; - -/** - * Utility methods for setting and testing {@link TieredBlockStore}. - */ -public final class TieredBlockStoreTestUtils { - /** - * Default configurations of a TieredBlockStore for use in {@link #defaultMetadataManager}. They - * represent a block store with a MEM tier and a SSD tier, there are two directories with capacity - * 2000 bytes and 3000 bytes separately in the MEM tier and three directories with capacity 10000, - * 20000, 30000 bytes separately in the SSD tier. - */ - public static final int[] TIER_ORDINAL = {0, 1}; - public static final String[] TIER_ALIAS = {Constants.MEDIUM_MEM, Constants.MEDIUM_SSD}; - public static final String[][] TIER_PATH = {{"/mem/0", "/mem/1"}, {"/ssd/0", "/ssd/1", "/ssd/2"}}; - public static final long[][] TIER_CAPACITY_BYTES = {{2000, 3000}, {10000, 20000, 30000}}; - public static final String[][] TIER_MEDIA_TYPES = - {{Constants.MEDIUM_MEM, Constants.MEDIUM_MEM}, - {Constants.MEDIUM_SSD, Constants.MEDIUM_SSD, Constants.MEDIUM_SSD}}; - public static final String WORKER_DATA_FOLDER = "/alluxioworker/"; - - /** - * Sets up a {@link Configuration} for a {@link TieredBlockStore} with several tiers - * configured by the parameters. For simplicity, you can use {@link #setupDefaultConf(String)} - * which calls this method with default values. - * - * @param baseDir the directory path as prefix for all the paths of directories in the tiered - * storage; when specified, the directory needs to exist before calling this method - * @param tierOrdinal like {@link #TIER_ORDINAL}, length must be > 0 - * @param tierAlias like {@link #TIER_ALIAS}, each corresponds to an element in tierLevel - * @param tierPath like {@link #TIER_PATH}, each list represents directories of the tier with the - * same list index in tierAlias - * @param tierCapacity like {@link #TIER_CAPACITY_BYTES}, should be in the same dimension with - * tierPath, each element is the capacity of the corresponding dir in tierPath - * @param tierMediumType should be in the same dimension with tierPath, each element is the - * medium type of the corresponding dir in tierPath - * @param workerDataFolder when specified it sets up the alluxio.worker.data.folder property - */ - public static void setupConfWithMultiTier(String baseDir, int[] tierOrdinal, String[] tierAlias, - String[][] tierPath, long[][] tierCapacity, String[][] tierMediumType, - String workerDataFolder) throws Exception { - // make sure dimensions are legal - Preconditions.checkNotNull(tierOrdinal, "tierOrdinal"); - Preconditions.checkNotNull(tierAlias, "tierAlias"); - Preconditions.checkNotNull(tierPath, "tierPath"); - Preconditions.checkNotNull(tierCapacity, "tierCapacity"); - Preconditions.checkNotNull(tierMediumType, "tierMediumType"); - - Preconditions.checkArgument(tierOrdinal.length > 0, "length of tierLevel should be > 0"); - Preconditions.checkArgument(tierOrdinal.length == tierAlias.length, - "tierAlias and tierLevel should have the same length"); - Preconditions.checkArgument(tierOrdinal.length == tierPath.length, - "tierPath and tierLevel should have the same length"); - Preconditions.checkArgument(tierOrdinal.length == tierCapacity.length, - "tierCapacity and tierLevel should have the same length"); - Preconditions.checkArgument(tierOrdinal.length == tierMediumType.length, - "tierMediumType and tierLevel should have the same length"); - int nTier = tierOrdinal.length; - - tierPath = createDirHierarchy(baseDir, tierPath); - if (workerDataFolder != null) { - Configuration.set(PropertyKey.WORKER_DATA_FOLDER, workerDataFolder); - } - Configuration.set(PropertyKey.WORKER_TIERED_STORE_LEVELS, nTier); - - // sets up each tier in turn - for (int i = 0; i < nTier; i++) { - setupConfTier(tierOrdinal[i], tierAlias[i], tierPath[i], tierCapacity[i], tierMediumType[i]); - } - } - - /** - * Sets up a {@link Configuration} for a {@link TieredBlockStore} with only *one tier* - * configured by the parameters. For simplicity, you can use {@link #setupDefaultConf(String)} - * which sets up the tierBlockStore with default values. - * - * This method modifies the configuration, so be sure to reset it when done. - * - * @param baseDir the directory path as prefix for all the paths of directories in the tiered - * storage; when specified, the directory needs to exist before calling this method - * @param tierOrdinal ordinal of this tier - * @param tierAlias alias of this tier - * @param tierPath path of this tier; when `baseDir` is specified, the actual test tierPath turns - * into `baseDir/tierPath` - * @param tierCapacity capacity of this tier - * @param workerDataFolder when specified it sets up the alluxio.worker.data.folder property - */ - public static void setupConfWithSingleTier(String baseDir, int tierOrdinal, String tierAlias, - String[] tierPath, long[] tierCapacity, String[] tierMedia, String workerDataFolder) - throws Exception { - if (baseDir != null) { - tierPath = createDirHierarchy(baseDir, tierPath); - } - if (workerDataFolder != null) { - Configuration.set(PropertyKey.WORKER_DATA_FOLDER, workerDataFolder); - } - Configuration.set(PropertyKey.WORKER_TIERED_STORE_LEVELS, 1); - setupConfTier(tierOrdinal, tierAlias, tierPath, tierCapacity, tierMedia); - } - - /** - * Sets up a specific tier's {@link Configuration} for a {@link TieredBlockStore}. - * - * @param tierAlias alias of the tier - * @param tierPath absolute path of the tier - * @param tierCapacity capacity of the tier - */ - private static void setupConfTier(int ordinal, String tierAlias, String[] tierPath, - long[] tierCapacity, String[] tierMediumType) { - Preconditions.checkNotNull(tierPath, "tierPath"); - Preconditions.checkNotNull(tierCapacity, "tierCapacity"); - Preconditions.checkNotNull(tierMediumType, "tierMediumType"); - Preconditions.checkArgument(tierPath.length == tierCapacity.length, - "tierPath and tierCapacity should have the same length"); - - Configuration - .set(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS.format(ordinal), - tierAlias); - - Configuration - .set(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(ordinal), - Arrays.stream(tierPath).collect(toImmutableList())); - - Configuration - .set(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA.format(ordinal), - Arrays.stream(tierCapacity).boxed().map(String::valueOf).collect(toImmutableList())); - - Configuration - .set(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_MEDIUMTYPE.format(ordinal), - Arrays.stream(tierMediumType).collect(toImmutableList())); - } - - /** - * Joins baseDir with all the paths listed in the array and then create the new generated path. - * - * @param baseDir the directory path as prefix for all the paths in the array 'dirs' - * @param dirs 2-D array of directory paths - * @return new joined and created paths array - */ - private static String[][] createDirHierarchy(String baseDir, final String[][] dirs) - throws Exception { - if (baseDir == null) { - return dirs; - } - String[][] newDirs = new String[dirs.length][]; - for (int i = 0; i < dirs.length; i++) { - newDirs[i] = createDirHierarchy(baseDir, dirs[i]); - } - return newDirs; - } - - /** - * Joins baseDir with all the paths listed in the array and then create the new generated path. - * - * @param baseDir the directory path as prefix for all the paths in the array 'dirs' - * @param dirs 1-D array of directory paths - * @return new joined and created paths array - */ - private static String[] createDirHierarchy(String baseDir, final String[] dirs) throws Exception { - if (baseDir == null) { - return dirs; - } - String[] newDirs = new String[dirs.length]; - for (int i = 0; i < dirs.length; i++) { - newDirs[i] = PathUtils.concatPath(baseDir, dirs[i]); - FileUtils.createDir(newDirs[i]); - } - return newDirs; - } - - /** - * Creates a BlockMetadataManager with {@link #setupDefaultConf(String)}. - * - * @param baseDir the directory path as prefix for paths of directories in the tiered storage; the - * directory needs to exist before calling this method - * @return the created metadata manager - */ - public static BlockMetadataManager defaultMetadataManager(String baseDir) throws Exception { - setupDefaultConf(baseDir); - return BlockMetadataManager.createBlockMetadataManager(); - } - - /** - * Creates a {@link BlockMetadataEvictorView} with {@link #setupDefaultConf(String)}. - * - * @param baseDir the directory path as prefix for paths of directories in the tiered storage; the - * directory needs to exist before calling this method - * @return the created metadata evictor view - */ - public static BlockMetadataEvictorView defaultMetadataManagerView(String baseDir) - throws Exception { - BlockMetadataManager metaManager = TieredBlockStoreTestUtils.defaultMetadataManager(baseDir); - return new BlockMetadataEvictorView(metaManager, Collections.emptySet(), - Collections.emptySet()); - } - - /** - * Sets up a {@link Configuration} with default values of {@link #TIER_ORDINAL}, - * {@link #TIER_ALIAS}, {@link #TIER_PATH} with the baseDir as path prefix, - * {@link #TIER_CAPACITY_BYTES}. - * - * @param baseDir the directory path as prefix for paths of directories in the tiered storage; the - * directory needs to exist before calling this method - */ - public static void setupDefaultConf(String baseDir) throws Exception { - setupConfWithMultiTier(baseDir, TIER_ORDINAL, TIER_ALIAS, TIER_PATH, TIER_CAPACITY_BYTES, - TIER_MEDIA_TYPES, WORKER_DATA_FOLDER); - } - - /** - * Caches bytes into {@link StorageDir}. - * - * @param sessionId session who caches the data - * @param blockId id of the cached block - * @param bytes size of the block in bytes - * @param dir the {@link StorageDir} the block resides in - * @param meta the metadata manager to update meta of the block - * @param evictor the evictor to be informed of the new block - */ - public static void cache(long sessionId, long blockId, long bytes, StorageDir dir, - BlockMetadataManager meta, Evictor evictor) throws Exception { - BlockStoreEventListener listener = null; - if (evictor instanceof BlockStoreEventListener) { - listener = (BlockStoreEventListener) evictor; - } - cache2(sessionId, blockId, bytes, dir, meta, listener); - } - - /** - * Caches bytes into {@link LocalBlockStore} at specific location. - * - * @param sessionId session who caches the data - * @param blockId id of the cached block - * @param bytes size of the block in bytes - * @param blockStore block store that the block is written into - * @param location the location where the block resides - * @param pinOnCreate whether to pin block on create - */ - public static void cache(long sessionId, long blockId, long bytes, LocalBlockStore blockStore, - BlockStoreLocation location, boolean pinOnCreate) throws Exception { - TempBlockMeta tempBlockMeta = blockStore.createBlock(sessionId, blockId, - AllocateOptions.forCreate(bytes, location)); - // write data - BlockWriter writer = new LocalFileBlockWriter(tempBlockMeta.getPath()); - writer.append(BufferUtils.getIncreasingByteBuffer(Ints.checkedCast(bytes))); - writer.close(); - - // commit block - blockStore.commitBlock(sessionId, blockId, pinOnCreate); - } - - /** - * Caches bytes into {@link LocalBlockStore} at specific location. - * - * @param sessionId session who caches the data - * @param blockId id of the cached block - * @param options allocation options - * @param blockStore block store that the block is written into - * @param pinOnCreate whether to pin block on create - */ - public static void cache(long sessionId, long blockId, AllocateOptions options, - LocalBlockStore blockStore, boolean pinOnCreate) throws Exception { - TempBlockMeta tempBlockMeta = blockStore.createBlock(sessionId, blockId, options); - // write data - BlockWriter writer = new LocalFileBlockWriter(tempBlockMeta.getPath()); - writer.append(BufferUtils.getIncreasingByteBuffer(Ints.checkedCast(options.getSize()))); - writer.close(); - - // commit block - blockStore.commitBlock(sessionId, blockId, pinOnCreate); - } - - /** - * Caches bytes into {@link StorageDir}. - * - * @param sessionId session who caches the data - * @param blockId id of the cached block - * @param bytes size of the block in bytes - * @param tierLevel tier level of the {@link StorageDir} the block resides in - * @param dirIndex index of directory in the tierLevel the block resides in - * @param meta the metadata manager to update meta of the block - * @param evictor the evictor to be informed of the new block - */ - public static void cache(long sessionId, long blockId, long bytes, int tierLevel, int dirIndex, - BlockMetadataManager meta, Evictor evictor) throws Exception { - StorageDir dir = meta.getTiers().get(tierLevel).getDir(dirIndex); - cache(sessionId, blockId, bytes, dir, meta, evictor); - } - - /** - * Caches bytes into {@link StorageDir}. - * - * @param sessionId session who caches the data - * @param blockId id of the cached block - * @param bytes size of the block in bytes - * @param dir the {@link StorageDir} the block resides in - * @param meta the metadata manager to update meta of the block - * @param iterator the iterator to be informed of the new block - */ - public static void cache2(long sessionId, long blockId, long bytes, StorageDir dir, - BlockMetadataManager meta, BlockIterator iterator) throws Exception { - cache2(sessionId, blockId, bytes, dir, meta, (BlockStoreEventListener) null); - if (iterator != null) { - for (BlockStoreEventListener listener : iterator.getListeners()) { - listener.onCommitBlock(blockId, dir.toBlockStoreLocation()); - } - } - } - - /** - * Caches bytes into {@link StorageDir}. - * - * @param sessionId session who caches the data - * @param blockId id of the cached block - * @param bytes size of the block in bytes - * @param dir the {@link StorageDir} the block resides in - * @param meta the metadata manager to update meta of the block - * @param listener the listener to be informed of the new block - */ - public static void cache2(long sessionId, long blockId, long bytes, StorageDir dir, - BlockMetadataManager meta, BlockStoreEventListener listener) throws Exception { - TempBlockMeta tempBlockMeta = createTempBlock(sessionId, blockId, bytes, dir); - - // commit block. - FileUtils.move(tempBlockMeta.getPath(), tempBlockMeta.getCommitPath()); - meta.commitTempBlockMeta(tempBlockMeta); - - // update iterator if a listener. - if (listener != null) { - listener.onCommitBlock(blockId, dir.toBlockStoreLocation()); - } - } - - /** - * Makes a temp block of a given size in {@link StorageDir}. - * - * @param sessionId session who caches the data - * @param blockId id of the cached block - * @param bytes size of the block in bytes - * @param dir the {@link StorageDir} the block resides in - * @return the temp block meta - */ - public static TempBlockMeta createTempBlock(long sessionId, long blockId, long bytes, - StorageDir dir) throws Exception { - // prepare temp block - TempBlockMeta tempBlockMeta = new DefaultTempBlockMeta(sessionId, blockId, bytes, dir); - dir.addTempBlockMeta(tempBlockMeta); - // write data - FileUtils.createFile(tempBlockMeta.getPath()); - BlockWriter writer = new LocalFileBlockWriter(tempBlockMeta.getPath()); - writer.append(BufferUtils.getIncreasingByteBuffer(Ints.checkedCast(bytes))); - writer.close(); - return tempBlockMeta; - } - - /** - * Gets the total capacity of all tiers in bytes. - * - * @return total capacity of all tiers in bytes - */ - public static long getDefaultTotalCapacityBytes() { - long totalCapacity = 0; - for (long[] tierCapacityBytes : TIER_CAPACITY_BYTES) { - for (long tierCapacityByte : tierCapacityBytes) { - totalCapacity += tierCapacityByte; - } - } - return totalCapacity; - } - - /** - * Gets the number of testing directories of all tiers. - * - * @return number of testing directories of all tiers - */ - public static long getDefaultDirNum() { - int dirNum = 0; - for (String[] tierPath : TIER_PATH) { - dirNum += tierPath.length; - } - return dirNum; - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/UnderFileSystemBlockReaderTest.java b/core/server/worker/src/test/java/alluxio/worker/block/UnderFileSystemBlockReaderTest.java deleted file mode 100644 index 718e9b950b4a..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/UnderFileSystemBlockReaderTest.java +++ /dev/null @@ -1,261 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.spy; - -import alluxio.AlluxioTestDirectory; -import alluxio.AlluxioURI; -import alluxio.ConfigurationRule; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.metrics.MetricInfo; -import alluxio.metrics.MetricKey; -import alluxio.metrics.MetricsSystem; -import alluxio.proto.dataserver.Protocol; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UfsManager.UfsClient; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.UnderFileSystemConfiguration; -import alluxio.util.io.BufferUtils; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.meta.UnderFileSystemBlockMeta; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Optional; - -public final class UnderFileSystemBlockReaderTest { - private static final long TEST_BLOCK_SIZE = 1024; - private static final long SESSION_ID = 1; - private static final long BLOCK_ID = 2; - - private UnderFileSystemBlockReader mReader; - private LocalBlockStore mAlluxioBlockStore; - private UnderFileSystemBlockMeta mUnderFileSystemBlockMeta; - private UfsManager.UfsClient mUfsClient; - private UfsInputStreamCache mUfsInstreamCache; - private Protocol.OpenUfsBlockOptions mOpenUfsBlockOptions; - private Counter mUfsBytesRead; - private Meter mUfsBytesReadThroughput; - - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - @Rule - public ConfigurationRule mConfigurationRule = - new ConfigurationRule(new HashMap() { - { - put(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, AlluxioTestDirectory - .createTemporaryDirectory("UnderFileSystemBlockReaderTest-RootUfs") - .getAbsolutePath()); - // ensure tiered storage uses different tmp dir for each test case - put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, AlluxioTestDirectory - .createTemporaryDirectory("UnderFileSystemBlockReaderTest-WorkerDataFolder") - .getAbsolutePath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVELS, 1); - } - }, Configuration.modifiableGlobal()); - - @Before - public void before() throws Exception { - String ufsFolder = Configuration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); - String testFilePath = File.createTempFile("temp", null, new File(ufsFolder)).getAbsolutePath(); - byte[] buffer = BufferUtils.getIncreasingByteArray((int) TEST_BLOCK_SIZE * 2); - BufferUtils.writeBufferToFile(testFilePath, buffer); - - mAlluxioBlockStore = new TieredBlockStore(); - mUfsInstreamCache = new UfsInputStreamCache(); - mUfsClient = new UfsClient( - () -> UnderFileSystem.Factory.create(testFilePath, - UnderFileSystemConfiguration.defaults(Configuration.global())), - new AlluxioURI(testFilePath)); - - mOpenUfsBlockOptions = Protocol.OpenUfsBlockOptions.newBuilder().setMaxUfsReadConcurrency(10) - .setBlockSize(TEST_BLOCK_SIZE).setOffsetInFile(TEST_BLOCK_SIZE).setUfsPath(testFilePath) - .build(); - mUnderFileSystemBlockMeta = - new UnderFileSystemBlockMeta(SESSION_ID, BLOCK_ID, mOpenUfsBlockOptions); - mUfsBytesRead = MetricsSystem.counterWithTags( - MetricKey.WORKER_BYTES_READ_UFS.getName(), - MetricKey.WORKER_BYTES_READ_UFS.isClusterAggregated(), - MetricInfo.TAG_UFS, MetricsSystem.escape(mUfsClient.getUfsMountPointUri())); - mUfsBytesReadThroughput = MetricsSystem.meterWithTags( - MetricKey.WORKER_BYTES_READ_UFS_THROUGHPUT.getName(), - MetricKey.WORKER_BYTES_READ_UFS_THROUGHPUT.isClusterAggregated(), - MetricInfo.TAG_UFS, - MetricsSystem.escape(mUfsClient.getUfsMountPointUri())); - } - - private void checkTempBlock(long start, long length) throws Exception { - Assert.assertTrue(mAlluxioBlockStore.hasTempBlockMeta(BLOCK_ID)); - mAlluxioBlockStore.commitBlock(SESSION_ID, BLOCK_ID, false); - Optional lock = mAlluxioBlockStore.pinBlock(SESSION_ID, BLOCK_ID); - BlockReader reader = mAlluxioBlockStore.createBlockReader(SESSION_ID, BLOCK_ID, 0); - Assert.assertEquals(length, reader.getLength()); - ByteBuffer buffer = reader.read(0, length); - assertTrue(BufferUtils.equalIncreasingByteBuffer((int) start, (int) length, buffer)); - reader.close(); - lock.get().close(); - } - - @Test - public void readFullBlock() throws Exception { - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - mAlluxioBlockStore, mUfsClient, mUfsInstreamCache, mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuffer buffer = mReader.read(0, TEST_BLOCK_SIZE); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE, buffer)); - mReader.close(); - checkTempBlock(0, TEST_BLOCK_SIZE); - } - - @Test - public void readPartialBlock() throws Exception { - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - mAlluxioBlockStore, mUfsClient, mUfsInstreamCache, mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuffer buffer = mReader.read(0, TEST_BLOCK_SIZE - 1); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE - 1, buffer)); - mReader.close(); - // partial block should not be cached - assertFalse(mAlluxioBlockStore.hasTempBlockMeta(BLOCK_ID)); - } - - @Test - public void offset() throws Exception { - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - mAlluxioBlockStore, mUfsClient, mUfsInstreamCache, mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuffer buffer = mReader.read(2, TEST_BLOCK_SIZE - 2); - assertTrue(BufferUtils - .equalIncreasingByteBuffer(2, (int) TEST_BLOCK_SIZE - 2, buffer)); - mReader.close(); - // partial block should not be cached - assertFalse(mAlluxioBlockStore.hasTempBlockMeta(BLOCK_ID)); - } - - @Test - public void readOverlap() throws Exception { - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 2, false, - mAlluxioBlockStore, mUfsClient, mUfsInstreamCache, mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuffer buffer = mReader.read(2, TEST_BLOCK_SIZE - 2); - assertTrue(BufferUtils.equalIncreasingByteBuffer(2, (int) TEST_BLOCK_SIZE - 2, buffer)); - buffer = mReader.read(0, TEST_BLOCK_SIZE - 2); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE - 2, buffer)); - buffer = mReader.read(3, TEST_BLOCK_SIZE); - assertTrue(BufferUtils.equalIncreasingByteBuffer(3, (int) TEST_BLOCK_SIZE - 3, buffer)); - mReader.close(); - // block should be cached as two reads covers the full block - checkTempBlock(0, TEST_BLOCK_SIZE); - } - - @Test - public void readFullBlockNoCache() throws Exception { - mUnderFileSystemBlockMeta = new UnderFileSystemBlockMeta(SESSION_ID, BLOCK_ID, - mOpenUfsBlockOptions.toBuilder().setNoCache(true).build()); - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - mAlluxioBlockStore, mUfsClient, mUfsInstreamCache, mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuffer buffer = mReader.read(0, TEST_BLOCK_SIZE); - // read should succeed even if error is thrown when caching - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE, buffer)); - mReader.close(); - assertFalse(mAlluxioBlockStore.hasTempBlockMeta(BLOCK_ID)); - } - - @Test - public void readFullBlockRequestSpaceError() throws Exception { - LocalBlockStore errorThrowingBlockStore = spy(mAlluxioBlockStore); - doThrow(new ResourceExhaustedRuntimeException("Ignored", false)) - .when(errorThrowingBlockStore) - .requestSpace(anyLong(), anyLong(), anyLong()); - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - errorThrowingBlockStore, mUfsClient, mUfsInstreamCache, - mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuffer buffer = mReader.read(0, TEST_BLOCK_SIZE); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE, buffer)); - mReader.close(); - assertFalse(mAlluxioBlockStore.hasTempBlockMeta(BLOCK_ID)); - } - - @Test - public void readFullBlockRequestCreateBlockError() throws Exception { - LocalBlockStore errorThrowingBlockStore = spy(mAlluxioBlockStore); - doThrow(new ResourceExhaustedRuntimeException("Ignored", false)).when(errorThrowingBlockStore) - .createBlock(anyLong(), anyLong(), any(AllocateOptions.class)); - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - errorThrowingBlockStore, mUfsClient, mUfsInstreamCache, - mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuffer buffer = mReader.read(0, TEST_BLOCK_SIZE); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE, buffer)); - mReader.close(); - assertFalse(mAlluxioBlockStore.hasTempBlockMeta(BLOCK_ID)); - } - - @Test - public void transferFullBlock() throws Exception { - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - mAlluxioBlockStore, mUfsClient, mUfsInstreamCache, mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuf buf = - PooledByteBufAllocator.DEFAULT.buffer((int) TEST_BLOCK_SIZE * 2, (int) TEST_BLOCK_SIZE * 2); - try { - while (buf.writableBytes() > 0 && mReader.transferTo(buf) != -1) { - } - assertTrue(BufferUtils - .equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE, buf.nioBuffer())); - mReader.close(); - } finally { - buf.release(); - } - checkTempBlock(0, TEST_BLOCK_SIZE); - } - - @Test - public void transferPartialBlock() throws Exception { - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - mAlluxioBlockStore, mUfsClient, mUfsInstreamCache, mUfsBytesRead, mUfsBytesReadThroughput); - ByteBuf buf = - PooledByteBufAllocator.DEFAULT.buffer((int) TEST_BLOCK_SIZE / 2, (int) TEST_BLOCK_SIZE / 2); - try { - while (buf.writableBytes() > 0 && mReader.transferTo(buf) != -1) { - } - assertTrue(BufferUtils - .equalIncreasingByteBuffer(0, (int) TEST_BLOCK_SIZE / 2, buf.nioBuffer())); - mReader.close(); - } finally { - buf.release(); - } - // partial block should not be cached - assertFalse(mAlluxioBlockStore.hasTempBlockMeta(BLOCK_ID)); - } - - @Test - public void getLocation() throws Exception { - mReader = UnderFileSystemBlockReader.create(mUnderFileSystemBlockMeta, 0, false, - mAlluxioBlockStore, mUfsClient, mUfsInstreamCache, mUfsBytesRead, mUfsBytesReadThroughput); - assertTrue(mReader.getLocation().startsWith(mOpenUfsBlockOptions.getUfsPath())); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/UnderFileSystemBlockStoreTest.java b/core/server/worker/src/test/java/alluxio/worker/block/UnderFileSystemBlockStoreTest.java deleted file mode 100644 index 658c3110e7f2..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/UnderFileSystemBlockStoreTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import alluxio.AlluxioURI; -import alluxio.conf.Configuration; -import alluxio.exception.BlockAlreadyExistsException; -import alluxio.exception.runtime.NotFoundRuntimeException; -import alluxio.master.NoopUfsManager; -import alluxio.proto.dataserver.Protocol; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystemConfiguration; -import alluxio.worker.block.io.BlockReader; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.util.Arrays; - -public final class UnderFileSystemBlockStoreTest { - private static final long TEST_BLOCK_SIZE = 1024; - private static final long BLOCK_ID = 2; - private static final long MOUNT_ID = 1; - - private LocalBlockStore mAlluxioBlockStore; - private Protocol.OpenUfsBlockOptions mOpenUfsBlockOptions; - private UfsManager mUfsManager; - - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - private File mTempFile; - private String mTempFilePath; - - @Before - public void before() throws Exception { - mTempFile = mFolder.newFile(); - mTempFilePath = mTempFile.getAbsolutePath(); - - mAlluxioBlockStore = Mockito.mock(LocalBlockStore.class); - - // make a mock UfsManager that returns a client connecting - // to our local file - mUfsManager = new NoopUfsManager(); - mUfsManager.addMount( - MOUNT_ID, - new AlluxioURI(mTempFilePath), - UnderFileSystemConfiguration.defaults(Configuration.global())); - - mOpenUfsBlockOptions = Protocol.OpenUfsBlockOptions.newBuilder().setMaxUfsReadConcurrency(5) - .setBlockSize(TEST_BLOCK_SIZE).setOffsetInFile(TEST_BLOCK_SIZE) - .setUfsPath(mTempFilePath).setMountId(MOUNT_ID).build(); - } - - @Test - public void acquireAccess() throws Exception { - UnderFileSystemBlockStore blockStore = - new UnderFileSystemBlockStore(mAlluxioBlockStore, mUfsManager); - for (int i = 0; i < 5; i++) { - assertTrue(blockStore.acquireAccess(i + 1, BLOCK_ID, mOpenUfsBlockOptions)); - } - } - - @Test - public void acquireDuplicateAccess() throws Exception { - UnderFileSystemBlockStore blockStore = - new UnderFileSystemBlockStore(mAlluxioBlockStore, mUfsManager); - int sessionId = 1; - assertTrue(blockStore.acquireAccess(sessionId, BLOCK_ID, mOpenUfsBlockOptions)); - assertThrows( - BlockAlreadyExistsException.class, - () -> blockStore.acquireAccess(sessionId, BLOCK_ID, mOpenUfsBlockOptions)); - } - - @Test - public void releaseAccess() throws Exception { - UnderFileSystemBlockStore blockStore = - new UnderFileSystemBlockStore(mAlluxioBlockStore, mUfsManager); - for (int i = 0; i < 5; i++) { - assertTrue(blockStore.acquireAccess(i + 1, BLOCK_ID, mOpenUfsBlockOptions)); - blockStore.releaseAccess(i + 1, BLOCK_ID); - } - - // release blocks that are already released should not throw an exception - blockStore.releaseAccess(1, BLOCK_ID); - - assertTrue(blockStore.acquireAccess(6, BLOCK_ID, mOpenUfsBlockOptions)); - } - - @Test - public void createBlockReader() throws Exception { - // flush some data to test file - byte[] data = new byte[(int) TEST_BLOCK_SIZE]; - Arrays.fill(data, (byte) 1); - - try (FileOutputStream fileOut = new FileOutputStream(mTempFile); - BufferedOutputStream bufOut = new BufferedOutputStream(fileOut)) { - bufOut.write(data, 0, data.length); - bufOut.flush(); - } - - // create block metastore that accesses the test file - long sessionId = 1L; - UnderFileSystemBlockStore blockStore = - new UnderFileSystemBlockStore(mAlluxioBlockStore, mUfsManager); - Protocol.OpenUfsBlockOptions options = Protocol.OpenUfsBlockOptions - .newBuilder() - .setBlockSize(TEST_BLOCK_SIZE) - .setMountId(MOUNT_ID) - .setOffsetInFile(0) - .setMaxUfsReadConcurrency(1) - .setUfsPath(mTempFilePath) - .build(); - - BlockReader reader = blockStore - .createBlockReader(sessionId, BLOCK_ID, 0, false, options); - - assertArrayEquals(data, reader.read(0, TEST_BLOCK_SIZE).array()); - } - - @Test - public void isNoCache() throws Exception { - UnderFileSystemBlockStore blockStore = - new UnderFileSystemBlockStore(mAlluxioBlockStore, mUfsManager); - long sessionId = 1L; - long blockId = 1L; - Protocol.OpenUfsBlockOptions options = - mOpenUfsBlockOptions.toBuilder().setNoCache(true).build(); - - // we have not acquired this block yet - assertThrows( - NotFoundRuntimeException.class, - () -> blockStore.isNoCache(sessionId, blockId)); - - blockStore.acquireAccess(sessionId, blockId, options); - - // this should success - assertTrue(blockStore.isNoCache(sessionId, blockId)); - } - - @Test - public void cleanUpSession() throws Exception { - UnderFileSystemBlockStore blockStore = - new UnderFileSystemBlockStore(mAlluxioBlockStore, mUfsManager); - long sessionId = 1L; - long blockId = 1L; - - blockStore.acquireAccess(sessionId, blockId, mOpenUfsBlockOptions); - - // before clean up, the session-block key is stored - // and cannot be re-acquired - assertThrows( - BlockAlreadyExistsException.class, - () -> blockStore.acquireAccess(sessionId, blockId, mOpenUfsBlockOptions) - ); - - blockStore.cleanupSession(sessionId); - - // now session is cleaned up, and we can acquire session-block keys - blockStore.acquireAccess(sessionId, blockId, mOpenUfsBlockOptions); - } - - @Test - public void closeReader() throws Exception { - // create block metastore that accesses the test file - long sessionId = 1L; - UnderFileSystemBlockStore blockStore = - new UnderFileSystemBlockStore(mAlluxioBlockStore, mUfsManager); - Protocol.OpenUfsBlockOptions options = Protocol.OpenUfsBlockOptions - .newBuilder() - .setBlockSize(TEST_BLOCK_SIZE) - .setMountId(MOUNT_ID) - .setOffsetInFile(0) - .setMaxUfsReadConcurrency(1) - .setUfsPath(mTempFilePath) - .build(); - - BlockReader reader = blockStore - .createBlockReader(sessionId, BLOCK_ID, 0, false, options); - - blockStore.closeBlock(sessionId, BLOCK_ID); - assertTrue(reader.isClosed()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorContractTest.java b/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorContractTest.java deleted file mode 100644 index cb7bdb5698d8..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorContractTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import static org.junit.Assert.fail; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.meta.StorageTier; - -import com.google.common.reflect.ClassPath; -import com.google.common.reflect.Reflection; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * This is the class to test the "contract" of different kinds of allocators, - * i.e., the general properties the allocators need to follow. - */ -public final class AllocatorContractTest extends AllocatorTestBase { - protected List mStrategies; - - /** - * Try to find all implementation classes of {@link Allocator} in the same package. - */ - @Before - @Override - public void before() throws Exception { - super.before(); - mStrategies = new ArrayList<>(); - try { - String packageName = Reflection.getPackageName(Allocator.class); - ClassPath path = ClassPath.from(Thread.currentThread().getContextClassLoader()); - List clazzInPackage = - new ArrayList<>(path.getTopLevelClassesRecursive(packageName)); - for (ClassPath.ClassInfo clazz : clazzInPackage) { - Set> interfaces = - new HashSet<>(Arrays.asList(clazz.load().getInterfaces())); - if (interfaces.size() > 0 && interfaces.contains(Allocator.class)) { - mStrategies.add(clazz.getName()); - } - } - } catch (Exception e) { - fail("Failed to find implementation of allocate strategy"); - } - } - - /** - * Tests that no allocation happens when the RAM, SSD and HDD size is more than the default one. - */ - @Test - public void shouldNotAllocate() throws Exception { - for (String strategyName : mStrategies) { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, strategyName); - resetManagerView(); - Allocator allocator = Allocator.Factory.create(getMetadataEvictorView()); - assertTempBlockMeta(allocator, mAnyDirInTierLoc1, DEFAULT_RAM_SIZE + 1, false); - assertTempBlockMeta(allocator, mAnyDirInTierLoc2, DEFAULT_SSD_SIZE + 1, false); - assertTempBlockMeta(allocator, mAnyDirInTierLoc3, DEFAULT_HDD_SIZE + 1, false); - assertTempBlockMeta(allocator, mAnyTierLoc, DEFAULT_HDD_SIZE + 1, false); - assertTempBlockMeta(allocator, mAnyTierLoc, DEFAULT_SSD_SIZE + 1, true); - } - } - - /** - * Tests that allocation happens when the RAM, SSD and HDD size is lower than the default size. - */ - @Test - public void shouldAllocate() throws Exception { - for (String strategyName : mStrategies) { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, strategyName); - resetManagerView(); - Allocator tierAllocator = Allocator.Factory.create(getMetadataEvictorView()); - for (int i = 0; i < DEFAULT_RAM_NUM; i++) { - assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc1, DEFAULT_RAM_SIZE - 1, true); - } - for (int i = 0; i < DEFAULT_SSD_NUM; i++) { - assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc2, DEFAULT_SSD_SIZE - 1, true); - } - for (int i = 0; i < DEFAULT_HDD_NUM; i++) { - assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc3, DEFAULT_HDD_SIZE - 1, true); - } - - resetManagerView(); - Allocator anyAllocator = Allocator.Factory.create(getMetadataEvictorView()); - for (int i = 0; i < DEFAULT_RAM_NUM; i++) { - assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_RAM_SIZE - 1, true); - } - for (int i = 0; i < DEFAULT_SSD_NUM; i++) { - assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_SSD_SIZE - 1, true); - } - for (int i = 0; i < DEFAULT_HDD_NUM; i++) { - assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_HDD_SIZE - 1, true); - } - } - } - - @Test - public void allocateAfterDirDeletion() throws Exception { - for (String strategyName : mStrategies) { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, strategyName); - resetManagerView(); - for (int i = 0; i < TIER_ALIAS.length; i++) { - StorageTier tier = mManager.getTier(TIER_ALIAS[i]); - tier.removeStorageDir(tier.getDir(0)); - } - Allocator tierAllocator = Allocator.Factory.create(getMetadataEvictorView()); - for (int i = 0; i < DEFAULT_RAM_NUM - 1; i++) { - assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc1, DEFAULT_RAM_SIZE - 1, true); - } - for (int i = 0; i < DEFAULT_SSD_NUM - 1; i++) { - assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc2, DEFAULT_SSD_SIZE - 1, true); - } - for (int i = 0; i < DEFAULT_HDD_NUM - 1; i++) { - assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc3, DEFAULT_HDD_SIZE - 1, true); - } - - resetManagerView(); - Allocator anyAllocator = Allocator.Factory.create(getMetadataEvictorView()); - for (int i = 0; i < DEFAULT_RAM_NUM - 1; i++) { - assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_RAM_SIZE - 1, true); - } - for (int i = 0; i < DEFAULT_SSD_NUM - 1; i++) { - assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_SSD_SIZE - 1, true); - } - for (int i = 0; i < DEFAULT_HDD_NUM - 1; i++) { - assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_HDD_SIZE - 1, true); - } - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorFactoryTest.java b/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorFactoryTest.java deleted file mode 100644 index 8ddbac24c72d..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorFactoryTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import static org.junit.Assert.assertTrue; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataView; -import alluxio.worker.block.TieredBlockStoreTestUtils; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -/** - * Test {@link Allocator.Factory} by passing different allocate strategy class names with alluxio - * conf and test if it generates the correct {@link Allocator} instance. - */ -public final class AllocatorFactoryTest { - private BlockMetadataEvictorView mMetadataView; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - String baseDir = mTestFolder.newFolder().getAbsolutePath(); - mMetadataView = TieredBlockStoreTestUtils.defaultMetadataManagerView(baseDir); - } - - @After - public void after() { - Configuration.reloadProperties(); - } - - /** - * Tests the creation of the {@link GreedyAllocator} via the - * {@link Allocator.Factory#create(BlockMetadataView)} method. - */ - @Test - public void createGreedyAllocator() { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, GreedyAllocator.class.getName()); - Allocator allocator = Allocator.Factory.create(mMetadataView); - assertTrue(allocator instanceof GreedyAllocator); - } - - /** - * Tests the creation of the {@link MaxFreeAllocator} via the - * {@link Allocator.Factory#create(BlockMetadataView)} method. - */ - @Test - public void createMaxFreeAllocator() { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, MaxFreeAllocator.class.getName()); - Allocator allocator = Allocator.Factory.create(mMetadataView); - assertTrue(allocator instanceof MaxFreeAllocator); - } - - /** - * Tests the creation of the {@link RoundRobinAllocator} via the - * {@link Allocator.Factory#create(BlockMetadataView)} method. - */ - @Test - public void createRoundRobinAllocator() { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, - RoundRobinAllocator.class.getName()); - Allocator allocator = Allocator.Factory.create(mMetadataView); - assertTrue(allocator instanceof RoundRobinAllocator); - } - - /** - * Tests the creation of the default allocator via the - * {@link Allocator.Factory#create(BlockMetadataView)} method. - */ - @Test - public void createDefaultAllocator() { - // Create a new instance of Alluxio configuration with original properties to test the default - // behavior of create. - Allocator allocator = Allocator.Factory.create(mMetadataView); - assertTrue(allocator instanceof MaxFreeAllocator); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorTestBase.java b/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorTestBase.java deleted file mode 100644 index facc6c2de280..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/allocator/AllocatorTestBase.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.TieredBlockStoreTestUtils; -import alluxio.worker.block.meta.DefaultBlockMeta; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageDirView; -import alluxio.worker.block.meta.StorageTier; -import alluxio.worker.block.meta.TempBlockMeta; -import alluxio.worker.block.reviewer.MockReviewer; - -import com.google.common.collect.Sets; -import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.util.HashSet; - -/** - * Base class for allocator tests. - */ -public class AllocatorTestBase { - - protected static final long SESSION_ID = 1; - protected int mTestBlockId = 0; - - // Default tier/dir configurations we use for testing - public static final long DEFAULT_RAM_SIZE = 1000; - public static final long DEFAULT_SSD_SIZE = 2000; - public static final long DEFAULT_HDD_SIZE = 3000; - - public static final String[] MEDIA_TYPES = - {Constants.MEDIUM_MEM, Constants.MEDIUM_SSD, Constants.MEDIUM_HDD}; - public static final int[] TIER_LEVEL = {0, 1, 2}; - public static final String[] TIER_ALIAS = - {Constants.MEDIUM_MEM, Constants.MEDIUM_SSD, Constants.MEDIUM_HDD}; - public static final String[][] TIER_PATH = {{"/ramdisk"}, {"/ssd1", "/ssd2"}, - {"/disk1", "/disk2", "/disk3"}}; - public static final String[][] TIER_MEDIA_TYPE = {{Constants.MEDIUM_MEM}, - {Constants.MEDIUM_SSD, Constants.MEDIUM_SSD}, - {Constants.MEDIUM_HDD, Constants.MEDIUM_HDD, Constants.MEDIUM_HDD}}; - public static final long[][] TIER_CAPACITY_BYTES = {{DEFAULT_RAM_SIZE}, - {DEFAULT_SSD_SIZE, DEFAULT_SSD_SIZE}, - {DEFAULT_HDD_SIZE, DEFAULT_HDD_SIZE, DEFAULT_HDD_SIZE}}; - - public static final int DEFAULT_RAM_NUM = TIER_PATH[0].length; - public static final int DEFAULT_SSD_NUM = TIER_PATH[1].length; - public static final int DEFAULT_HDD_NUM = TIER_PATH[2].length; - - protected BlockMetadataManager mManager = null; - protected Allocator mAllocator = null; - - protected BlockStoreLocation mAnyTierLoc = BlockStoreLocation.anyTier(); - protected BlockStoreLocation mAnyDirInTierLoc1 = - BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM); - protected BlockStoreLocation mAnyDirInTierLoc2 = - BlockStoreLocation.anyDirInTier(Constants.MEDIUM_SSD); - protected BlockStoreLocation mAnyDirInTierLoc3 = - BlockStoreLocation.anyDirInTier(Constants.MEDIUM_HDD); - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - resetManagerView(); - } - - /** - * Resets the manager view configured by the parameters in this base class. - */ - protected void resetManagerView() throws Exception { - String alluxioHome = mTestFolder.newFolder().getAbsolutePath(); - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED, false); - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED, false); - Configuration.set(PropertyKey.WORKER_REVIEWER_CLASS, - "alluxio.worker.block.reviewer.MockReviewer"); - // Reviewer will not reject by default. - MockReviewer.resetBytesToReject(Sets.newHashSet()); - TieredBlockStoreTestUtils.setupConfWithMultiTier(alluxioHome, TIER_LEVEL, TIER_ALIAS, - TIER_PATH, TIER_CAPACITY_BYTES, TIER_MEDIA_TYPE, null); - mManager = BlockMetadataManager.createBlockMetadataManager(); - } - - /** - * Given an allocator with the location and blockSize, we assert whether the block can be - * allocated. - * - * @param allocator the allocation manager of Alluxio managed data - * @param location the location in block store - * @param blockSize the size of block in bytes - * @param avail the block should be successfully allocated or not - */ - protected void assertTempBlockMeta(Allocator allocator, BlockStoreLocation location, - long blockSize, boolean avail) throws IOException { - - mTestBlockId++; - StorageDirView dirView = - allocator.allocateBlockWithView(blockSize, location, - getMetadataEvictorView(), true); - TempBlockMeta tempBlockMeta = - dirView == null ? null : dirView.createTempBlockMeta(SESSION_ID, mTestBlockId, blockSize); - - if (!avail) { - assertTrue(tempBlockMeta == null); - } else { - assertTrue(tempBlockMeta != null); - } - } - - /** - * Given an allocator with the location, blockSize, tierAlias and dirIndex, - * we assert whether the block can be allocated. - * - * @param allocator the allocation manager of Alluxio managed data - * @param location the location in block store - * @param blockSize the size of block in bytes - * @param avail the block should be successfully allocated or not - * @param tierAlias the block should be allocated at this tier - * @param dirIndex the block should be allocated at this dir - */ - protected void assertTempBlockMeta(Allocator allocator, BlockStoreLocation location, - int blockSize, boolean avail, String tierAlias, int dirIndex) throws Exception { - - mTestBlockId++; - - StorageDirView dirView = - allocator.allocateBlockWithView(blockSize, location, - getMetadataEvictorView(), false); - TempBlockMeta tempBlockMeta = - dirView == null ? null : dirView.createTempBlockMeta(SESSION_ID, mTestBlockId, blockSize); - - if (!avail) { - assertTrue(tempBlockMeta == null); - } else { - assertTrue(tempBlockMeta != null); - - StorageDir pDir = tempBlockMeta.getParentDir(); - StorageTier pTier = pDir.getParentTier(); - - assertEquals(dirIndex, pDir.getDirIndex()); - assertEquals(tierAlias, pTier.getTierAlias()); - - //update the dir meta info - pDir.addBlockMeta(new DefaultBlockMeta(mTestBlockId, blockSize, pDir)); - } - } - - protected BlockMetadataEvictorView getMetadataEvictorView() { - return new BlockMetadataEvictorView(mManager, new HashSet(), new HashSet()); - } - - /** - * For each tier, test if anyDirInTier location gives a dir in the target tier. - * */ - protected void assertAllocationAnyDirInTier() throws Exception { - BlockStoreLocation[] locations = new BlockStoreLocation[]{mAnyDirInTierLoc1, - mAnyDirInTierLoc2, mAnyDirInTierLoc3}; - for (int i = 0; i < locations.length; i++) { - StorageDirView dirView = - mAllocator.allocateBlockWithView(1, - locations[i], getMetadataEvictorView(), true); - assertNotNull(dirView); - assertEquals(TIER_ALIAS[i], dirView.getParentTierView().getTierViewAlias()); - } - } - - /** - * For each medium, test if anyDirInAnyTierWithMedium gives a dir with the target medium. - * */ - protected void assertAllocationAnyDirInAnyTierWithMedium() throws Exception { - for (String medium : MEDIA_TYPES) { - BlockStoreLocation loc = BlockStoreLocation.anyDirInAnyTierWithMedium(medium); - StorageDirView dirView = - mAllocator.allocateBlockWithView(1, loc, - getMetadataEvictorView(), true); - assertNotNull(dirView); - assertEquals(medium, dirView.getMediumType()); - } - } - - protected void assertAllocationInSpecificDir() throws Exception { - for (int i = 0; i < TIER_ALIAS.length; i++) { - String tier = TIER_ALIAS[i]; - for (int j = 0; j < TIER_PATH[i].length; j++) { - BlockStoreLocation loc = new BlockStoreLocation(tier, j); - StorageDirView dirView = - mAllocator.allocateBlockWithView(1, loc, - getMetadataEvictorView(), true); - assertNotNull(dirView); - assertEquals(tier, dirView.getParentTierView().getTierViewAlias()); - assertEquals(j, dirView.getDirViewIndex()); - } - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/allocator/GreedyAllocatorTest.java b/core/server/worker/src/test/java/alluxio/worker/block/allocator/GreedyAllocatorTest.java deleted file mode 100644 index 12cf01d102cd..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/allocator/GreedyAllocatorTest.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.reviewer.MockReviewer; - -import com.google.common.collect.Sets; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Unit tests for {@link GreedyAllocator}. - */ -public final class GreedyAllocatorTest extends AllocatorTestBase { - @Before - public void initialize() { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, GreedyAllocator.class.getName()); - mAllocator = Allocator.Factory.create(getMetadataEvictorView()); - } - - @After - public void reset() { - Configuration.reloadProperties(); - } - - /** - * Tests that blocks are allocated in the first storage directory which has enough free space. - */ - @Test - public void allocateBlock() throws Exception { - // - // idx | tier1 | tier2 | tier3 - // 0 1000 - // 0 ├───── 2000 - // 1 └───── 2000 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 500, true, Constants.MEDIUM_MEM, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 <--- alloc - // 0 ├───── 2000 - // 1 └───── 2000 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 1000, true, Constants.MEDIUM_SSD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 1000 <--- alloc - // 1 └───── 2000 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 1500, true, Constants.MEDIUM_SSD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 1000 - // 1 └───── 500 <--- alloc - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 1000, true, Constants.MEDIUM_SSD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 0 <--- alloc - // 1 └───── 500 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 1000, true, Constants.MEDIUM_HDD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 0 - // 1 └───── 500 - // 0 ├─── 2000 <--- alloc - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 2000, true, Constants.MEDIUM_HDD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 0 - // 1 └───── 500 - // 0 ├─── 0 <--- alloc - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 500, true, Constants.MEDIUM_MEM, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 0 <--- alloc - // 0 ├───── 0 - // 1 └───── 500 - // 0 ├─── 0 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 500, true, Constants.MEDIUM_SSD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 0 - // 0 ├───── 0 - // 1 └───── 0 <--- alloc - // 0 ├─── 0 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 1000, true, Constants.MEDIUM_HDD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 0 - // 0 ├───── 0 - // 1 └───── 500 - // 0 ├─── 0 - // 1 ├─── 2000 <--- alloc - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 700, true, Constants.MEDIUM_HDD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 0 - // 0 ├───── 0 - // 1 └───── 500 - // 0 ├─── 0 - // 1 ├─── 1300 <--- alloc - // 2 └─── 3000 - // - - /** Reviewer's opinion affects the test */ - MockReviewer.resetBytesToReject(Sets.newHashSet(500L)); - - assertTempBlockMeta(mAllocator, mAnyTierLoc, 100, true, "HDD", 1); - // - // idx | tier1 | tier2 | tier3 - // 0 0 - // 0 ├───── 0 - // 1 └───── 500 - // 0 ├─── 0 - // 1 ├─── 1200 <--- alloc - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 100, false, "", 0); - // - // idx | tier1 | tier2 | tier3 - // 0 0 - // 0 ├───── 0 - // 1 └───── 500 - // 0 ├─── 0 - // 1 ├─── 1200 - // 2 └─── 3000 - // - } - - @Test - public void testAnyDirInTier() throws Exception { - assertAllocationAnyDirInTier(); - } - - @Test - public void testAnyDirInAnyTierWithMedium() throws Exception { - assertAllocationAnyDirInAnyTierWithMedium(); - } - - @Test - public void testSpecificDir() throws Exception { - assertAllocationInSpecificDir(); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/allocator/MaxFreeAllocatorTest.java b/core/server/worker/src/test/java/alluxio/worker/block/allocator/MaxFreeAllocatorTest.java deleted file mode 100644 index 6ba4c9609ba5..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/allocator/MaxFreeAllocatorTest.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.reviewer.MockReviewer; - -import com.google.common.collect.Sets; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Unit tests for {@link MaxFreeAllocator}. - */ -public final class MaxFreeAllocatorTest extends AllocatorTestBase { - @Before - public void initialize() { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, MaxFreeAllocator.class.getName()); - mAllocator = Allocator.Factory.create(getMetadataEvictorView()); - } - - @After - public void reset() { - Configuration.reloadProperties(); - } - - /** - * Tests that blocks are allocated in the storage directory with the most available free space. - */ - @Test - public void allocateBlock() throws Exception { - // - // idx | tier1 | tier2 | tier3 - // 0 1000 - // 0 ├───── 2000 - // 1 └───── 2000 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 1500, true, Constants.MEDIUM_SSD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 1000 - // 0 ├───── 500 <--- alloc - // 1 └───── 2000 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 2000, true, Constants.MEDIUM_SSD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 1000 - // 0 ├───── 500 - // 1 └───── 0 <--- alloc - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 300, true, Constants.MEDIUM_MEM, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 700 <--- alloc - // 0 ├───── 500 - // 1 └───── 0 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 300, true, Constants.MEDIUM_SSD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 700 - // 0 ├───── 200 <--- alloc - // 1 └───── 0 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - - /** Reviewer's opinion affects the test */ - MockReviewer.resetBytesToReject(Sets.newHashSet(700L, 2700L, 3000L)); - assertTempBlockMeta(mAllocator, mAnyTierLoc, 300, false, "HDD", 0); - // - // idx | tier1 | tier2 | tier3 - // 0 700 - // 0 ├───── 200 - // 1 └───── 0 - // 0 ├─── 3000 <--- alloc (reviewer rejects) - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 300, false, "HDD", 1); - // - // idx | tier1 | tier2 | tier3 - // 0 700 - // 0 ├───── 200 - // 1 └───── 0 - // 0 ├─── 3000 - // 1 ├─── 3000 <--- alloc (reviewer rejects) - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc1, 300, false, "", 0); - // - // idx | tier1 | tier2 | tier3 - // 0 700 <--- alloc (reviewer rejects) - // 0 ├───── 200 - // 1 └───── 0 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - } - - @Test - public void testAnyDirInTier() throws Exception { - assertAllocationAnyDirInTier(); - } - - @Test - public void testAnyDirInAnyTierWithMedium() throws Exception { - assertAllocationAnyDirInAnyTierWithMedium(); - } - - @Test - public void testSpecificDir() throws Exception { - assertAllocationInSpecificDir(); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/allocator/RoundRobinAllocatorTest.java b/core/server/worker/src/test/java/alluxio/worker/block/allocator/RoundRobinAllocatorTest.java deleted file mode 100644 index ae11b751bfd2..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/allocator/RoundRobinAllocatorTest.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.allocator; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.reviewer.MockReviewer; - -import com.google.common.collect.Sets; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Unit tests for {@link RoundRobinAllocator}. - */ -public final class RoundRobinAllocatorTest extends AllocatorTestBase { - @Before - public void initialize() { - Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, - RoundRobinAllocator.class.getName()); - mAllocator = Allocator.Factory.create(getMetadataEvictorView()); - } - - @After - public void reset() { - Configuration.reloadProperties(); - } - - /** - * Tests that blocks are allocated in a round robin fashion. - */ - @Test - public void allocateBlock() throws Exception { - // - // idx | tier1 | tier2 | tier3 - // 0 1000 - // 0 ├───── 2000 - // 1 └───── 2000 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 500, true, Constants.MEDIUM_MEM, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 <--- alloc - // 0 ├───── 2000 - // 1 └───── 2000 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 600, true, Constants.MEDIUM_SSD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 1400 <--- alloc - // 1 └───── 2000 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 700, true, Constants.MEDIUM_SSD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 1400 - // 1 └───── 1300 <--- alloc - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 700, true, Constants.MEDIUM_SSD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 700 <--- alloc - // 1 └───── 1300 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 1000, true, Constants.MEDIUM_SSD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 700 - // 1 └───── 300 <--- alloc - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 700, true, Constants.MEDIUM_SSD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 0 <--- alloc - // 1 └───── 300 - // 0 ├─── 3000 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyTierLoc, 700, true, Constants.MEDIUM_HDD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 500 - // 0 ├───── 0 - // 1 └───── 300 - // 0 ├─── 2300 <--- alloc - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc1, 200, true, Constants.MEDIUM_MEM, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 300 <--- alloc - // 0 ├───── 0 - // 1 └───── 300 - // 0 ├─── 2300 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc1, 100, true, Constants.MEDIUM_MEM, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 200 <--- alloc - // 0 ├───── 0 - // 1 └───── 300 - // 0 ├─── 2300 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc1, 700, false, "", 0); - // - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 300 - // 0 ├─── 2300 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 100, true, Constants.MEDIUM_SSD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 200 <--- alloc - // 0 ├─── 2300 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 100, true, Constants.MEDIUM_SSD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 100 <--- alloc - // 0 ├─── 2300 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 1500, false, "", 0); - // - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 100 - // 0 ├─── 2300 - // 1 ├─── 3000 - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 2000, true, Constants.MEDIUM_HDD, 1); - // - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 100 - // 0 ├─── 2300 - // 1 ├─── 1000 <--- alloc - // 2 └─── 3000 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 3000, true, Constants.MEDIUM_HDD, 2); - // - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 100 - // 0 ├─── 2300 - // 1 ├─── 1000 - // 2 └─── 0 <--- alloc - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 500, true, Constants.MEDIUM_HDD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 100 - // 0 ├─── 1800 <--- alloc - // 1 ├─── 1000 - // 2 └─── 0 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 2000, false, Constants.MEDIUM_HDD, 0); - // - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 100 - // 0 ├─── 1800 - // 1 ├─── 1000 - // 2 └─── 0 - // - // tier 3, dir 0, remain 0 - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 300, true, Constants.MEDIUM_HDD, 1); - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 100 - // 0 ├─── 1800 - // 1 ├─── 700 <--- alloc - // 2 └─── 0 - // - - /** Reviewer's opinion affects the test */ - MockReviewer.resetBytesToReject(Sets.newHashSet(200L)); - - assertTempBlockMeta(mAllocator, mAnyTierLoc, 50, true, "SSD", 1); - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 50 <--- alloc - // 0 ├─── 1800 - // 1 ├─── 700 - // 2 └─── 0 - // - assertTempBlockMeta(mAllocator, mAnyDirInTierLoc1, 50, false, "", 0); - // idx | tier1 | tier2 | tier3 - // 0 200 - // 0 ├───── 0 - // 1 └───── 50 - // 0 ├─── 1800 - // 1 ├─── 700 - // 2 └─── 0 - // - } - - @Test - public void testAnyDirInTier() throws Exception { - assertAllocationAnyDirInTier(); - } - - @Test - public void testAnyDirInAnyTierWithMedium() throws Exception { - assertAllocationAnyDirInAnyTierWithMedium(); - } - - @Test - public void testSpecificDir() throws Exception { - assertAllocationInSpecificDir(); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/annotator/AbstractBlockAnnotatorTest.java b/core/server/worker/src/test/java/alluxio/worker/block/annotator/AbstractBlockAnnotatorTest.java deleted file mode 100644 index 5ffa7c6b71f5..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/annotator/AbstractBlockAnnotatorTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.StorageTierAssoc; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreEventListener; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.TieredBlockStoreTestUtils; -import alluxio.worker.block.meta.StorageDir; - -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; - -public abstract class AbstractBlockAnnotatorTest { - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - private HashMap mBlockLocation; - private Long mUserSession = 1L; - - protected BlockMetadataManager mMetaManager; - protected BlockIterator mBlockIterator; - protected BlockStoreEventListener mBlockEventListener; - protected StorageTierAssoc mTierAssoc; - - /** - * Sets up all dependencies before a test runs. - */ - public void init() throws Exception { - File tempFolder = mTestFolder.newFolder(); - mMetaManager = TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath()); - mBlockIterator = mMetaManager.getBlockIterator(); - mBlockEventListener = mBlockIterator.getListeners().get(0); - mTierAssoc = mMetaManager.getStorageTierAssoc(); - mBlockLocation = new HashMap<>(); - } - - protected void createBlock(long blockId, StorageDir dir) throws Exception { - TieredBlockStoreTestUtils.cache2(mUserSession++, blockId, 1, dir, mMetaManager, mBlockIterator); - mBlockLocation.put(blockId, dir); - } - - protected void moveBlock(long blockId, StorageDir destDir) throws Exception { - mMetaManager.removeBlockMeta(mMetaManager.getBlockMeta(blockId).get()); - TieredBlockStoreTestUtils.cache2(mUserSession++, blockId, 1, destDir, mMetaManager, - (BlockIterator) null); - mBlockEventListener.onMoveBlockByWorker(blockId, - mBlockLocation.get(blockId).toBlockStoreLocation(), destDir.toBlockStoreLocation()); - mBlockLocation.put(blockId, destDir); - } - - protected void removeBlock(long blockId) throws Exception { - mMetaManager.removeBlockMeta(mMetaManager.getBlockMeta(blockId).get()); - mBlockEventListener.onRemoveBlock(blockId, - mBlockLocation.remove(blockId).toBlockStoreLocation()); - } - - protected void accessBlock(long blockId) throws Exception { - mBlockEventListener.onAccessBlock(blockId, - mBlockLocation.get(blockId).toBlockStoreLocation()); - } - - protected StorageDir getDir(int tierIndex, int dirIndex) { - return mMetaManager.getDir(new BlockStoreLocation(mTierAssoc.getAlias(tierIndex), dirIndex)); - } - - protected void validateIterator(Iterator iterator, Iterator expected) { - while (true) { - Assert.assertFalse(iterator.hasNext() ^ expected.hasNext()); - if (iterator.hasNext()) { - Assert.assertEquals(expected.next(), iterator.next()); - } else { - break; - } - } - } - - // Common tests for {@link EvictionOrderProvider} implementations. - - @Test - public void testRemovedBlock() throws Exception { - StorageDir dir = getDir(0, 0); - createBlock(0, dir); - List expectedList = new ArrayList() { - { - add(0L); - } - }; - validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.NATURAL), - expectedList.iterator()); - - removeBlock(0); - expectedList.clear(); - validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.NATURAL), - expectedList.iterator()); - } - - @Test - public void testMovedBlock() throws Exception { - StorageDir srcDir = getDir(0, 0); - StorageDir dstDir = getDir(0, 1); - createBlock(0, srcDir); - List expectedList = new ArrayList() { - { - add(0L); - } - }; - validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.NATURAL), - expectedList.iterator()); - - moveBlock(0, dstDir); - validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.NATURAL), - expectedList.iterator()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/annotator/EmulatingBlockIteratorTest.java b/core/server/worker/src/test/java/alluxio/worker/block/annotator/EmulatingBlockIteratorTest.java deleted file mode 100644 index 8e2b9ea02399..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/annotator/EmulatingBlockIteratorTest.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.StorageTierAssoc; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreEventListener; -import alluxio.worker.block.TieredBlockStoreTestUtils; -import alluxio.worker.block.evictor.LRUEvictor; -import alluxio.worker.block.meta.StorageDir; -import alluxio.worker.block.meta.StorageTier; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class EmulatingBlockIteratorTest { - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - private HashMap mBlockLocation; - private Long mUserSession = 1L; - - protected BlockMetadataManager mMetaManager; - protected BlockIterator mBlockIterator; - protected BlockStoreEventListener mBlockEventListener; - protected StorageTierAssoc mTierAssoc; - - /** - * Sets up base class for LRUAnnotator. - */ - @Before - public void before() throws Exception { - // Set an evictor class in order to activate emulation. - Configuration.set(PropertyKey.WORKER_EVICTOR_CLASS, LRUEvictor.class.getName()); - // No reserved bytes for precise capacity planning. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED, false); - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED, false); - - File tempFolder = mTestFolder.newFolder(); - mMetaManager = TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath()); - mBlockIterator = mMetaManager.getBlockIterator(); - mBlockEventListener = mBlockIterator.getListeners().get(0); - mTierAssoc = mMetaManager.getStorageTierAssoc(); - mBlockLocation = new HashMap<>(); - } - - @Test - public void testLRUEmulation() throws Exception { - // Block size that is common denominator to each dir capacity. - final long blockSize = 1000; - // Fill all directories and calculate expected block order for each of them as per LRU. - Map> expectedListPerDir = new HashMap<>(); - long blockId = 0; - for (StorageTier tier : mMetaManager.getTiers()) { - for (StorageDir dir : tier.getStorageDirs()) { - expectedListPerDir.put(dir, new LinkedList<>()); - while (dir.getAvailableBytes() > 0) { - createBlock(++blockId, blockSize, dir); - accessBlock(blockId); - expectedListPerDir.get(dir).add(blockId); - } - } - } - - // Validate that emulation iterator yields the same order as expected. - // Iterators not validated at full length as evictor implementations can stop - // after it has fulfilled the requested available bytes. - // That's also why emulation can't be tested reliably with 'Reverse' iteration order. - for (StorageDir dir : expectedListPerDir.keySet()) { - validateIteratorHeader( - mBlockIterator.getIterator(dir.toBlockStoreLocation(), BlockOrder.NATURAL), - expectedListPerDir.get(dir).iterator()); - } - } - - private void createBlock(long blockId, long blockSize, StorageDir dir) throws Exception { - TieredBlockStoreTestUtils.cache2(mUserSession++, blockId, blockSize, dir, mMetaManager, - mBlockIterator); - mBlockLocation.put(blockId, dir); - } - - private void accessBlock(long blockId) throws Exception { - mBlockEventListener.onAccessBlock(blockId); - mBlockEventListener.onAccessBlock(blockId, - mBlockLocation.get(blockId).toBlockStoreLocation()); - } - - /** - * Validates one of the iterator contains the other in the beginning. - */ - private void validateIteratorHeader(Iterator iterator, Iterator expected) { - // Validate initial state is the same. - Assert.assertFalse(iterator.hasNext() ^ expected.hasNext()); - while (true) { - // Quit validation if any of them finished. - if (!iterator.hasNext() || !expected.hasNext()) { - break; - } - // Validate header is still the same. - Assert.assertEquals(expected.next(), iterator.next()); - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/annotator/LRFUAnnotatorTest.java b/core/server/worker/src/test/java/alluxio/worker/block/annotator/LRFUAnnotatorTest.java deleted file mode 100644 index 91e9bf7e922b..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/annotator/LRFUAnnotatorTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.meta.StorageDir; - -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Random; - -public class LRFUAnnotatorTest extends AbstractBlockAnnotatorTest { - /** - * Sets up base class for LRUAnnotator. - */ - @Before - public void before() throws Exception { - Configuration.set(PropertyKey.WORKER_BLOCK_ANNOTATOR_CLASS, - LRFUAnnotator.class.getName()); - // To make it behave close to an absolute LFU. - Configuration.set(PropertyKey.WORKER_BLOCK_ANNOTATOR_LRFU_STEP_FACTOR, 0); - init(); - } - - @Test - public void testLRFU() throws Exception { - Random rand = new Random(); - List expectedList = new ArrayList<>(); - for (long i = 0; i < 100; i++) { - StorageDir pickedDir = getDir(rand.nextInt(2), rand.nextInt(2)); - createBlock(i, pickedDir); - for (int j = 0; j < i * 2 + 1; j++) { - accessBlock(i); - } - expectedList.add(i); - } - - validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.NATURAL), - expectedList.iterator()); - - Collections.reverse(expectedList); - validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.REVERSE), - expectedList.iterator()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/annotator/LRUAnnotatorTest.java b/core/server/worker/src/test/java/alluxio/worker/block/annotator/LRUAnnotatorTest.java deleted file mode 100644 index 0bfd22705658..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/annotator/LRUAnnotatorTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.annotator; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.meta.StorageDir; - -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Random; - -public class LRUAnnotatorTest extends AbstractBlockAnnotatorTest { - /** - * Sets up base class for LRUAnnotator. - */ - @Before - public void before() throws Exception { - Configuration.set(PropertyKey.WORKER_BLOCK_ANNOTATOR_CLASS, - LRUAnnotator.class.getName()); - init(); - } - - @Test - public void testLRU() throws Exception { - Random rand = new Random(); - List expectedList = new ArrayList<>(); - for (long i = 0; i < 100; i++) { - StorageDir pickedDir = getDir(rand.nextInt(2), rand.nextInt(2)); - createBlock(i, pickedDir); - accessBlock(i); - expectedList.add(i); - } - - validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.NATURAL), - expectedList.iterator()); - - Collections.reverse(expectedList); - validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.REVERSE), - expectedList.iterator()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/AlignTaskTest.java b/core/server/worker/src/test/java/alluxio/worker/block/management/tier/AlignTaskTest.java deleted file mode 100644 index bada0955beba..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/AlignTaskTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.TieredBlockStoreTestUtils; -import alluxio.worker.block.annotator.BlockOrder; -import alluxio.worker.block.meta.StorageDir; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Random; - -public class AlignTaskTest extends BaseTierManagementTaskTest { - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - Configuration.reloadProperties(); - // Current tier layout could end up swapping 2 blocks concurrently. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_RESERVED_BYTES, - 2 * BLOCK_SIZE); - // Disable promotions to avoid interference. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED, false); - // Initialize the tier layout. - init(); - } - - @Test - public void testTierAlignment() throws Exception { - Random rnd = new Random(); - StorageDir[] dirArray = new StorageDir[] {mTestDir1, mTestDir2, mTestDir3, mTestDir4}; - - // Start simulating random load on worker. - startSimulateLoad(); - - // Fill each directory. - long sessionIdCounter = 1000; - long blockIdCounter = 1000; - for (StorageDir dir : dirArray) { - while (dir.getAvailableBytes() > 0) { - TieredBlockStoreTestUtils.cache(sessionIdCounter++, blockIdCounter++, BLOCK_SIZE, - mBlockStore, dir.toBlockStoreLocation(), false); - } - } - - // Access blocks randomly. - for (int i = 0; i < 100; i++) { - StorageDir dirToAccess = dirArray[rnd.nextInt(dirArray.length)]; - List blockIdList = dirToAccess.getBlockIds(); - if (!blockIdList.isEmpty()) { - mBlockStore.accessBlock(sessionIdCounter++, - blockIdList.get(rnd.nextInt(blockIdList.size()))); - } - } - - // Validate tiers are not aligned. (It's not guaranteed but using LRU helps.) - Assert.assertTrue(!mBlockIterator.aligned(BlockStoreLocation.anyDirInTier(FIRST_TIER_ALIAS), - BlockStoreLocation.anyDirInTier(SECOND_TIER_ALIAS), BlockOrder.NATURAL, (b) -> false)); - - // Stop the load for align task to continue. - stopSimulateLoad(); - - CommonUtils.waitFor("Tiers to be aligned by a background task.", - () -> mBlockIterator.aligned(BlockStoreLocation.anyDirInTier(FIRST_TIER_ALIAS), - BlockStoreLocation.anyDirInTier(SECOND_TIER_ALIAS), BlockOrder.NATURAL, (b) -> false), - WaitForOptions.defaults().setTimeoutMs(60000)); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/BaseTierManagementTaskTest.java b/core/server/worker/src/test/java/alluxio/worker/block/management/tier/BaseTierManagementTaskTest.java deleted file mode 100644 index 65113fc1317b..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/BaseTierManagementTaskTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.AllocateOptions; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.TieredBlockStore; -import alluxio.worker.block.TieredBlockStoreTestUtils; -import alluxio.worker.block.annotator.BlockIterator; -import alluxio.worker.block.annotator.LRUAnnotator; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.meta.StorageDir; - -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.lang.reflect.Field; - -/** - * Provides base functionality for tier-management tests. - */ -public abstract class BaseTierManagementTaskTest { - protected static final String FIRST_TIER_ALIAS = TieredBlockStoreTestUtils.TIER_ALIAS[0]; - protected static final String SECOND_TIER_ALIAS = TieredBlockStoreTestUtils.TIER_ALIAS[1]; - protected static final long SIMULATE_LOAD_SESSION_ID = 1; - protected static final long SIMULATE_LOAD_BLOCK_ID = 1; - protected static final long SMALL_BLOCK_SIZE = 10; - protected static final long BLOCK_SIZE = 100; - - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - protected TieredBlockStore mBlockStore; - protected BlockMetadataManager mMetaManager; - protected BlockIterator mBlockIterator; - - protected StorageDir mTestDir1; - protected StorageDir mTestDir2; - protected StorageDir mTestDir3; - protected StorageDir mTestDir4; - - protected BlockWriter mSimulateWriter; - - /** - * Sets up all dependencies before a test runs. - */ - protected void init() throws Exception { - // Disable reviewer to make sure the allocator behavior stays deterministic - Configuration.set(PropertyKey.WORKER_REVIEWER_CLASS, - "alluxio.worker.block.reviewer.AcceptingReviewer"); - // Use LRU for stronger overlap guarantee. - Configuration.set(PropertyKey.WORKER_BLOCK_ANNOTATOR_CLASS, LRUAnnotator.class.getName()); - Configuration.set(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT, BLOCK_SIZE); - // Set timeout for faster task execution. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_LOAD_DETECTION_COOL_DOWN_TIME, "100ms"); - - File tempFolder = mTestFolder.newFolder(); - TieredBlockStoreTestUtils.setupDefaultConf(tempFolder.getAbsolutePath()); - mBlockStore = new TieredBlockStore(); - Field field = mBlockStore.getClass().getDeclaredField("mMetaManager"); - field.setAccessible(true); - mMetaManager = (BlockMetadataManager) field.get(mBlockStore); - mBlockIterator = mMetaManager.getBlockIterator(); - - mTestDir1 = mMetaManager.getTier(FIRST_TIER_ALIAS).getDir(0); - mTestDir2 = mMetaManager.getTier(FIRST_TIER_ALIAS).getDir(1); - mTestDir3 = mMetaManager.getTier(SECOND_TIER_ALIAS).getDir(1); - mTestDir4 = mMetaManager.getTier(SECOND_TIER_ALIAS).getDir(2); - } - - /** - * Stars simulating load on the worker. - */ - protected void startSimulateLoad() throws Exception { - mBlockStore.createBlock(SIMULATE_LOAD_SESSION_ID, SIMULATE_LOAD_BLOCK_ID, - AllocateOptions.forCreate(0, BlockStoreLocation.anyTier())); - mSimulateWriter = - mBlockStore.createBlockWriter(SIMULATE_LOAD_SESSION_ID, SIMULATE_LOAD_BLOCK_ID); - } - - /** - * Stops simulating load on the worker. - */ - protected void stopSimulateLoad() throws Exception { - mBlockStore.abortBlock(SIMULATE_LOAD_SESSION_ID, SIMULATE_LOAD_BLOCK_ID); - mSimulateWriter.close(); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/BlockTransferPartitionerTest.java b/core/server/worker/src/test/java/alluxio/worker/block/management/tier/BlockTransferPartitionerTest.java deleted file mode 100644 index 29ca15b8ad2a..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/BlockTransferPartitionerTest.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.evictor.BlockTransferInfo; -import alluxio.worker.block.management.BlockTransferPartitioner; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -public class BlockTransferPartitionerTest { - private BlockTransferPartitioner mPartitioner = new BlockTransferPartitioner(); - - @Test - public void testEmptyList() throws Exception { - validatePartitions(mPartitioner.partitionTransfers(Collections.emptyList(), 1), 1, 0); - validatePartitions(mPartitioner.partitionTransfers(Collections.emptyList(), 2), 1, 0); - } - - @Test - public void testPartitioning() throws Exception { - // Test partitioning for when there are 2 distinct locations with "2 transfers each". - for (List transfers : generateTransferLists(4, new int[] {2, 2})) { - validatePartitions(mPartitioner.partitionTransfers(transfers, 1), 1, 4); - validatePartitions(mPartitioner.partitionTransfers(transfers, 2), 2, 2, 2); - validatePartitions(mPartitioner.partitionTransfers(transfers, 3), 2, 2, 2); - } - - // Test partitioning for when there are 2 distinct locations with "3 and 1" transfers. - for (List transfers : generateTransferLists(4, new int[] {3, 1})) { - validatePartitions(mPartitioner.partitionTransfers(transfers, 1), 1, 4); - validatePartitions(mPartitioner.partitionTransfers(transfers, 2), 2, 3, 1); - validatePartitions(mPartitioner.partitionTransfers(transfers, 3), 2, 3, 1); - } - - // Test partitioning for when there are 3 distinct locations with "1, 1 and 2" transfers. - for (List transfers : generateTransferLists(4, new int[] {1, 1, 2})) { - validatePartitions(mPartitioner.partitionTransfers(transfers, 1), 1, 4); - validatePartitions(mPartitioner.partitionTransfers(transfers, 2), 2, 3, 1); - validatePartitions(mPartitioner.partitionTransfers(transfers, 3), 3, 1, 1, 2); - } - - // Test partitioning for when there are 4 distinct locations with "a transfer for each". - for (List transfers : generateTransferLists(4, new int[] {1, 1, 1, 1})) { - validatePartitions(mPartitioner.partitionTransfers(transfers, 1), 1, 4); - validatePartitions(mPartitioner.partitionTransfers(transfers, 2), 2, 2, 2); - validatePartitions(mPartitioner.partitionTransfers(transfers, 3), 3, 2, 1, 1); - validatePartitions(mPartitioner.partitionTransfers(transfers, 4), 4, 1, 1, 1, 1); - } - } - - /** - * Generates transfer-lists to validate for when SRC/DST or both are - * allocated(fully identified) locations. - */ - private List> generateTransferLists(int count, int[] locDistribution) { - List> transfersToValidate = new LinkedList<>(); - transfersToValidate.add(generateTransfers(count, true, false, locDistribution, new int[] {4})); - transfersToValidate.add(generateTransfers(count, false, true, new int[] {4}, locDistribution)); - transfersToValidate.add(generateTransfers(count, true, true, locDistribution, locDistribution)); - return transfersToValidate; - } - - /** - * Generates transfers with controls to specify if src/dst is allocated. - * It also acceptslocation-group parameter to specify how many transfers will - * each unique location contain. - */ - List generateTransfers(int count, boolean srcAllocated, boolean dstAllocated, - int[] srcLocGroups, int[] dstLocGroups) { - Assert.assertEquals(count, Arrays.stream(srcLocGroups).sum()); - Assert.assertEquals(count, Arrays.stream(dstLocGroups).sum()); - - List transferInfos = new ArrayList<>(count); - int[] srcLocGroupsCopy = Arrays.copyOf(srcLocGroups, srcLocGroups.length); - int[] dstLocGroupsCopy = Arrays.copyOf(dstLocGroups, dstLocGroups.length); - int srcTierOrdinal = 0; - int dstTierOrdinal = 0; - int srcDirIndex = 0; - int dstDirIndex = 0; - int currLocationGroupSrc = 0; - int currLocationGroupDst = 0; - for (int blockId = 0; blockId < count; blockId++) { - // Generate the src location. - BlockStoreLocation srcLoc; - if (!srcAllocated) { - srcLoc = BlockStoreLocation.anyDirInTier(Integer.toString(srcTierOrdinal)); - } else { - srcLoc = new BlockStoreLocation(Integer.toString(srcTierOrdinal), srcDirIndex); - } - - // Generate the dst location. - BlockStoreLocation dstLoc; - if (!dstAllocated) { - dstLoc = BlockStoreLocation.anyDirInTier(Integer.toString(dstTierOrdinal)); - } else { - dstLoc = new BlockStoreLocation(Integer.toString(dstTierOrdinal), dstDirIndex); - } - - // Create the transfer info. - transferInfos.add(BlockTransferInfo.createMove(srcLoc, blockId, dstLoc)); - - // Advance counters for the next location based on requested location groups. - if (--srcLocGroupsCopy[currLocationGroupSrc] == 0) { - currLocationGroupSrc++; - srcTierOrdinal++; - srcDirIndex++; - } - if (--dstLocGroupsCopy[currLocationGroupDst] == 0) { - currLocationGroupDst++; - dstTierOrdinal++; - dstDirIndex++; - } - } - - return transferInfos; - } - - /** - * Validates given generated partitions. - * - * @param transferPartitions partitions list - * @param expectedPartitionCount number of partitions - * @param expectedPartitionSizes sizes for each partition - */ - private void validatePartitions(List> transferPartitions, - int expectedPartitionCount, int... expectedPartitionSizes) { - Assert.assertEquals(expectedPartitionCount, transferPartitions.size()); - - for (int i = 0; i < transferPartitions.size(); i++) { - Assert.assertEquals(expectedPartitionSizes[i], transferPartitions.get(i).size()); - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/PromoteTaskTest.java b/core/server/worker/src/test/java/alluxio/worker/block/management/tier/PromoteTaskTest.java deleted file mode 100644 index b8cf183e2733..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/PromoteTaskTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; -import alluxio.worker.block.TieredBlockStoreTestUtils; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class PromoteTaskTest extends BaseTierManagementTaskTest { - private static final int MOVE_QUOTA_PERCENT = 80; - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - Configuration.reloadProperties(); - // Disable tier alignment to avoid interference. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED, false); - // Set promotion quota percentage. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_QUOTA_PERCENT, - MOVE_QUOTA_PERCENT); - // Initialize the tier layout. - init(); - } - - @Test - public void testBlockPromotion() throws Exception { - // Start simulating random load on worker. - startSimulateLoad(); - - // Fill 'mTestDir3' on lower tier. - long sessionIdCounter = 1000; - long blockIdCounter = 1000; - while (mTestDir3.getAvailableBytes() > 0) { - TieredBlockStoreTestUtils.cache(sessionIdCounter++, blockIdCounter++, BLOCK_SIZE, mBlockStore, - mTestDir3.toBlockStoreLocation(), false); - } - - // Assert that tiers above has no files. - Assert.assertEquals(0, mTestDir1.getCommittedBytes()); - Assert.assertEquals(0, mTestDir2.getCommittedBytes()); - - // Stop the load for move task to continue. - stopSimulateLoad(); - - // Calculate the expected available bytes on tier after promotions finished. - long usedBytesLimit = (long) (mMetaManager.getTier(FIRST_TIER_ALIAS).getCapacityBytes() - * (double) MOVE_QUOTA_PERCENT / 100); - - CommonUtils.waitFor("Higher tier to be filled with blocs from lower tier.", - () -> mTestDir1.getCommittedBytes() + mTestDir2.getCommittedBytes() == usedBytesLimit, - WaitForOptions.defaults().setTimeoutMs(60000)); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/SwapRestoreTaskTest.java b/core/server/worker/src/test/java/alluxio/worker/block/management/tier/SwapRestoreTaskTest.java deleted file mode 100644 index db03656ad47d..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/management/tier/SwapRestoreTaskTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.management.tier; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.TieredBlockStoreTestUtils; -import alluxio.worker.block.annotator.BlockOrder; -import alluxio.worker.block.meta.StorageDir; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Random; - -public class SwapRestoreTaskTest extends BaseTierManagementTaskTest { - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - Configuration.reloadProperties(); - // Disable move task to avoid interference. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED, false); - // Current tier layout could end up swapping 1 big block. - Configuration.set(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_RESERVED_BYTES, BLOCK_SIZE); - // Initialize the tier layout. - init(); - } - - @Test - public void testTierAlignment() throws Exception { - Random rnd = new Random(); - - // Start simulating random load on worker. - startSimulateLoad(); - - // Fill first dir with small blocks. - long sessionIdCounter = 1000; - long blockIdCounter = 1000; - while (mTestDir1.getAvailableBytes() > 0) { - TieredBlockStoreTestUtils.cache(sessionIdCounter++, blockIdCounter++, SMALL_BLOCK_SIZE, - mBlockStore, mTestDir1.toBlockStoreLocation(), false); - } - - // Fill the rest with big blocks. - StorageDir[] dirArray = new StorageDir[] {mTestDir2, mTestDir3, mTestDir4}; - for (StorageDir dir : dirArray) { - while (dir.getAvailableBytes() > 0) { - TieredBlockStoreTestUtils.cache(sessionIdCounter++, blockIdCounter++, BLOCK_SIZE, - mBlockStore, dir.toBlockStoreLocation(), false); - } - } - - // Access big blocks randomly. - // - // This will cause swaps from below to the top tier. - // This, in turn, is expected to exhaust swap space at the top tier - // which is filled with small blocks. - for (int i = 0; i < 100; i++) { - StorageDir dirToAccess = dirArray[rnd.nextInt(dirArray.length)]; - List blockIdList = dirToAccess.getBlockIds(); - if (!blockIdList.isEmpty()) { - mBlockStore.accessBlock(sessionIdCounter++, - blockIdList.get(rnd.nextInt(blockIdList.size()))); - } - } - - // Validate tiers are not aligned. (It's not guaranteed but using LRU helps.) - Assert.assertTrue(!mBlockIterator.aligned(BlockStoreLocation.anyDirInTier(FIRST_TIER_ALIAS), - BlockStoreLocation.anyDirInTier(SECOND_TIER_ALIAS), BlockOrder.NATURAL, (b) -> false)); - - // Stop the load for swap task to continue. - stopSimulateLoad(); - - // TODO(ggezer): Validate swap-restore task was activated. - CommonUtils.waitFor("Tiers to be aligned by background swap task.", - () -> mBlockIterator.aligned(BlockStoreLocation.anyDirInTier(FIRST_TIER_ALIAS), - BlockStoreLocation.anyDirInTier(SECOND_TIER_ALIAS), BlockOrder.NATURAL, (b) -> false), - WaitForOptions.defaults().setTimeoutMs(60000)); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultBlockMetaTest.java b/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultBlockMetaTest.java deleted file mode 100644 index f90bca3e702f..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultBlockMetaTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.Constants; -import alluxio.util.io.BufferUtils; -import alluxio.util.io.PathUtils; -import alluxio.worker.block.TieredBlockStoreTestUtils; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; - -/** - * Unit tests for {@link DefaultBlockMeta}. - */ -public class DefaultBlockMetaTest { - private static final long TEST_SESSION_ID = 2; - private static final long TEST_BLOCK_ID = 9; - private static final long TEST_BLOCK_SIZE = 100; - private static final int TEST_TIER_ORDINAL = 0; - private static final String TEST_TIER_ALIAS = Constants.MEDIUM_MEM; - private static final long[] TEST_TIER_CAPACITY_BYTES = {100}; - private static final String[] TEST_TIER_MEDIUM_TYPES = {Constants.MEDIUM_MEM}; - private static final String TEST_WORKER_DIR = "testworker"; - private BlockMeta mBlockMeta; - private TempBlockMeta mTempBlockMeta; - private String mTestDirPath; - private String mTestBlockDirPath; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - mTestDirPath = mFolder.newFolder().getAbsolutePath(); - // Sets up tier with one storage dir under mTestDirPath with 100 bytes capacity. - TieredBlockStoreTestUtils.setupConfWithSingleTier(null, TEST_TIER_ORDINAL, - TEST_TIER_ALIAS, new String[] {mTestDirPath}, TEST_TIER_CAPACITY_BYTES, - TEST_TIER_MEDIUM_TYPES, TEST_WORKER_DIR); - - mTestBlockDirPath = PathUtils.concatPath(mTestDirPath, TEST_WORKER_DIR); - StorageTier tier = DefaultStorageTier.newStorageTier(TEST_TIER_ALIAS, 0, false); - StorageDir dir = tier.getDir(0); - mTempBlockMeta = new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir); - } - - /** - * Tests the {@link BlockMeta#getBlockSize()} method. - */ - @Test - public void getBlockSize() throws IOException { - // With the block file not really existing, expect committed block size to be zero. - mBlockMeta = new DefaultBlockMeta(mTempBlockMeta); - Assert.assertEquals(0, mBlockMeta.getBlockSize()); - - // With the block file partially written, expect committed block size equals real file size. - byte[] buf = BufferUtils.getIncreasingByteArray((int) TEST_BLOCK_SIZE - 1); - BufferUtils.writeBufferToFile(mTempBlockMeta.getCommitPath(), buf); - mBlockMeta = new DefaultBlockMeta(mTempBlockMeta); - Assert.assertEquals(TEST_BLOCK_SIZE - 1, mBlockMeta.getBlockSize()); - - // With the block file fully written, expect committed block size equals target block size. - buf = BufferUtils.getIncreasingByteArray((int) TEST_BLOCK_SIZE); - BufferUtils.writeBufferToFile(mTempBlockMeta.getCommitPath(), buf); - mBlockMeta = new DefaultBlockMeta(mTempBlockMeta); - Assert.assertEquals(TEST_BLOCK_SIZE, mBlockMeta.getBlockSize()); - } - - /** - * Tests the {@link BlockMeta#getPath()} method. - */ - @Test - public void getPath() { - mBlockMeta = new DefaultBlockMeta(mTempBlockMeta); - Assert.assertEquals(PathUtils.concatPath(mTestBlockDirPath, TEST_BLOCK_ID), - mBlockMeta.getPath()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultStorageDirTest.java b/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultStorageDirTest.java deleted file mode 100644 index 28e16678b142..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultStorageDirTest.java +++ /dev/null @@ -1,547 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; -import alluxio.exception.ExceptionMessage; -import alluxio.exception.runtime.ResourceExhaustedRuntimeException; -import alluxio.util.io.BufferUtils; -import alluxio.worker.block.BlockStoreLocation; -import alluxio.worker.block.TieredBlockStoreTestUtils; - -import com.google.common.collect.Sets; -import com.google.common.primitives.Ints; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; - -/** - * Unit tests for {@link DefaultStorageDir}. - */ -public final class DefaultStorageDirTest { - private static final long TEST_SESSION_ID = 2; - private static final long TEST_BLOCK_ID = 9; - private static final long TEST_BLOCK_SIZE = 20; - private static final long TEST_TEMP_BLOCK_ID = 10; - private static final long TEST_TEMP_BLOCK_SIZE = 30; - private static final int TEST_TIER_ORDINAL = 0; - private static final int TEST_DIR_INDEX = 1; - private static final long TEST_DIR_CAPACITY = 1000; - private static final long TEST_REVERSED_BYTES = TEST_DIR_CAPACITY / 4; - private String mTestDirPath; - private StorageTier mTier; - private StorageDir mDir; - private BlockMeta mBlockMeta; - private TempBlockMeta mTempBlockMeta; - - /** The exception expected to be thrown. */ - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - // Creates a dummy test dir under mTestDirPath with 1 byte space so initialization can occur - mTestDirPath = mFolder.newFolder().getAbsolutePath(); - String[] testDirPaths = {mTestDirPath}; - long[] testDirCapacity = {1}; - String[] testDirMediumType = {Constants.MEDIUM_MEM}; - - TieredBlockStoreTestUtils.setupConfWithSingleTier(null, TEST_TIER_ORDINAL, Constants.MEDIUM_MEM, - testDirPaths, testDirCapacity, testDirMediumType, null); - - mTier = DefaultStorageTier.newStorageTier(Constants.MEDIUM_MEM, 0, false); - mDir = DefaultStorageDir.newStorageDir(mTier, TEST_DIR_INDEX, TEST_DIR_CAPACITY, - TEST_REVERSED_BYTES, mTestDirPath, Constants.MEDIUM_MEM); - mBlockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, mDir); - mTempBlockMeta = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_TEMP_BLOCK_SIZE, mDir); - } - - private StorageDir newStorageDir(File testDir) { - return DefaultStorageDir.newStorageDir(mTier, TEST_DIR_INDEX, TEST_DIR_CAPACITY, 0, - testDir.getAbsolutePath(), Constants.MEDIUM_MEM); - } - - /** - * Create a block file. - * - * @param dir test directory file identifier - * @param name block file name - * @param lenBytes bytes length of the block file - * @throws IOException exception - */ - private void newBlockFile(File dir, String name, int lenBytes) throws IOException { - File block = new File(dir, name); - block.createNewFile(); - byte[] data = BufferUtils.getIncreasingByteArray(lenBytes); - BufferUtils.writeBufferToFile(block.getAbsolutePath(), data); - } - - /** - * Tests that a new storage directory has metadata for a created block. - */ - @Test - public void initializeMetaNoException() throws Exception { - File testDir = mFolder.newFolder(); - - int nBlock = 10; - long availableBytes = TEST_DIR_CAPACITY; - for (int blockId = 0; blockId < nBlock; blockId++) { - int blockSizeBytes = blockId + 1; - newBlockFile(testDir, String.valueOf(blockId), blockSizeBytes); - availableBytes -= blockSizeBytes; - } - - mDir = newStorageDir(testDir); - assertEquals(TEST_DIR_CAPACITY, mDir.getCapacityBytes()); - assertEquals(availableBytes, mDir.getAvailableBytes()); - for (int blockId = 0; blockId < nBlock; blockId++) { - assertTrue(mDir.hasBlockMeta(blockId)); - } - } - - /** - * Assert meta data is empty. - * - * @param dir storage dir - * @param capacity capacity bytes - */ - private void assertMetadataEmpty(StorageDir dir, long capacity) { - assertEquals(capacity, dir.getCapacityBytes()); - assertEquals(capacity, dir.getAvailableBytes()); - assertTrue(dir.getBlockIds().isEmpty()); - } - - /** - * Assert storage dir is empty. - * - * @param dirPath directory path - * @param dir storage directory - * @param capacity capacity bytes - */ - private void assertStorageDirEmpty(File dirPath, StorageDir dir, long capacity) { - assertMetadataEmpty(dir, capacity); - File[] files = dirPath.listFiles(); - Assert.assertNotNull(files); - assertEquals(0, files.length); - } - - /** - * Tests that the metadata of the files and directory is empty when creating an inappropriate - * file. - */ - @Test - public void initializeMetaDeleteInappropriateFile() throws Exception { - File testDir = mFolder.newFolder(); - - newBlockFile(testDir, "block", 1); - - mDir = newStorageDir(testDir); - assertStorageDirEmpty(testDir, mDir, TEST_DIR_CAPACITY); - } - - /** - * Tests that the metadata of the files and directory is empty when creating an inappropriate - * directory. - */ - @Test - public void initializeMetaDeleteInappropriateDir() throws Exception { - File testDir = mFolder.newFolder(); - - File newDir = new File(testDir, "dir"); - boolean created = newDir.mkdir(); - assertTrue(created); - - mDir = newStorageDir(testDir); - assertStorageDirEmpty(testDir, mDir, TEST_DIR_CAPACITY); - } - - /** - * Tests that an exception is thrown when trying to initialize a block that is larger than the - * capacity. - */ - @Test - public void initializeMetaBlockLargerThanCapacity() throws Exception { - File testDir = mFolder.newFolder(); - - newBlockFile(testDir, String.valueOf(TEST_BLOCK_ID), Ints.checkedCast(TEST_DIR_CAPACITY + 1)); - String alias = Constants.MEDIUM_MEM; - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_SPACE_FOR_BLOCK_META.getMessage(TEST_BLOCK_ID, - TEST_DIR_CAPACITY + 1, TEST_DIR_CAPACITY, alias)); - mDir = newStorageDir(testDir); - assertMetadataEmpty(mDir, TEST_DIR_CAPACITY); - // assert file not deleted - File[] files = testDir.listFiles(); - Assert.assertNotNull(files); - assertEquals(1, files.length); - } - - /** - * Tests the {@link StorageDir#getCapacityBytes()}, the {@link StorageDir#getAvailableBytes()} and - * the {@link StorageDir#getCommittedBytes()} methods. - */ - @Test - public void getBytes() throws Exception { - // Initial state - assertEquals(TEST_DIR_CAPACITY, mDir.getCapacityBytes()); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES, mDir.getAvailableBytes()); - assertEquals(0, mDir.getCommittedBytes()); - - // Add a temp block - mDir.addTempBlockMeta(mTempBlockMeta); - assertEquals(TEST_DIR_CAPACITY, mDir.getCapacityBytes()); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES - TEST_TEMP_BLOCK_SIZE, - mDir.getAvailableBytes()); - assertEquals(0, mDir.getCommittedBytes()); - - // Add a committed block - mDir.addBlockMeta(mBlockMeta); - assertEquals(TEST_DIR_CAPACITY, mDir.getCapacityBytes()); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES - TEST_BLOCK_SIZE - TEST_TEMP_BLOCK_SIZE, - mDir.getAvailableBytes()); - assertEquals(TEST_BLOCK_SIZE, mDir.getCommittedBytes()); - - // Remove the temp block added - mDir.removeTempBlockMeta(mTempBlockMeta); - assertEquals(TEST_DIR_CAPACITY, mDir.getCapacityBytes()); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES - TEST_BLOCK_SIZE, - mDir.getAvailableBytes()); - assertEquals(TEST_BLOCK_SIZE, mDir.getCommittedBytes()); - - // Remove the committed block added - mDir.removeBlockMeta(mBlockMeta); - assertEquals(TEST_DIR_CAPACITY, mDir.getCapacityBytes()); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES, mDir.getAvailableBytes()); - assertEquals(0, mDir.getCommittedBytes()); - } - - /** - * Tests the {@link StorageDir#getDirPath()} method. - */ - @Test - public void getDirPath() { - assertEquals(mTestDirPath, mDir.getDirPath()); - } - - /** - * Tests the {@link StorageDir#getParentTier()} method. - */ - @Test - public void getParentTier() { - assertEquals(mTier, mDir.getParentTier()); - } - - /** - * Tests the {@link StorageDir#getDirIndex()} method. - */ - @Test - public void getDirIndex() { - assertEquals(TEST_DIR_INDEX, mDir.getDirIndex()); - } - - /** - * Tests the {@link StorageDir#getBlockIds()} method. - */ - @Test - public void getBlockIds() throws Exception { - long blockId1 = TEST_BLOCK_ID + 1; - long blockId2 = TEST_BLOCK_ID + 2; - - BlockMeta blockMeta1 = new DefaultBlockMeta(blockId1, TEST_BLOCK_SIZE, mDir); - BlockMeta blockMeta2 = new DefaultBlockMeta(blockId2, TEST_BLOCK_SIZE, mDir); - mDir.addBlockMeta(blockMeta1); - mDir.addBlockMeta(blockMeta2); - - List actual = mDir.getBlockIds(); - assertEquals(Sets.newHashSet(blockId1, blockId2), new HashSet<>(actual)); - } - - /** - * Tests the {@link StorageDir#getBlocks()} method. - */ - @Test - public void getBlocks() throws Exception { - long blockId1 = TEST_BLOCK_ID + 1; - long blockId2 = TEST_BLOCK_ID + 2; - - BlockMeta blockMeta1 = new DefaultBlockMeta(blockId1, TEST_BLOCK_SIZE, mDir); - BlockMeta blockMeta2 = new DefaultBlockMeta(blockId2, TEST_BLOCK_SIZE, mDir); - mDir.addBlockMeta(blockMeta1); - mDir.addBlockMeta(blockMeta2); - - List actual = mDir.getBlocks(); - assertEquals(Sets.newHashSet(blockMeta1, blockMeta2), new HashSet<>(actual)); - } - - /** - * Tests that an exception is thrown when trying to add metadata of a block that is too big. - */ - @Test - public void addBlockMetaTooBig() { - final long bigBlockSize = TEST_DIR_CAPACITY + 1; - BlockMeta bigBlockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, bigBlockSize, mDir); - String alias = bigBlockMeta.getBlockLocation().tierAlias(); - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_SPACE_FOR_BLOCK_META.getMessage(TEST_BLOCK_ID, - bigBlockSize, TEST_DIR_CAPACITY - TEST_REVERSED_BYTES, alias)); - mDir.addBlockMeta(bigBlockMeta); - } - - /** - * Tests that an exception is thrown when trying to add metadata of a block that already exists. - */ - @Test - public void addBlockMetaExisting() throws Exception { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.ADD_EXISTING_BLOCK - .getMessage(TEST_BLOCK_ID, Constants.MEDIUM_MEM)); - mDir.addBlockMeta(mBlockMeta); - BlockMeta dupBlockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, mDir); - mDir.addBlockMeta(dupBlockMeta); - } - - /** - * Tests get the metadata of a block which does not exist returns Optional.empty(). - */ - @Test - public void getBlockMetaNotExisting() { - assertFalse(mDir.getBlockMeta(TEST_BLOCK_ID).isPresent()); - } - - /** - * Tests that an exception is thrown when trying to add the metadata of a temporary block which is - * too big. - */ - @Test - public void addTempBlockMetaTooBig() { - final long bigBlockSize = TEST_DIR_CAPACITY + 1; - TempBlockMeta bigTempBlockMeta = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, bigBlockSize, mDir); - String alias = bigTempBlockMeta.getBlockLocation().tierAlias(); - mThrown.expect(ResourceExhaustedRuntimeException.class); - mThrown.expectMessage(ExceptionMessage.NO_SPACE_FOR_BLOCK_META.getMessage(TEST_TEMP_BLOCK_ID, - bigBlockSize, TEST_DIR_CAPACITY - TEST_REVERSED_BYTES, alias)); - mDir.addTempBlockMeta(bigTempBlockMeta); - } - - /** - * Tests that an exception is thrown when trying to add the metadata of a termpoary block which - * already exists. - */ - @Test - public void addTempBlockMetaExisting() { - mThrown.expect(IllegalStateException.class); - mThrown - .expectMessage(ExceptionMessage.ADD_EXISTING_BLOCK - .getMessage(TEST_TEMP_BLOCK_ID, Constants.MEDIUM_MEM)); - mDir.addTempBlockMeta(mTempBlockMeta); - TempBlockMeta dupTempBlockMeta = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_TEMP_BLOCK_SIZE, mDir); - mDir.addTempBlockMeta(dupTempBlockMeta); - } - - /** - * Tests that an exception is thrown when trying to remove the metadata of a tempoary block which - * does not exist. - */ - @Test - public void removeTempBlockMetaNotExisting() { - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.BLOCK_META_NOT_FOUND.getMessage(TEST_TEMP_BLOCK_ID)); - mDir.removeTempBlockMeta(mTempBlockMeta); - } - - /** - * Tests that an exception is thrown when trying to remove the metadata of a temporary block which - * is not owned. - */ - @Test - public void removeTempBlockMetaNotOwner() { - final long wrongSessionId = TEST_SESSION_ID + 1; - String alias = Constants.MEDIUM_MEM; - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage(ExceptionMessage.BLOCK_NOT_FOUND_FOR_SESSION - .getMessage(TEST_TEMP_BLOCK_ID, alias, wrongSessionId)); - mDir.addTempBlockMeta(mTempBlockMeta); - TempBlockMeta wrongTempBlockMeta = - new DefaultTempBlockMeta(wrongSessionId, TEST_TEMP_BLOCK_ID, TEST_TEMP_BLOCK_SIZE, mDir); - mDir.removeTempBlockMeta(wrongTempBlockMeta); - } - - /** - * Tests getting the metadata of a temporary block that does not exist returns Optional.empty(). - */ - @Test - public void getTempBlockMetaNotExisting() { - assertFalse(mDir.getBlockMeta(TEST_TEMP_BLOCK_ID).isPresent()); - } - - /** - * Tests the {@link StorageDir#addBlockMeta(BlockMeta)} and the - * {@link StorageDir#removeBlockMeta(BlockMeta)} methods. - */ - @Test - public void blockMeta() throws Exception { - assertFalse(mDir.hasBlockMeta(TEST_BLOCK_ID)); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES, mDir.getAvailableBytes()); - - mDir.addBlockMeta(mBlockMeta); - assertTrue(mDir.hasBlockMeta(TEST_BLOCK_ID)); - assertEquals(mBlockMeta, mDir.getBlockMeta(TEST_BLOCK_ID).get()); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES - TEST_BLOCK_SIZE, - mDir.getAvailableBytes()); - - mDir.removeBlockMeta(mBlockMeta); - assertFalse(mDir.hasBlockMeta(TEST_BLOCK_ID)); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES, mDir.getAvailableBytes()); - } - - /** - * Tests the {@link StorageDir#addTempBlockMeta(TempBlockMeta)} and the - * {@link StorageDir#removeTempBlockMeta(TempBlockMeta)} methods. - */ - @Test - public void tempBlockMeta() throws Exception { - assertFalse(mDir.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES, mDir.getAvailableBytes()); - - mDir.addTempBlockMeta(mTempBlockMeta); - assertTrue(mDir.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertEquals(mTempBlockMeta, mDir.getTempBlockMeta(TEST_TEMP_BLOCK_ID).get()); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES - TEST_TEMP_BLOCK_SIZE, - mDir.getAvailableBytes()); - - mDir.removeTempBlockMeta(mTempBlockMeta); - assertFalse(mDir.hasTempBlockMeta(TEST_TEMP_BLOCK_ID)); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES, mDir.getAvailableBytes()); - } - - /** - * Tests the {@link StorageDir#resizeTempBlockMeta(TempBlockMeta, long)} method. - */ - @Test - public void resizeTempBlockMeta() { - mDir.addTempBlockMeta(mTempBlockMeta); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES - TEST_TEMP_BLOCK_SIZE, - mDir.getAvailableBytes()); - final long newSize = TEST_TEMP_BLOCK_SIZE + 10; - mDir.resizeTempBlockMeta(mTempBlockMeta, newSize); - assertEquals(TEST_DIR_CAPACITY - TEST_REVERSED_BYTES - newSize, mDir.getAvailableBytes()); - } - - /** - * Tests that an exception is thrown when trying to shrink a block via - * the {@link StorageDir#resizeTempBlockMeta(TempBlockMeta, long)} method. - */ - @Test - public void resizeTempBlockMetaInvalidStateException() { - mDir.addTempBlockMeta(mTempBlockMeta); - final long newSize = TEST_TEMP_BLOCK_SIZE - 10; - assertThrows("Shrinking block, not supported!", IllegalStateException.class, - () -> mDir.resizeTempBlockMeta(mTempBlockMeta, newSize)); - assertEquals(TEST_TEMP_BLOCK_SIZE, mTempBlockMeta.getBlockSize()); - } - - /** - * Tests that an exception is thrown when trying to resize a temporary block via the - * {@link StorageDir#resizeTempBlockMeta(TempBlockMeta, long)} method without no available bytes. - */ - @Test - public void resizeTempBlockMetaNoAvailableBytes() { - mDir.addTempBlockMeta(mTempBlockMeta); - // resize the temp block size to the dir capacity, which is the limit - mDir.resizeTempBlockMeta(mTempBlockMeta, TEST_DIR_CAPACITY); - assertEquals(TEST_DIR_CAPACITY, mTempBlockMeta.getBlockSize()); - mThrown.expect(IllegalStateException.class); - mThrown.expectMessage("Available bytes should always be non-negative"); - // resize again, now the newSize is more than available bytes, exception thrown - mDir.resizeTempBlockMeta(mTempBlockMeta, TEST_DIR_CAPACITY + 1); - } - - /** - * Tests the {@link StorageDir#cleanupSessionTempBlocks(long, List)} method. - */ - @Test - public void cleanupSession() throws Exception { - // TODO(bin): Also test claimed space. - // Create blocks under TEST_SESSION_ID - mDir.addBlockMeta(mBlockMeta); - - // Create temp blocks under TEST_SESSION_ID - long tempBlockId1 = TEST_TEMP_BLOCK_ID + 1; - long tempBlockId2 = TEST_TEMP_BLOCK_ID + 2; - long tempBlockId3 = TEST_TEMP_BLOCK_ID + 3; - long otherSessionId = TEST_SESSION_ID + 1; - - TempBlockMeta tempBlockMeta1 = - new DefaultTempBlockMeta(TEST_SESSION_ID, tempBlockId1, TEST_TEMP_BLOCK_SIZE, mDir); - TempBlockMeta tempBlockMeta2 = - new DefaultTempBlockMeta(TEST_SESSION_ID, tempBlockId2, TEST_TEMP_BLOCK_SIZE, mDir); - TempBlockMeta tempBlockMeta3 = - new DefaultTempBlockMeta(otherSessionId, tempBlockId3, TEST_TEMP_BLOCK_SIZE, mDir); - mDir.addTempBlockMeta(tempBlockMeta1); - mDir.addTempBlockMeta(tempBlockMeta2); - mDir.addTempBlockMeta(tempBlockMeta3); - - // Check the temporary blocks belonging to TEST_SESSION_ID - List actual = mDir.getSessionTempBlocks(TEST_SESSION_ID); - List actualBlockIds = new ArrayList<>(actual.size()); - for (TempBlockMeta tempBlockMeta : actual) { - actualBlockIds.add(tempBlockMeta.getBlockId()); - } - assertEquals(Sets.newHashSet(tempBlockMeta1, tempBlockMeta2), - new HashSet<>(actual)); - assertTrue(mDir.hasTempBlockMeta(tempBlockId1)); - assertTrue(mDir.hasTempBlockMeta(tempBlockId2)); - - // Two temp blocks created by TEST_SESSION_ID are expected to be removed - mDir.cleanupSessionTempBlocks(TEST_SESSION_ID, actualBlockIds); - assertFalse(mDir.hasTempBlockMeta(tempBlockId1)); - assertFalse(mDir.hasTempBlockMeta(tempBlockId2)); - // Temp block created by otherSessionId is expected to stay - assertTrue(mDir.hasTempBlockMeta(tempBlockId3)); - // Block created by TEST_SESSION_ID is expected to stay - assertTrue(mDir.hasBlockMeta(TEST_BLOCK_ID)); - } - - /** - * Tests the {@link StorageDir#toBlockStoreLocation()} method. - */ - @Test - public void toBlockStoreLocation() { - StorageTier tier = mDir.getParentTier(); - assertEquals(new BlockStoreLocation(tier.getTierAlias(), mDir.getDirIndex(), - mDir.getDirMedium()), mDir.toBlockStoreLocation()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultStorageTierTest.java b/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultStorageTierTest.java deleted file mode 100644 index 111922ffab48..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultStorageTierTest.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.io.PathUtils; -import alluxio.worker.block.TieredBlockStoreTestUtils; - -import com.google.common.collect.ImmutableList; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.util.List; - -/** - * Unit tests for {@link DefaultStorageTier}. - */ -public class DefaultStorageTierTest { - private static final long TEST_SESSION_ID = 2; - private static final long TEST_TEMP_BLOCK_ID = 10; - private static final long TEST_BLOCK_SIZE = 20; - private static final long TEST_DIR1_CAPACITY = 2000; - private static final long TEST_DIR2_CAPACITY = 3000; - private static final int TEST_TIER_ORDINAL = 0; - private static final String TEST_TIER_ALIAS = Constants.MEDIUM_MEM; - private static final String TEST_WORKER_DATA_DIR = "testworker"; - - private static final long[] TIER_CAPACITY_BYTES = {TEST_DIR1_CAPACITY, TEST_DIR2_CAPACITY}; - private static final String[] TEST_TIER_MEDIUM_TYPES = - {Constants.MEDIUM_MEM, Constants.MEDIUM_MEM}; - private StorageTier mTier; - private StorageDir mDir1; - private TempBlockMeta mTempBlockMeta; - private String mTestDirPath1; - private String mTestDirPath2; - private String mTestBlockDirPath1; - private String mTestBlockDirPath2; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - /** The exception expected to be thrown. */ - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public final void before() throws Exception { - mTestDirPath1 = mFolder.newFolder().getAbsolutePath(); - mTestDirPath2 = mFolder.newFolder().getAbsolutePath(); - String[] tierPath = {mTestDirPath1, mTestDirPath2}; - - TieredBlockStoreTestUtils.setupConfWithSingleTier(null, TEST_TIER_ORDINAL, - TEST_TIER_ALIAS, tierPath, TIER_CAPACITY_BYTES, - TEST_TIER_MEDIUM_TYPES, TEST_WORKER_DATA_DIR); - - mTestBlockDirPath1 = PathUtils.concatPath(mTestDirPath1, TEST_WORKER_DATA_DIR); - mTestBlockDirPath2 = PathUtils.concatPath(mTestDirPath2, TEST_WORKER_DATA_DIR); - mTier = DefaultStorageTier.newStorageTier(Constants.MEDIUM_MEM, 0, false); - mDir1 = mTier.getDir(0); - mTempBlockMeta = - new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_BLOCK_SIZE, mDir1); - } - - /** - * Tests the {@link StorageTier#getTierAlias()} method. - */ - @Test - public void getTierAlias() { - Assert.assertEquals(TEST_TIER_ALIAS, mTier.getTierAlias()); - } - - /** - * Tests the {@link StorageTier#getTierOrdinal()} method. - */ - @Test - public void getTierLevel() { - Assert.assertEquals(TEST_TIER_ORDINAL, mTier.getTierOrdinal()); - } - - /** - * Tests the {@link StorageTier#getCapacityBytes()} method. - */ - @Test - public void getCapacityBytes() throws Exception { - Assert.assertEquals(TEST_DIR1_CAPACITY + TEST_DIR2_CAPACITY, mTier.getCapacityBytes()); - - // Capacity should not change after adding block to a dir. - mDir1.addTempBlockMeta(mTempBlockMeta); - Assert.assertEquals(TEST_DIR1_CAPACITY + TEST_DIR2_CAPACITY, mTier.getCapacityBytes()); - } - - /** - * Tests the {@link StorageTier#getAvailableBytes()} method. - */ - @Test - public void getAvailableBytes() throws Exception { - Assert.assertEquals(TEST_DIR1_CAPACITY + TEST_DIR2_CAPACITY, mTier.getAvailableBytes()); - - // Capacity should subtract block size after adding block to a dir. - mDir1.addTempBlockMeta(mTempBlockMeta); - Assert.assertEquals(TEST_DIR1_CAPACITY + TEST_DIR2_CAPACITY - TEST_BLOCK_SIZE, - mTier.getAvailableBytes()); - } - - /** - * Tests that an exception is thrown when trying to get a directory by a non-existing index. - */ - @Test - public void getDir() { - StorageDir dir1 = mTier.getDir(0); - Assert.assertEquals(mTestBlockDirPath1, dir1.getDirPath()); - StorageDir dir2 = mTier.getDir(1); - Assert.assertEquals(mTestBlockDirPath2, dir2.getDirPath()); - // Get dir by a non-existing index, expect getDir to fail and throw IndexOutOfBoundsException - Assert.assertNull(mTier.getDir(2)); - } - - /** - * Tests the {@link StorageTier#getStorageDirs()} method. - */ - @Test - public void getStorageDirs() { - List dirs = mTier.getStorageDirs(); - Assert.assertEquals(2, dirs.size()); - Assert.assertEquals(mTestBlockDirPath1, dirs.get(0).getDirPath()); - Assert.assertEquals(mTestBlockDirPath2, dirs.get(1).getDirPath()); - } - - @Test - public void tolerantFailureInStorageDir() { - PropertyKey tierDirPathConf = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(0); - Configuration.set(tierDirPathConf, "/dev/null/invalid," + mTestDirPath1); - mTier = DefaultStorageTier.newStorageTier(Constants.MEDIUM_MEM, 0, false); - List dirs = mTier.getStorageDirs(); - Assert.assertEquals(1, dirs.size()); - Assert.assertEquals(mTestBlockDirPath1, dirs.get(0).getDirPath()); - } - - @Test - public void tolerantMisconfigurationInStorageDir() { - Configuration - .set(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_MEDIUMTYPE.format(0), - Constants.MEDIUM_MEM); - Configuration - .set(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA.format(0), - "2000"); - mTier = DefaultStorageTier.newStorageTier(Constants.MEDIUM_MEM, 0, false); - List dirs = mTier.getStorageDirs(); - Assert.assertEquals(2, dirs.size()); - Assert.assertEquals(mTestBlockDirPath1, dirs.get(0).getDirPath()); - } - - @Test - public void removeDir() { - List dirs = mTier.getStorageDirs(); - Assert.assertEquals(2, dirs.size()); - StorageDir dir0 = dirs.get(0); - StorageDir dir1 = dirs.get(1); - mTier.removeStorageDir(dir0); - Assert.assertEquals(ImmutableList.of(dir1), mTier.getStorageDirs()); - mTier.removeStorageDir(dir1); - Assert.assertEquals(ImmutableList.of(), mTier.getStorageDirs()); - - StorageTier anotherTier = DefaultStorageTier.newStorageTier("anotherTier", 0, false); - StorageDir dirInAnotherTier = - DefaultStorageDir.newStorageDir(anotherTier, 0, 0, 0, "dir", "medium"); - Assert.assertThrows("should not remove a dir that does not belong to this tier", - IllegalArgumentException.class, () -> mTier.removeStorageDir(dirInAnotherTier)); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultTempBlockMetaTest.java b/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultTempBlockMetaTest.java deleted file mode 100644 index 2bb85771ecc1..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/meta/DefaultTempBlockMetaTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.Constants; -import alluxio.util.io.PathUtils; -import alluxio.worker.block.TieredBlockStoreTestUtils; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -/** - * Unit tests for {@link DefaultTempBlockMeta}. - */ -public class DefaultTempBlockMetaTest { - private static final long TEST_SESSION_ID = 2; - private static final long TEST_BLOCK_ID = 9; - private static final long TEST_BLOCK_SIZE = 100; - private static final int TEST_TIER_ORDINAL = 0; - private static final String TEST_TIER_ALIAS = Constants.MEDIUM_MEM; - private static final long[] TEST_TIER_CAPACITY_BYTES = {100}; - private static final String[] TEST_TIER_MEDIUM_TYPES = {Constants.MEDIUM_MEM}; - private static final String TEST_WORKER_DATA_FOLDER = "workertest"; - private String mTestDirPath; - private String mTestBlockDirPath; - private TempBlockMeta mTempBlockMeta; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - mTestDirPath = mFolder.newFolder().getAbsolutePath(); - // Sets up tier with one storage dir under mTestDirPath with 100 bytes capacity. - TieredBlockStoreTestUtils.setupConfWithSingleTier(null, TEST_TIER_ORDINAL, - TEST_TIER_ALIAS, new String[] {mTestDirPath}, TEST_TIER_CAPACITY_BYTES, - TEST_TIER_MEDIUM_TYPES, TEST_WORKER_DATA_FOLDER); - - StorageTier tier = DefaultStorageTier.newStorageTier(TEST_TIER_ALIAS, 0, false); - StorageDir dir = tier.getDir(0); - // Append the worker data folder to the expected path. - mTestBlockDirPath = PathUtils.concatPath(mTestDirPath, TEST_WORKER_DATA_FOLDER); - mTempBlockMeta = new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir); - } - - /** - * Tests the {@link TempBlockMeta#getPath()} method. - */ - @Test - public void getPath() { - Assert.assertEquals(PathUtils.concatPath(mTestBlockDirPath, - ".tmp_blocks", TEST_SESSION_ID % 1024, - String.format("%x-%x", TEST_SESSION_ID, TEST_BLOCK_ID)), - mTempBlockMeta.getPath()); - } - - /** - * Tests the {@link TempBlockMeta#getCommitPath()} method. - */ - @Test - public void getCommitPath() { - Assert.assertEquals(PathUtils.concatPath(mTestBlockDirPath, TEST_BLOCK_ID), - mTempBlockMeta.getCommitPath()); - } - - /** - * Tests the {@link TempBlockMeta#getSessionId()} method. - */ - @Test - public void getSessionId() { - Assert.assertEquals(TEST_SESSION_ID, mTempBlockMeta.getSessionId()); - } - - /** - * Tests the {@link TempBlockMeta#setBlockSize(long)} method. - */ - @Test - public void setBlockSize() { - Assert.assertEquals(TEST_BLOCK_SIZE, mTempBlockMeta.getBlockSize()); - mTempBlockMeta.setBlockSize(1); - Assert.assertEquals(1, mTempBlockMeta.getBlockSize()); - mTempBlockMeta.setBlockSize(100); - Assert.assertEquals(100, mTempBlockMeta.getBlockSize()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/meta/StorageDirViewTest.java b/core/server/worker/src/test/java/alluxio/worker/block/meta/StorageDirViewTest.java deleted file mode 100644 index 59197e80c53d..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/meta/StorageDirViewTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.TieredBlockStoreTestUtils; - -import com.google.common.collect.Lists; -import org.hamcrest.CoreMatchers; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -import java.io.File; -import java.util.HashSet; -import java.util.List; - -/** - * Tests for the {@link StorageDirView} class. - */ -public class StorageDirViewTest { - private static final int TEST_TIER_LEVEL = 0; - private static final int TEST_DIR = 0; - private static final long TEST_SESSION_ID = 2; - private static final long TEST_BLOCK_ID = 9; - private static final long TEST_TEMP_BLOCK_ID = 10; - private static final long TEST_BLOCK_SIZE = 20; - private StorageDir mTestDir; - private StorageDirEvictorView mTestDirView; - private StorageTierEvictorView mTestTierView; - private BlockMetadataEvictorView mMetadataView; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - File tempFolder = mTestFolder.newFolder(); - BlockMetadataManager metaManager = - TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath()); - mMetadataView = - Mockito.spy(new BlockMetadataEvictorView(metaManager, new HashSet(), - new HashSet())); - StorageTier testTier = metaManager.getTiers().get(TEST_TIER_LEVEL); - mTestDir = testTier.getDir(TEST_DIR); - mTestTierView = new StorageTierEvictorView(testTier, mMetadataView); - mTestDirView = new StorageDirEvictorView(mTestDir, mTestTierView, mMetadataView); - } - - /** - * Tests the {@link StorageDirEvictorView#getParentTierView()} method. - */ - @Test - public void getParentTierView() { - Assert.assertEquals(mTestTierView, mTestDirView.getParentTierView()); - } - - /** - * Tests the {@link StorageDirView#getAvailableBytes()} method. - */ - @Test - public void getAvailableBytes() { - Assert.assertEquals(mTestDir.getAvailableBytes(), mTestDirView.getAvailableBytes()); - } - - /** - * Tests the {@link StorageDirView#getCommittedBytes()} method. - */ - @Test - public void getCommittedBytes() { - Assert.assertEquals(mTestDir.getCommittedBytes(), mTestDirView.getCommittedBytes()); - } - - /** - * Tests the {@link StorageDirView#getCapacityBytes()} method. - */ - @Test - public void getCapacityBytes() { - Assert.assertEquals(mTestDir.getCapacityBytes(), mTestDirView.getCapacityBytes()); - } - - /** - * Tests the {@link StorageDirView#getDirViewIndex()} method. - */ - @Test - public void getDirViewIndex() { - Assert.assertEquals(mTestDir.getDirIndex(), mTestDirView.getDirViewIndex()); - } - - /** - * Tests the {@link StorageDirView#getMediumType()} method. - */ - @Test - public void getMediumType() { - Assert.assertEquals(mTestDir.getDirMedium(), mTestDirView.getMediumType()); - } - - /** - * Tests the {@link StorageDirView#toBlockStoreLocation()} method. - */ - @Test - public void toBlockStoreLocation() { - Assert.assertEquals(mTestDir.toBlockStoreLocation(), mTestDirView.toBlockStoreLocation()); - } - - /** - * Tests the {@link StorageDirEvictorView#getEvictableBlocks()} method. - */ - @Test - public void getEvictableBlocks() throws Exception { - // When test dir is empty, expect no block to be evictable - Assert.assertEquals(0, mTestDirView.getEvitableBytes()); - Assert.assertTrue(mTestDirView.getEvictableBlocks().isEmpty()); - - // Add one block to test dir, expect this block to be evictable - BlockMeta blockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, mTestDir); - mTestDir.addBlockMeta(blockMeta); - Assert.assertEquals(TEST_BLOCK_SIZE, mTestDirView.getEvitableBytes()); - Assert.assertThat(mTestDirView.getEvictableBlocks(), - CoreMatchers.is((List) Lists.newArrayList(blockMeta))); - - // Lock this block, expect this block to be non-evictable - Mockito.when(mMetadataView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(false); - Mockito.when(mMetadataView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(true); - Assert.assertEquals(0, mTestDirView.getEvitableBytes()); - Assert.assertTrue(mTestDirView.getEvictableBlocks().isEmpty()); - - // Pin this block, expect this block to be non-evictable - Mockito.when(mMetadataView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(true); - Mockito.when(mMetadataView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(false); - Assert.assertEquals(0, mTestDirView.getEvitableBytes()); - Assert.assertTrue(mTestDirView.getEvictableBlocks().isEmpty()); - - // Release pin/lock, expect this block to be evictable - Mockito.when(mMetadataView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(false); - Mockito.when(mMetadataView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(false); - Assert.assertEquals(TEST_BLOCK_SIZE, mTestDirView.getEvitableBytes()); - Assert.assertThat(mTestDirView.getEvictableBlocks(), - CoreMatchers.is((List) Lists.newArrayList(blockMeta))); - } - - /** - * Tests the {@link StorageDirView#createTempBlockMeta(long, long, long)} method. - */ - @Test - public void createTempBlockMeta() { - TempBlockMeta tempBlockMeta = - mTestDirView.createTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_BLOCK_SIZE); - Assert.assertEquals(TEST_SESSION_ID, tempBlockMeta.getSessionId()); - Assert.assertEquals(TEST_TEMP_BLOCK_ID, tempBlockMeta.getBlockId()); - Assert.assertEquals(TEST_BLOCK_SIZE, tempBlockMeta.getBlockSize()); - Assert.assertEquals(mTestDir, tempBlockMeta.getParentDir()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/meta/StorageTierViewTest.java b/core/server/worker/src/test/java/alluxio/worker/block/meta/StorageTierViewTest.java deleted file mode 100644 index 47e9bb350594..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/meta/StorageTierViewTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.meta; - -import alluxio.worker.block.BlockMetadataEvictorView; -import alluxio.worker.block.BlockMetadataManager; -import alluxio.worker.block.TieredBlockStoreTestUtils; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.util.HashSet; - -/** - * Unit tests for {@link StorageTierView}. - */ -public class StorageTierViewTest { - private static final int TEST_TIER_LEVEL = 0; - private StorageTier mTestTier; - private StorageTierEvictorView mTestTierView; - private BlockMetadataEvictorView mMetadataView; - - /** Rule to create a new temporary folder during each test. */ - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - /** The exception expected to be thrown. */ - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - /** - * Sets up all dependencies before a test runs. - */ - @Before - public void before() throws Exception { - File tempFolder = mTestFolder.newFolder(); - BlockMetadataManager metaManager = - TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath()); - mMetadataView = - new BlockMetadataEvictorView(metaManager, new HashSet(), - new HashSet()); - mTestTier = metaManager.getTiers().get(TEST_TIER_LEVEL); - mTestTierView = new StorageTierEvictorView(mTestTier, mMetadataView); - } - - /** - * Tests the {@link StorageTierEvictorView#getDirViews()} method. - */ - @Test - public void getDirViews() { - Assert.assertEquals(TieredBlockStoreTestUtils.TIER_PATH[TEST_TIER_LEVEL].length, mTestTierView - .getDirViews().size()); - } - - /** - * Tests the {@link StorageTierEvictorView#getDirView(int)} method. - */ - @Test - public void getDirView() { - for (int i = 0; i < TieredBlockStoreTestUtils.TIER_PATH[TEST_TIER_LEVEL].length; i++) { - Assert.assertEquals(i, mTestTierView.getDirView(i).getDirViewIndex()); - } - } - - /** - * Tests that an exception is thrown when trying to get a storage directory view with a bad index. - */ - @Test - public void getDirViewBadIndex() { - int badDirIndex = TieredBlockStoreTestUtils.TIER_PATH[TEST_TIER_LEVEL].length; - Assert.assertNull(mTestTierView.getDirView(badDirIndex)); - } - - /** - * Tests the {@link StorageTierEvictorView#getTierViewAlias()} method. - */ - @Test - public void getTierViewAlias() { - Assert.assertEquals(mTestTier.getTierAlias(), mTestTierView.getTierViewAlias()); - } - - /** - * Tests the {@link StorageTierEvictorView#getTierViewOrdinal()} method. - */ - @Test - public void getTierViewOrdinal() { - Assert.assertEquals(mTestTier.getTierOrdinal(), mTestTierView.getTierViewOrdinal()); - } - - /** - * Tests the {@link StorageTierEvictorView#getTierViewOrdinal()} method. - */ - @Test - public void getBlockMetadataEvictorView() { - Assert.assertEquals(mMetadataView, mTestTierView.getBlockMetadataEvictorView()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/reviewer/MockReviewer.java b/core/server/worker/src/test/java/alluxio/worker/block/reviewer/MockReviewer.java deleted file mode 100644 index a41deb9a6ea2..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/reviewer/MockReviewer.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.reviewer; - -import alluxio.worker.block.meta.StorageDirView; - -import com.google.common.collect.Sets; - -import java.util.Set; - -/** - * An implementation of {@link Reviewer}. This is used for Reviewer unit test. - * If a directory's `availableBytes` matches the bytes in `BYTES_TO_REJECT`, - * allocation will be rejected. - */ -public class MockReviewer implements Reviewer { - - private static final Set BYTES_TO_REJECT = Sets.newHashSet(); - - public static void resetBytesToReject(Set bytes) { - BYTES_TO_REJECT.clear(); - BYTES_TO_REJECT.addAll(bytes); - } - - @Override - public boolean acceptAllocation(StorageDirView dirView) { - long availableBytes = dirView.getAvailableBytes(); - return !BYTES_TO_REJECT.contains(availableBytes); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/reviewer/ProbabilisticBufferReviewerTest.java b/core/server/worker/src/test/java/alluxio/worker/block/reviewer/ProbabilisticBufferReviewerTest.java deleted file mode 100644 index 1cf7dbc14155..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/reviewer/ProbabilisticBufferReviewerTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.reviewer; - -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.FormatUtils; -import alluxio.worker.block.meta.StorageDirView; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class ProbabilisticBufferReviewerTest { - private static final long DISK_SIZE = FormatUtils.parseSpaceSize("16GB"); - private static final String SOFT_LIMIT = "256MB"; - private static final long SOFT_LIMIT_BYTES = FormatUtils.parseSpaceSize(SOFT_LIMIT); - private static final String HARD_LIMIT = "64MB"; - private static final long HARD_LIMIT_BYTES = FormatUtils.parseSpaceSize(HARD_LIMIT); - - private ProbabilisticBufferReviewer mReviewer; - - @Before - public void createReviewerInstance() { - Configuration.set(PropertyKey.WORKER_REVIEWER_CLASS, - ProbabilisticBufferReviewer.class.getName()); - Configuration.set(PropertyKey.WORKER_REVIEWER_PROBABILISTIC_HARDLIMIT_BYTES, HARD_LIMIT); - Configuration.set(PropertyKey.WORKER_REVIEWER_PROBABILISTIC_SOFTLIMIT_BYTES, SOFT_LIMIT); - - Reviewer reviewer = Reviewer.Factory.create(); - assertTrue(reviewer instanceof ProbabilisticBufferReviewer); - mReviewer = (ProbabilisticBufferReviewer) reviewer; - } - - @After - public void reset() { - mReviewer = null; - Configuration.reloadProperties(); - } - - @Test - public void testProbabilityFunction() throws Exception { - // Empty - 100% - StorageDirView mockEmptyDir = mock(StorageDirView.class); - when(mockEmptyDir.getAvailableBytes()).thenReturn(DISK_SIZE); - when(mockEmptyDir.getCapacityBytes()).thenReturn(DISK_SIZE); - double probEmptyDir = mReviewer.getProbability(mockEmptyDir); - assertEquals(1.0, probEmptyDir, 1e-6); - - // Higher than soft limit - 100% - StorageDirView mockMoreThanSoft = mock(StorageDirView.class); - when(mockMoreThanSoft.getAvailableBytes()).thenReturn(SOFT_LIMIT_BYTES + 1); - when(mockMoreThanSoft.getCapacityBytes()).thenReturn(DISK_SIZE); - double probMoreThanSoft = mReviewer.getProbability(mockMoreThanSoft); - assertEquals(1.0, probMoreThanSoft, 1e-6); - - // Lower than soft limit - less than 100% - StorageDirView mockLessThanSoft = mock(StorageDirView.class); - when(mockLessThanSoft.getAvailableBytes()).thenReturn(SOFT_LIMIT_BYTES - 1); - when(mockLessThanSoft.getCapacityBytes()).thenReturn(DISK_SIZE); - double probLessThanSoft = mReviewer.getProbability(mockLessThanSoft); - assertEquals(0.99999999, probLessThanSoft, 1e-4); - - // Between soft limit and hard limit - linear - StorageDirView mockMoreThanHard = mock(StorageDirView.class); - when(mockMoreThanHard.getAvailableBytes()).thenReturn(FormatUtils.parseSpaceSize("128MB")); - when(mockMoreThanHard.getCapacityBytes()).thenReturn(DISK_SIZE); - double probMoreThanHard = mReviewer.getProbability(mockMoreThanHard); - assertEquals(1.0 / 3, probMoreThanHard, 1e-6); - - // Hard limit reached - 0.0 - StorageDirView mockHardLimit = mock(StorageDirView.class); - when(mockHardLimit.getAvailableBytes()).thenReturn(HARD_LIMIT_BYTES); - when(mockHardLimit.getCapacityBytes()).thenReturn(DISK_SIZE); - double probHardLimit = mReviewer.getProbability(mockHardLimit); - assertEquals(0.0, probHardLimit, 1e-6); - - // Below hard limit - 0.0 - StorageDirView mockLessThanHard = mock(StorageDirView.class); - when(mockLessThanHard.getAvailableBytes()).thenReturn(HARD_LIMIT_BYTES - 1); - when(mockLessThanHard.getCapacityBytes()).thenReturn(DISK_SIZE); - double probLessThanHard = mReviewer.getProbability(mockLessThanHard); - assertEquals(0.0, probLessThanHard, 1e-6); - - // Full - 0.0 - StorageDirView mockFull = mock(StorageDirView.class); - when(mockFull.getAvailableBytes()).thenReturn(0L); - when(mockFull.getCapacityBytes()).thenReturn(DISK_SIZE); - double probFull = mReviewer.getProbability(mockFull); - assertEquals(0.0, probFull, 1e-6); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/reviewer/ReviewerFactoryTest.java b/core/server/worker/src/test/java/alluxio/worker/block/reviewer/ReviewerFactoryTest.java deleted file mode 100644 index 392d728308bb..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/reviewer/ReviewerFactoryTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.reviewer; - -import static org.junit.Assert.assertTrue; - -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; - -import org.junit.Test; - -/** - * Test {@link Reviewer.Factory} by passing different allocate strategy class names with alluxio - * conf and test if it generates the correct {@link Reviewer} instance. - * */ -public class ReviewerFactoryTest { - @Test - public void createProbabilisticBufferReviewer() { - Configuration.set(PropertyKey.WORKER_REVIEWER_CLASS, - ProbabilisticBufferReviewer.class.getName()); - Reviewer allocator = Reviewer.Factory.create(); - assertTrue(allocator instanceof ProbabilisticBufferReviewer); - } - - /** - * Tests the creation of the default reviewer via the - * {@link Reviewer.Factory#create()} method. - */ - @Test - public void createDefaultAllocator() { - // Create a new instance of Alluxio configuration with original properties to test the default - // behavior of create. - Reviewer allocator = Reviewer.Factory.create(); - assertTrue(allocator instanceof ProbabilisticBufferReviewer); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/block/stream/BlockWorkerDataReaderTest.java b/core/server/worker/src/test/java/alluxio/worker/block/stream/BlockWorkerDataReaderTest.java deleted file mode 100644 index 5bab8defc190..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/block/stream/BlockWorkerDataReaderTest.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.block.stream; - -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import alluxio.AlluxioTestDirectory; -import alluxio.AlluxioURI; -import alluxio.ConfigurationRule; -import alluxio.Constants; -import alluxio.Sessions; -import alluxio.client.block.stream.BlockWorkerDataReader; -import alluxio.client.block.stream.DataReader; -import alluxio.client.file.FileSystemContext; -import alluxio.client.file.URIStatus; -import alluxio.client.file.options.InStreamOptions; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.OpenFilePOptions; -import alluxio.grpc.ReadPType; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.UnderFileSystemConfiguration; -import alluxio.util.FileSystemOptionsUtils; -import alluxio.util.io.BufferUtils; -import alluxio.wire.BlockInfo; -import alluxio.wire.FileBlockInfo; -import alluxio.wire.FileInfo; -import alluxio.worker.block.BlockMasterClient; -import alluxio.worker.block.BlockMasterClientPool; -import alluxio.worker.block.BlockWorker; -import alluxio.worker.block.CreateBlockOptions; -import alluxio.worker.block.DefaultBlockWorker; -import alluxio.worker.block.MonoBlockStore; -import alluxio.worker.block.TieredBlockStore; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.file.FileSystemMasterClient; - -import com.google.common.collect.ImmutableMap; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.Collections; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Unit tests for {@link BlockWorkerDataReader}. - */ -public class BlockWorkerDataReaderTest { - // Use 2L block id here since block id includes the special sequence number info - private static final long BLOCK_ID = 2L; - private static final int CHUNK_SIZE = 128; - private static final long SESSION_ID = 10L; - private static final int LOCK_NUM = 5; - - private final InstancedConfiguration mConf = Configuration.modifiableGlobal(); - private final String mMemDir = - AlluxioTestDirectory.createTemporaryDirectory(Constants.MEDIUM_MEM).getAbsolutePath(); - - private BlockWorker mBlockWorker; - private BlockWorkerDataReader.Factory mDataReaderFactory; - private String mRootUfs; - - @Rule - public ConfigurationRule mConfigurationRule = - new ConfigurationRule(new ImmutableMap.Builder() - .put(PropertyKey.WORKER_TIERED_STORE_LEVELS, 1) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_ALIAS, Constants.MEDIUM_MEM) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_MEDIUMTYPE, Constants.MEDIUM_MEM) - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_QUOTA, "1GB") - .put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, mMemDir) - .put(PropertyKey.WORKER_TIERED_STORE_BLOCK_LOCKS, LOCK_NUM) - .put(PropertyKey.WORKER_RPC_PORT, 0) - .put(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, - AlluxioTestDirectory.createTemporaryDirectory("BlockWorkerDataReaderTest") - .getAbsolutePath()).build(), Configuration.modifiableGlobal()); - - @Before - public void before() throws Exception { - BlockMasterClient blockMasterClient = mock(BlockMasterClient.class); - BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool()); - when(blockMasterClientPool.createNewResource()).thenReturn(blockMasterClient); - - FileSystemMasterClient fileSystemMasterClient = mock(FileSystemMasterClient.class); - Sessions sessions = mock(Sessions.class); - - // Connect to the real UFS for UFS read testing - UfsManager ufsManager = mock(UfsManager.class); - mRootUfs = Configuration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); - UfsManager.UfsClient ufsClient = new UfsManager.UfsClient( - () -> UnderFileSystem.Factory.create(mRootUfs, - UnderFileSystemConfiguration.defaults(Configuration.global())), - new AlluxioURI(mRootUfs)); - when(ufsManager.get(anyLong())).thenReturn(ufsClient); - - TieredBlockStore tieredBlockStore = new TieredBlockStore(); - AtomicReference workerId = new AtomicReference<>(-1L); - MonoBlockStore blockStore = - new MonoBlockStore(tieredBlockStore, blockMasterClientPool, ufsManager, workerId); - mBlockWorker = new DefaultBlockWorker(blockMasterClientPool, fileSystemMasterClient, - sessions, blockStore, workerId); - - URIStatus dummyStatus = - new URIStatus(new FileInfo().setBlockIds(Collections.singletonList(BLOCK_ID))); - InStreamOptions options = - new InStreamOptions(dummyStatus, FileSystemOptionsUtils.openFileDefaults(mConf), mConf, - FileSystemContext.create(mConf)); - mDataReaderFactory = - new BlockWorkerDataReader.Factory(mBlockWorker, BLOCK_ID, CHUNK_SIZE, options); - } - - @Test - public void createWithBlockNotExists() { - assertThrows(IOException.class, () -> mDataReaderFactory.create(BLOCK_ID, 100)); - } - - @Test - public void create() throws Exception { - mBlockWorker.createBlock(SESSION_ID, BLOCK_ID, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - mBlockWorker.commitBlock(SESSION_ID, BLOCK_ID, true); - DataReader dataReader = mDataReaderFactory.create(100, 200); - assertEquals(100, dataReader.pos()); - } - - // See https://github.com/Alluxio/alluxio/issues/13255 - @Test - public void createAndCloseManyReader() throws Exception { - for (int i = 0; i < LOCK_NUM * 10; i++) { - long blockId = i; - mBlockWorker.createBlock(SESSION_ID, blockId, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - mBlockWorker.commitBlock(SESSION_ID, blockId, true); - InStreamOptions inStreamOptions = new InStreamOptions( - new URIStatus(new FileInfo().setBlockIds(Collections.singletonList(blockId))), - FileSystemOptionsUtils.openFileDefaults(mConf), mConf, FileSystemContext.create(mConf)); - mDataReaderFactory = - new BlockWorkerDataReader.Factory(mBlockWorker, blockId, CHUNK_SIZE, inStreamOptions); - DataReader dataReader = mDataReaderFactory.create(0, 100); - dataReader.close(); - } - } - - @Test - public void readChunkUfs() throws Exception { - // Write an actual file to UFS - String testFilePath = File.createTempFile("temp", null, new File(mRootUfs)).getAbsolutePath(); - byte[] buffer = BufferUtils.getIncreasingByteArray(CHUNK_SIZE * 5); - BufferUtils.writeBufferToFile(testFilePath, buffer); - - BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE * 5); - URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true) - .setUfsPath(testFilePath) - .setBlockIds(Collections.singletonList(BLOCK_ID)) - .setLength(CHUNK_SIZE * 5) - .setBlockSizeBytes(CHUNK_SIZE) - .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); - OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() - .setReadType(ReadPType.NO_CACHE).build(); - InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, mConf, - FileSystemContext.create(mConf)); - - BlockWorkerDataReader.Factory factory = new BlockWorkerDataReader - .Factory(mBlockWorker, BLOCK_ID, CHUNK_SIZE, options); - int len = CHUNK_SIZE * 3 / 2; - try (DataReader dataReader = factory.create(0, len)) { - validateBuffer(dataReader.readChunk(), 0, CHUNK_SIZE); - assertEquals(CHUNK_SIZE, dataReader.pos()); - validateBuffer(dataReader.readChunk(), CHUNK_SIZE, len - CHUNK_SIZE); - assertEquals(len, dataReader.pos()); - } - } - - @Test - public void readChunkFullFile() throws Exception { - int len = CHUNK_SIZE * 2; - mBlockWorker.createBlock(SESSION_ID, BLOCK_ID, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - try (BlockWriter writer = mBlockWorker.createBlockWriter(SESSION_ID, BLOCK_ID)) { - writer.append(BufferUtils.getIncreasingByteBuffer(len)); - } - mBlockWorker.commitBlock(SESSION_ID, BLOCK_ID, true); - DataReader dataReader = mDataReaderFactory.create(0, len); - validateBuffer(dataReader.readChunk(), 0, CHUNK_SIZE); - assertEquals(CHUNK_SIZE, dataReader.pos()); - validateBuffer(dataReader.readChunk(), CHUNK_SIZE, CHUNK_SIZE); - assertEquals(len, dataReader.pos()); - dataReader.close(); - } - - @Test - public void readChunkPartial() throws Exception { - int len = CHUNK_SIZE * 5; - mBlockWorker.createBlock(SESSION_ID, BLOCK_ID, 0, - new CreateBlockOptions(null, Constants.MEDIUM_MEM, 1)); - try (BlockWriter writer = mBlockWorker.createBlockWriter(SESSION_ID, BLOCK_ID)) { - writer.append(BufferUtils.getIncreasingByteBuffer(len)); - } - mBlockWorker.commitBlock(SESSION_ID, BLOCK_ID, true); - int start = len / 5 * 2; - int end = len / 5 * 4; - DataReader dataReader = mDataReaderFactory.create(start, end); - for (int s = start; s < end; s += CHUNK_SIZE) { - int currentLen = Math.min(CHUNK_SIZE, end - s); - validateBuffer(dataReader.readChunk(), s, currentLen); - } - } - - private void validateBuffer(DataBuffer buffer, int start, int len) { - byte[] bytes = new byte[buffer.readableBytes()]; - buffer.readBytes(bytes, 0, buffer.readableBytes()); - assertTrue(BufferUtils.equalIncreasingByteArray(start, len, bytes)); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/grpc/BlockReadHandlerTest.java b/core/server/worker/src/test/java/alluxio/worker/grpc/BlockReadHandlerTest.java deleted file mode 100644 index 5f90ec876c3b..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/grpc/BlockReadHandlerTest.java +++ /dev/null @@ -1,256 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.verify; - -import alluxio.Constants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.ReadRequest; -import alluxio.grpc.ReadResponse; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; -import alluxio.util.io.BufferUtils; -import alluxio.worker.block.DefaultBlockWorker; -import alluxio.worker.block.io.BlockReader; -import alluxio.worker.block.io.LocalFileBlockReader; - -import com.google.protobuf.ByteString; -import io.grpc.Status; -import io.grpc.StatusException; -import io.grpc.stub.ServerCallStreamObserver; -import io.grpc.stub.StreamObserver; -import io.netty.util.ResourceLeakDetector; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; -import org.mockito.stubbing.Answer; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.TimeoutException; - -/** - * Unit tests for {@link BlockReadHandler}. - */ -public class BlockReadHandlerTest { - private static final long CHUNK_SIZE = - Configuration.getBytes(PropertyKey.USER_STREAMING_READER_CHUNK_SIZE_BYTES); - private final Random mRandom = new Random(); - - private BlockReadHandler mReadHandler; - private ServerCallStreamObserver mResponseObserver; - private final List mResponses = new ArrayList<>(); - private boolean mResponseCompleted; - private Throwable mError; - private BlockReader mBlockReader; - private File mFile; - - @Rule - public TemporaryFolder mTestFolder = new TemporaryFolder(); - - @Before - public void before() throws Exception { - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.ADVANCED); - - mFile = mTestFolder.newFile(); - mBlockReader = new LocalFileBlockReader(mFile.getPath()); - mResponseObserver = Mockito.mock(ServerCallStreamObserver.class); - Mockito.when(mResponseObserver.isReady()).thenReturn(true); - doAnswer(args -> { - mResponseCompleted = true; - return null; - }).when(mResponseObserver).onCompleted(); - doAnswer(args -> { - mResponseCompleted = true; - mError = args.getArgument(0, Throwable.class); - return null; - }).when(mResponseObserver).onError(any(Throwable.class)); - doAnswer((args) -> { - // make a copy of response data before it is released - mResponses.add(ReadResponse.parseFrom( - args.getArgument(0, ReadResponse.class).toByteString())); - return null; - }).when(mResponseObserver).onNext(any(ReadResponse.class)); - DefaultBlockWorker blockWorker = Mockito.mock(DefaultBlockWorker.class); - Mockito.when(blockWorker - .createBlockReader(anyLong(), anyLong(), anyLong(), anyBoolean(), any())) - .thenAnswer((Answer) invocationOnMock -> { - long offset = invocationOnMock.getArgument(2); - ((FileChannel) mBlockReader.getChannel()).position(offset); - return mBlockReader; - }); - mReadHandler = new BlockReadHandler(GrpcExecutors.BLOCK_READER_EXECUTOR, blockWorker, - mResponseObserver, false); - } - - /** - * Reads all bytes of a file. - */ - @Test - public void readFullFile() throws Exception { - long checksumExpected = populateInputFile(CHUNK_SIZE * 10, 0, CHUNK_SIZE * 10 - 1); - mReadHandler.onNext(buildReadRequest(0, CHUNK_SIZE * 10)); - checkAllReadResponses(mResponses, checksumExpected); - } - - /** - * Reads a sub-region of a file. - */ - @Test - public void readPartialFile() throws Exception { - long start = 3; - long end = CHUNK_SIZE * 10 - 99; - long checksumExpected = populateInputFile(CHUNK_SIZE * 10, start, end); - mReadHandler.onNext(buildReadRequest(start, end + 1 - start)); - checkAllReadResponses(mResponses, checksumExpected); - } - - /** - * Fails if the read request tries to read an empty file. - */ - @Test - public void readEmptyFile() throws Exception { - populateInputFile(0, 0, 0); - mReadHandler.onNext(buildReadRequest(0, 0)); - checkErrorCode(mResponseObserver, Status.Code.INVALID_ARGUMENT); - } - - /** - * Cancels the read request immediately after the read request is sent. - */ - @Test - public void cancelRequest() throws Exception { - long fileSize = CHUNK_SIZE * 100 + 1; - populateInputFile(fileSize, 0, fileSize - 1); - mReadHandler.onNext(buildReadRequest(0, fileSize)); - mReadHandler.onCompleted(); - - CommonUtils.waitFor("response", () -> mResponseCompleted || mError != null, - WaitForOptions.defaults().setTimeoutMs(Constants.MINUTE_MS)); - verify((StreamObserver) mResponseObserver).onCompleted(); - } - - @Test - public void ErrorReceived() { - mReadHandler.onError(new IOException("test error")); - } - - @Test - public void ErrorReceivedAfterRequest() throws Exception { - populateInputFile(CHUNK_SIZE * 10, 0, CHUNK_SIZE * 10 - 1); - mReadHandler.onNext(buildReadRequest(0, CHUNK_SIZE * 10)); - mReadHandler.onError(new IOException("test error")); - } - - @Test - public void readFailure() throws Exception { - long fileSize = CHUNK_SIZE * 10 + 1; - populateInputFile(0, 0, fileSize - 1); - mBlockReader.close(); - mReadHandler.onNext(buildReadRequest(0, fileSize)); - checkErrorCode(mResponseObserver, Status.Code.FAILED_PRECONDITION); - } - - /** - * Populates the input file, also computes the checksum for part of the file. - * - * @param length the length of the file - * @param start the start position to compute the checksum - * @param end the last position to compute the checksum - * @return the checksum - */ - private long populateInputFile(long length, long start, long end) throws Exception { - long checksum = 0; - long pos = 0; - if (length > 0) { - try (FileOutputStream fileOutputStream = new FileOutputStream(mFile)) { - while (length > 0) { - byte[] buffer = new byte[(int) Math.min(length, Constants.MB)]; - mRandom.nextBytes(buffer); - for (byte b : buffer) { - if (pos >= start && pos <= end) { - checksum += BufferUtils.byteToInt(b); - } - pos++; - } - fileOutputStream.write(buffer); - length -= buffer.length; - } - } - } - return checksum; - } - - /** - * Checks all the read responses. - */ - private void checkAllReadResponses(List responses, - long checksumExpected) throws Exception { - long checksumActual = 0; - CommonUtils.waitFor("response", () -> mResponseCompleted || mError != null, - WaitForOptions.defaults().setTimeoutMs(Constants.MINUTE_MS)); - for (ReadResponse readResponse : responses) { - if (readResponse == null) { - Assert.fail(); - break; - } - assertTrue(readResponse.hasChunk()); - assertTrue(readResponse.getChunk().hasData()); - ByteString buffer = readResponse.getChunk().getData(); - if (buffer != null) { - for (byte b : buffer) { - checksumActual += BufferUtils.byteToInt(b); - } - } - } - assertEquals(checksumExpected, checksumActual); - } - - /** - * Checks an error is returned with given code. - * - * @param responseObserver the response stream observer - * @param code the expected error code - */ - private void checkErrorCode(final StreamObserver responseObserver, - Status.Code code) throws TimeoutException, InterruptedException { - CommonUtils.waitFor("response", () -> mResponseCompleted || mError != null, - WaitForOptions.defaults().setTimeoutMs(Constants.MINUTE_MS)); - ArgumentCaptor captor = ArgumentCaptor.forClass(Throwable.class); - verify(responseObserver).onError(captor.capture()); - Throwable t = captor.getValue(); - assertTrue(t instanceof StatusException); - assertEquals(code, ((StatusException) t).getStatus().getCode()); - } - - private ReadRequest buildReadRequest(long offset, long len) { - return ReadRequest.newBuilder().setBlockId(1L).setOffset(offset).setLength(len) - .setChunkSize(CHUNK_SIZE).build(); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/grpc/BlockWriteHandlerTest.java b/core/server/worker/src/test/java/alluxio/worker/grpc/BlockWriteHandlerTest.java deleted file mode 100644 index 39f52533a08c..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/grpc/BlockWriteHandlerTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import static org.junit.Assert.assertTrue; - -import alluxio.grpc.RequestType; -import alluxio.util.CommonUtils; -import alluxio.worker.block.BlockWorker; -import alluxio.worker.block.NoopBlockWorker; -import alluxio.worker.block.io.BlockWriter; -import alluxio.worker.block.io.LocalFileBlockWriter; - -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -/** - * Unit tests for {@link BlockWriteHandler}. - */ -public final class BlockWriteHandlerTest extends AbstractWriteHandlerTest { - private BlockWorker mBlockWorker; - private BlockWriter mBlockWriter; - private File mFile; - - @Before - public void before() throws Exception { - mFile = mTestFolder.newFile(); - mBlockWorker = new NoopBlockWorker() { - @Override - public BlockWriter createBlockWriter(long sessionId, long blockId) { - return mBlockWriter; - } - }; - mBlockWriter = new LocalFileBlockWriter(mFile.getPath()); - mResponseObserver = Mockito.mock(StreamObserver.class); - mWriteHandler = new BlockWriteHandler(mBlockWorker, mResponseObserver, mUserInfo, false); - setupResponseTrigger(); - } - - @Test - public void writeFailure() throws Exception { - mWriteHandler.write(newWriteRequestCommand(0)); - mBlockWriter.close(); - mWriteHandler.write(newWriteRequest(newDataBuffer(CHUNK_SIZE))); - waitForResponses(); - checkErrorCode(mResponseObserver, Status.Code.FAILED_PRECONDITION); - } - - @Test - public void getLocation() throws Exception { - mWriteHandler.write(newWriteRequestCommand(0)); - CommonUtils.waitFor("location is not null", () -> !"null".equals(mWriteHandler.getLocation())); - assertTrue(mWriteHandler.getLocation().startsWith("temp-block-")); - } - - @Override - protected RequestType getWriteRequestType() { - return RequestType.ALLUXIO_BLOCK; - } - - @Override - protected InputStream getWriteDataStream() throws IOException { - return new FileInputStream(mFile); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/grpc/ShortCircuitBlockReadHandlerTest.java b/core/server/worker/src/test/java/alluxio/worker/grpc/ShortCircuitBlockReadHandlerTest.java deleted file mode 100644 index bf617a6dceeb..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/grpc/ShortCircuitBlockReadHandlerTest.java +++ /dev/null @@ -1,316 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import static alluxio.AlluxioMockUtil.setStaticInternalState; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import alluxio.ConfigurationRule; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.OpenLocalBlockRequest; -import alluxio.grpc.OpenLocalBlockResponse; -import alluxio.underfs.UfsManager; -import alluxio.util.io.PathUtils; -import alluxio.worker.block.BlockMasterClient; -import alluxio.worker.block.BlockMasterClientPool; -import alluxio.worker.block.BlockStore; -import alluxio.worker.block.BlockStoreType; -import alluxio.worker.block.CreateBlockOptions; -import alluxio.worker.block.MonoBlockStore; -import alluxio.worker.block.TieredBlockStore; -import alluxio.worker.block.io.BlockWriter; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.grpc.Status; -import io.grpc.StatusException; -import io.grpc.stub.StreamObserver; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; - -public class ShortCircuitBlockReadHandlerTest { - // templates used in current configuration - private static final PropertyKey.Template ALIAS_TEMPLATE = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS; - private static final PropertyKey.Template PATH_TEMPLATE = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH; - private static final PropertyKey.Template QUOTA_TEMPLATE = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA; - private static final PropertyKey.Template MEDIUM_TEMPLATE = - PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_MEDIUMTYPE; - - // test configuration values - private static final String TIER_0_ALIAS = "ALIAS_0"; - private static final String TIER_1_ALIAS = "ALIAS_1"; - private static final String WORKER_DATA_FOLDER = "TestWorker"; - private static final String WORKER_TMP_FOLDER = "TestTmpWorker"; - - // default session id - private static final long SESSION_ID = 1L; - - // default block id - private static final long BLOCK_ID = 2L; - - // first tier id - private static final int FIRST_TIER = 0; - - // second tier id - private static final int SECOND_TIER = 1; - - @Rule - public ConfigurationRule mConfiguration = new ConfigurationRule( - new ImmutableMap.Builder() - // disable paging for this test - .put(PropertyKey.WORKER_BLOCK_STORE_TYPE, BlockStoreType.FILE) - // use 2 tiers to test possible moves of blocks - .put(PropertyKey.WORKER_TIERED_STORE_LEVELS, 2) - // disable alignment so that no space needs to be reserved - // on directories - .put(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED, false) - .put(PropertyKey.WORKER_DATA_FOLDER, WORKER_DATA_FOLDER) - .put(PropertyKey.WORKER_DATA_TMP_FOLDER, WORKER_TMP_FOLDER) - // tier configurations - .put(ALIAS_TEMPLATE.format(0), TIER_0_ALIAS) - .put(ALIAS_TEMPLATE.format(1), TIER_1_ALIAS) - .put(QUOTA_TEMPLATE.format(0), "1024") - .put(QUOTA_TEMPLATE.format(1), "1024") - .put(MEDIUM_TEMPLATE.format(0), "MEM") - .put(MEDIUM_TEMPLATE.format(1), "SSD") - // the default reviewer is ProbabilisticBufferReviewer, - // it stops accepting new blocks into it when free space is under a threshold - .put(PropertyKey.WORKER_REVIEWER_CLASS, - "alluxio.worker.block.reviewer.AcceptingReviewer") - .build(), - Configuration.modifiableGlobal() - ); - - @Rule - public TemporaryFolder mFolder = new TemporaryFolder(); - - // each tier's storage directory - private File mTier0Dir; - private File mTier1Dir; - - // mocked response observer to interact with the handler - private final TestResponseObserver mResponseObserver = new TestResponseObserver(); - - // local store that backs the short circuit read - private BlockStore mBlockStore; - private ShortCircuitBlockReadHandler mTestHandler; - - @Before - public void before() throws Exception { - // set up storage tier directories - mTier0Dir = mFolder.newFolder(); - mTier1Dir = mFolder.newFolder(); - mConfiguration.set(PATH_TEMPLATE.format(0), mTier0Dir.getAbsolutePath()); - mConfiguration.set(PATH_TEMPLATE.format(1), mTier1Dir.getAbsolutePath()); - - // set up local storage - // todo(yangchen): BlockStore should not worry about the logic of reporting to master, - // could leave that work to BlockWorker so that we can get rid of these irrelevant mocks - BlockMasterClientPool pool = mock(BlockMasterClientPool.class); - BlockMasterClient cli = mock(BlockMasterClient.class); - when(pool.createNewResource()).thenReturn(cli); - when(pool.acquire()).thenReturn(cli); - - setStaticInternalState(TieredBlockStore.class, "REMOVE_BLOCK_TIMEOUT_MS", 1000L); - TieredBlockStore tieredBlockStore = new TieredBlockStore(); - - mBlockStore = new MonoBlockStore(tieredBlockStore, pool, - mock(UfsManager.class), new AtomicReference<>(1L)); - - mTestHandler = new ShortCircuitBlockReadHandler(mBlockStore, mResponseObserver); - } - - @Test - public void accessNonExistentBlock() { - // we started with empty directories without any block - OpenLocalBlockRequest request = createRequest(BLOCK_ID, false); - mTestHandler.onNext(request); - - // check that we get a proper error and no response - assertTrue(mResponseObserver.getResponses().isEmpty()); - assertFalse(mResponseObserver.isCompleted()); - Throwable t = mResponseObserver.getError(); - assertTrue(t instanceof StatusException - && ((StatusException) t).getStatus().getCode() == Status.Code.NOT_FOUND); - } - - @Test - public void accessBlockNoPromote() throws Exception { - accessBlock(false); - } - - @Test - public void accessBlockPromote() throws Exception { - accessBlock(true); - } - - @Test - public void accessBlockPinned() throws Exception { - createLocalBlock(SESSION_ID, BLOCK_ID, FIRST_TIER); - - // accessing a block through short circuit will pin the block - OpenLocalBlockRequest request = createRequest(BLOCK_ID, false); - mTestHandler.onNext(request); - - // block 2 should be pinned now - // and an attempt to remove the block from local store should fail - assertThrows(Exception.class, () -> mBlockStore.removeBlock(3L, BLOCK_ID)); - } - - @Test(timeout = 2000) - public void unpinBlockOnError() throws Exception { - createLocalBlock(SESSION_ID, BLOCK_ID, FIRST_TIER); - - OpenLocalBlockRequest request = createRequest(BLOCK_ID, false); - mTestHandler.onNext(request); - mTestHandler.onError(new RuntimeException()); - - // block should be unpinned and another session can remove it - long anotherSessionId = 4L; - mBlockStore.removeBlock(anotherSessionId, BLOCK_ID); - } - - @Test(timeout = 2000) - public void errorReAccessingBlock() throws Exception { - createLocalBlock(SESSION_ID, BLOCK_ID, FIRST_TIER); - - OpenLocalBlockRequest request = createRequest(BLOCK_ID, false); - - // first call should be ok - mTestHandler.onNext(request); - assertNull(mResponseObserver.getError()); - - // second call should cause an error - mTestHandler.onNext(request); - - assertFalse(mResponseObserver.isCompleted()); - assertNotNull(mResponseObserver.getError()); - - // the block should be unpinned - long anotherSessionId = 3L; - mBlockStore.removeBlock(anotherSessionId, BLOCK_ID); - } - - private static OpenLocalBlockRequest createRequest(long blockId, boolean promote) { - return OpenLocalBlockRequest.newBuilder().setBlockId(blockId).setPromote(promote).build(); - } - - private void accessBlock(boolean promote) throws Exception { - // create a block in second tier(tier1) so that promotion will move block - createLocalBlock(SESSION_ID, BLOCK_ID, SECOND_TIER); - - // access the new block via short circuit read - OpenLocalBlockRequest request = createRequest(BLOCK_ID, promote); - mTestHandler.onNext(request); - mTestHandler.onCompleted(); - - // if promote, the block should be moved to tier 1 - // real block path is // - String expectedRootDirPath = (promote ? mTier0Dir : mTier1Dir).getAbsolutePath(); - String expectedBlockPath = PathUtils.concatPath( - expectedRootDirPath, WORKER_DATA_FOLDER, BLOCK_ID); - - // check that the block file is present on disk - assertTrue(Files.exists(Paths.get(expectedBlockPath))); - - // check the response is correct - assertNull(mResponseObserver.getError()); - assertTrue(mResponseObserver.isCompleted()); - - List responses = mResponseObserver.getResponses(); - assertEquals(1, responses.size()); - OpenLocalBlockResponse response = responses.get(0); - - // check that we get the expected path where the block lies - assertEquals(expectedBlockPath, response.getPath()); - } - - // create and commit a block in LocalStorage in the specified location - // so that further request can read the block via short circuit - private void createLocalBlock( - long sessionId, long blockId, int tier) throws Exception { - mBlockStore.createBlock( - sessionId, - blockId, - tier, - new CreateBlockOptions(null, null, 64)); - - byte[] data = new byte[64]; - Arrays.fill(data, (byte) 1); - ByteBuffer buf = ByteBuffer.wrap(data); - try (BlockWriter writer = mBlockStore.createBlockWriter(sessionId, blockId)) { - writer.append(buf); - } - - mBlockStore.commitBlock(sessionId, blockId, false); - } - - // a testing response observer that keeps track of the interactions with - // the handler - private static class TestResponseObserver implements StreamObserver { - // keeps track of received response - private final List mResponses = new ArrayList<>(); - // keeps track of received error - private Throwable mError = null; - // keeps track of completion - private boolean mCompleted = false; - - @Override - public void onNext(OpenLocalBlockResponse value) { - mResponses.add(value); - } - - @Override - public void onError(Throwable t) { - mError = t; - } - - @Override - public void onCompleted() { - mCompleted = true; - } - - public List getResponses() { - return ImmutableList.copyOf(mResponses); - } - - public Throwable getError() { - return mError; - } - - public boolean isCompleted() { - return mCompleted; - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/grpc/ShortCircuitBlockWriteHandlerTest.java b/core/server/worker/src/test/java/alluxio/worker/grpc/ShortCircuitBlockWriteHandlerTest.java deleted file mode 100644 index e751a64522bb..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/grpc/ShortCircuitBlockWriteHandlerTest.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import alluxio.grpc.CreateLocalBlockRequest; -import alluxio.grpc.CreateLocalBlockResponse; -import alluxio.util.io.PathUtils; -import alluxio.worker.block.CreateBlockOptions; -import alluxio.worker.block.NoopBlockWorker; - -import com.amazonaws.annotation.NotThreadSafe; -import com.google.common.collect.ImmutableList; -import io.grpc.stub.StreamObserver; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class ShortCircuitBlockWriteHandlerTest { - - private static final long BLOCK_ID = 1L; - - @Rule - public TemporaryFolder mTempFolder = new TemporaryFolder(); - - private TestResponseObserver mResponseObserver; - private TestBlockWorker mBlockWorker; - private ShortCircuitBlockWriteHandler mHandler; - private CreateLocalBlockRequest mRequest; - - @Before - public void before() throws Exception { - File tmpDir = mTempFolder.newFolder(); - mBlockWorker = new TestBlockWorker(tmpDir.getAbsolutePath()); - - mResponseObserver = new TestResponseObserver(); - - mHandler = new ShortCircuitBlockWriteHandler(mBlockWorker, mResponseObserver); - - mRequest = CreateLocalBlockRequest - .newBuilder() - .setBlockId(BLOCK_ID) - .build(); - } - - @Test - public void createBlock() { - // request to create a new local block - mHandler.onNext(mRequest); - - // verify that we get one response from the handler - // that contains the right path to the local block file - assertEquals(1, mResponseObserver.getResponses().size()); - assertNull(mResponseObserver.getError()); - assertFalse(mResponseObserver.isCompleted()); - - String path = mResponseObserver.getResponses().get(0).getPath(); - // verify that the local file exists - assertTrue(Files.exists(Paths.get(path))); - - // verify that the blockId is recorded into the temp blocks - // but is not publicly available yet - assertTrue(mBlockWorker.isTempBlockCreated(BLOCK_ID)); - assertFalse(mBlockWorker.isBlockCommitted(BLOCK_ID)); - - mHandler.onCompleted(); - - // verify that the block is committed and publicly available - assertTrue(mBlockWorker.isBlockCommitted(BLOCK_ID)); - } - - @Test - public void nextRequestWithoutCommitting() { - mHandler.onNext(mRequest); - - long anotherBlockId = 2L; - CreateLocalBlockRequest anotherRequest = CreateLocalBlockRequest - .newBuilder() - .setBlockId(anotherBlockId) - .build(); - // this request should fail and abort the previous session - mHandler.onNext(anotherRequest); - - // verify that we get an error and both blocks are aborted - assertNotNull(mResponseObserver.getError()); - assertFalse(mResponseObserver.isCompleted()); - assertFalse(mBlockWorker.isTempBlockCreated(BLOCK_ID)); - assertFalse(mBlockWorker.isTempBlockCreated(anotherBlockId)); - } - - @Test - public void cannotReserveSpaceForNonExistingBlock() { - mRequest = mRequest.toBuilder().setOnlyReserveSpace(true).build(); - // this request should fail as the block is not created yet - mHandler.onNext(mRequest); - - assertTrue(mResponseObserver.getResponses().isEmpty()); - assertFalse(mResponseObserver.isCompleted()); - assertNotNull(mResponseObserver.getError()); - } - - @Test - public void abortBlockOnCancel() { - mHandler.onNext(mRequest); - mHandler.onCancel(); - - // block should be aborted - assertFalse(mBlockWorker.isBlockCommitted(BLOCK_ID)); - assertFalse(mBlockWorker.isTempBlockCreated(BLOCK_ID)); - } - - @Test - public void abortBlockOnError() { - mHandler.onNext(mRequest); - - // now the temp block is created - assertTrue(mBlockWorker.isTempBlockCreated(BLOCK_ID)); - - mHandler.onError(new RuntimeException()); - - // now the block should be aborted and - // the session cleaned up - assertFalse(mBlockWorker.isTempBlockCreated(BLOCK_ID)); - assertFalse(mBlockWorker.isBlockCommitted(BLOCK_ID)); - - // verify that we get the correct response - assertFalse(mResponseObserver.isCompleted()); - assertNotNull(mResponseObserver.getError()); - } - - private static final class TestResponseObserver - implements StreamObserver { - - private final List mReceivedResponses = new ArrayList<>(); - private Throwable mError = null; - private boolean mCompleted = false; - - @Override - public void onNext(CreateLocalBlockResponse value) { - mReceivedResponses.add(value); - } - - @Override - public void onError(Throwable t) { - mError = t; - } - - @Override - public void onCompleted() { - mCompleted = true; - } - - public List getResponses() { - return ImmutableList.copyOf(mReceivedResponses); - } - - public Throwable getError() { - return mError; - } - - public boolean isCompleted() { - return mCompleted; - } - } - - /** - * A test block worker that interacts with ShortCircuitBlockWriteHandler. - * It does the minimum necessary book-keeping and file I/O to make short circuit - * write possible. - */ - @NotThreadSafe - private static final class TestBlockWorker extends NoopBlockWorker { - // path to the working directory of - // this worker - private final String mRootDirPath; - - // keeps track of - // temporary block -> owning session - private Map mTempBlocks; - // public blocks - // an invariant is maintained that no block can be in both mTempBlocks and - // mPublicBlocks - private final List mPublicBlocks; - - private TestBlockWorker(String rootDirectory) { - mRootDirPath = rootDirectory; - mTempBlocks = new HashMap<>(); - mPublicBlocks = new ArrayList<>(); - } - - @Override - public void abortBlock(long sessionId, long blockId) { - if (mTempBlocks.containsKey(blockId) && mTempBlocks.get(blockId).equals(sessionId)) { - mTempBlocks.remove(blockId); - } - } - - @Override - public void commitBlock(long sessionId, long blockId, boolean pinOnCreate) { - if (mTempBlocks.containsKey(blockId) && mTempBlocks.get(blockId).equals(sessionId)) { - mTempBlocks.remove(blockId); - mPublicBlocks.add(blockId); - return; - } - throw new RuntimeException( - String.format("Block %d does not exists for session %d", blockId, sessionId)); - } - - @Override - public String createBlock(long sessionId, long blockId, int tier, - CreateBlockOptions createBlockOptions) { - if (mTempBlocks.containsKey(blockId) && mPublicBlocks.contains(blockId)) { - throw new RuntimeException(String.format("Block %d already exists", blockId)); - } - mTempBlocks.put(blockId, sessionId); - String filePath = getPath(blockId); - File f = new File(filePath); - try { - if (!f.createNewFile()) { - throw new RuntimeException("Block File Already Exists"); - } - } catch (IOException e) { - throw new RuntimeException("fail creating new file"); - } - return filePath; - } - - @Override - public void requestSpace(long sessionId, long blockId, long additionalBytes) { - if (mTempBlocks.containsKey(blockId) && mTempBlocks.get(blockId).equals(sessionId)) { - // no need to do anything for this class - return; - } - throw new RuntimeException( - String.format("Block %d does not exists for session %d", blockId, sessionId)); - } - - @Override - public void cleanupSession(long sessionId) { - // remove all temp blocks owned by the session - mTempBlocks = mTempBlocks - .entrySet() - .stream() - .filter(entry -> !entry.getValue().equals(sessionId)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - public boolean isTempBlockCreated(long blockId) { - return mTempBlocks.containsKey(blockId); - } - - public boolean isBlockCommitted(long blockId) { - return mPublicBlocks.contains(blockId); - } - - // a block's file path is / - private String getPath(long blockId) { - return PathUtils.concatPath(mRootDirPath, blockId); - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/grpc/UfsFallbackBlockWriteHandlerTest.java b/core/server/worker/src/test/java/alluxio/worker/grpc/UfsFallbackBlockWriteHandlerTest.java deleted file mode 100644 index 9bf7d911f566..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/grpc/UfsFallbackBlockWriteHandlerTest.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.grpc; - -import static org.junit.Assert.assertTrue; - -import alluxio.AlluxioTestDirectory; -import alluxio.AlluxioURI; -import alluxio.ConfigurationRule; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.grpc.RequestType; -import alluxio.grpc.WriteRequest; -import alluxio.network.protocol.databuffer.DataBuffer; -import alluxio.proto.dataserver.Protocol; -import alluxio.underfs.UfsManager; -import alluxio.underfs.UnderFileSystem; -import alluxio.underfs.options.CreateOptions; -import alluxio.util.CommonUtils; -import alluxio.worker.block.BlockMasterClientPool; -import alluxio.worker.block.BlockStore; -import alluxio.worker.block.CreateBlockOptions; -import alluxio.worker.block.DefaultBlockWorker; -import alluxio.worker.block.MonoBlockStore; -import alluxio.worker.block.TieredBlockStore; -import alluxio.worker.block.io.BlockWriter; - -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import io.netty.buffer.ByteBuf; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.concurrent.atomic.AtomicReference; - -public class UfsFallbackBlockWriteHandlerTest extends AbstractWriteHandlerTest { - private static final long TEST_SESSION_ID = 123L; - private static final long TEST_WORKER_ID = 456L; - private static final int PARTIAL_WRITTEN = 512; - - private OutputStream mOutputStream; - private BlockStore mBlockStore; - /** The file used to hold the data written by the test. */ - private File mFile; - private long mPartialChecksum; - - @Rule - public ConfigurationRule mConfigurationRule = - new ConfigurationRule(new HashMap() { - { - put(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, - AlluxioTestDirectory.createTemporaryDirectory( - "UfsFallbackBlockWriteHandlerTest-RootUfs") - .getAbsolutePath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, AlluxioTestDirectory - .createTemporaryDirectory("UfsFallbackBlockWriteHandlerTest-WorkerDataFolder") - .getAbsolutePath()); - put(PropertyKey.WORKER_TIERED_STORE_LEVELS, 1); - } - }, Configuration.modifiableGlobal()); - - @Before - public void before() throws Exception { - mFile = mTestFolder.newFile(); - mOutputStream = new FileOutputStream(mFile); - UfsManager ufsManager = Mockito.mock(UfsManager.class); - AtomicReference workerId = new AtomicReference<>(-1L); - mBlockStore = - new MonoBlockStore(new TieredBlockStore(), Mockito.mock(BlockMasterClientPool.class), - ufsManager, workerId); - DefaultBlockWorker blockWorker = Mockito.mock(DefaultBlockWorker.class); - Mockito.when(blockWorker.getWorkerId()).thenReturn(new AtomicReference<>(TEST_WORKER_ID)); - Mockito.when(blockWorker.getBlockStore()).thenReturn(mBlockStore); - UnderFileSystem mockUfs = Mockito.mock(UnderFileSystem.class); - - UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> mockUfs, AlluxioURI.EMPTY_URI); - Mockito.when(ufsManager.get(Mockito.anyLong())).thenReturn(ufsClient); - Mockito.when(mockUfs.createNonexistingFile(Mockito.anyString(), - Mockito.any(CreateOptions.class))).thenReturn(mOutputStream) - .thenReturn(new FileOutputStream(mFile, true)); - - mResponseObserver = Mockito.mock(StreamObserver.class); - mWriteHandler = new UfsFallbackBlockWriteHandler(blockWorker, ufsManager, mResponseObserver, - mUserInfo, false); - setupResponseTrigger(); - - // create a partial block in block store first - mBlockStore.createBlock(TEST_SESSION_ID, TEST_BLOCK_ID, 0, - new CreateBlockOptions(null, "MEM", CHUNK_SIZE)); - BlockWriter writer = mBlockStore.createBlockWriter(TEST_SESSION_ID, TEST_BLOCK_ID); - DataBuffer buffer = newDataBuffer(PARTIAL_WRITTEN); - mPartialChecksum = getChecksum(buffer); - writer.append((ByteBuf) buffer.getNettyOutput()); - writer.close(); - } - - @After - public void after() throws Exception { - mOutputStream.close(); - } - - @Test - public void noTempBlockFound() throws Exception { - // remove the block partially created - mBlockStore.abortBlock(TEST_SESSION_ID, TEST_BLOCK_ID); - mWriteHandler.write(newFallbackInitRequest(PARTIAL_WRITTEN)); - waitForResponses(); - checkErrorCode(mResponseObserver, Status.Code.INTERNAL); - } - - @Test - public void tempBlockWritten() throws Exception { - DataBuffer buffer = newDataBuffer(CHUNK_SIZE); - long checksum = mPartialChecksum + getChecksum(buffer); - mWriteHandler.write(newFallbackInitRequest(PARTIAL_WRITTEN)); - mWriteHandler.write(newWriteRequest(buffer)); - mWriteHandler.onCompleted(); - waitForResponses(); - checkComplete(mResponseObserver); - checkWriteData(checksum, PARTIAL_WRITTEN + CHUNK_SIZE); - } - - @Test - public void getLocation() throws Exception { - mWriteHandler.write(newFallbackInitRequest(0)); - CommonUtils.waitFor("location is not null", () -> !"null".equals(mWriteHandler.getLocation())); - assertTrue(mWriteHandler.getLocation().startsWith("/.alluxio_ufs_blocks")); - } - - protected WriteRequest newFallbackInitRequest(long bytesInBlockStore) { - Protocol.CreateUfsBlockOptions createUfsBlockOptions = - newWriteRequestCommand(0).getCommand().getCreateUfsBlockOptions().toBuilder() - .setBytesInBlockStore(bytesInBlockStore) - .build(); - return super.newWriteRequestCommand(0).toBuilder().setCommand( - super.newWriteRequestCommand(0).getCommand().toBuilder() - .setCreateUfsBlockOptions(createUfsBlockOptions)).build(); - } - - @Override - protected WriteRequest newWriteRequestCommand(long offset) { - Protocol.CreateUfsBlockOptions createUfsBlockOptions = - Protocol.CreateUfsBlockOptions.newBuilder().setMountId(TEST_MOUNT_ID).setFallback(true) - .build(); - return super.newWriteRequestCommand(offset).toBuilder().setCommand( - super.newWriteRequestCommand(offset).getCommand().toBuilder() - .setCreateUfsBlockOptions(createUfsBlockOptions)).build(); - } - - @Override - protected RequestType getWriteRequestType() { - return RequestType.UFS_FALLBACK_BLOCK; - } - - @Override - protected InputStream getWriteDataStream() throws IOException { - return new FileInputStream(mFile); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/page/BlockPageEvictorTest.java b/core/server/worker/src/test/java/alluxio/worker/page/BlockPageEvictorTest.java deleted file mode 100644 index 908fd4b7fe20..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/page/BlockPageEvictorTest.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static org.junit.Assert.assertEquals; - -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.evictor.CacheEvictor; -import alluxio.client.file.cache.evictor.CacheEvictorOptions; -import alluxio.client.file.cache.evictor.FIFOCacheEvictor; -import alluxio.client.file.cache.evictor.LFUCacheEvictor; -import alluxio.client.file.cache.evictor.LRUCacheEvictor; -import alluxio.util.CommonUtils; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.LongStream; -import java.util.stream.Stream; - -@RunWith(Parameterized.class) -public class BlockPageEvictorTest { - private static final int NUM_ACCESSES = 100000; - @Parameterized.Parameter(0) - public Class mEvictorType; - - @Parameterized.Parameter(1) - public int mNumBlocks; - - @Parameterized.Parameter(2) - public int mNumPages; - - @Parameterized.Parameter(3) - public int mNumPinnedBlocks; - - private CacheEvictor mInner; - - private BlockPageEvictor mEvictor; - - @Parameterized.Parameters( - name = "Evictor: {0}, Blocks: {1}, Pages: {2}, PinnedBlocks: {3}") - public static Collection data() { - List> evictors = ImmutableList.of( - LRUCacheEvictor.class, - LFUCacheEvictor.class, - FIFOCacheEvictor.class - ); - List numBlocks = ImmutableList.of(10, 20); - List numPages = ImmutableList.of(10, 20); - List numPinnedBlocks = ImmutableList.of(1, 2, 5); - return Lists.cartesianProduct(evictors, numBlocks, numPages, numPinnedBlocks) - .stream() - .map(list -> list.toArray(new Object[0])) - .collect(Collectors.toList()); - } - - @Before - public void setup() { - mInner = CommonUtils.createNewClassInstance( - mEvictorType, - new Class[] {CacheEvictorOptions.class}, - new Object[] {new CacheEvictorOptions()}); - mEvictor = new BlockPageEvictor(mInner); - } - - @Test - public void noPinnedBlocks() { - runAndInspect(NUM_ACCESSES, mNumBlocks, mNumPages, () -> mInner.evict()); - } - - @Test - public void pinnedBlocks() { - Set pinnedBlocks = - LongStream.range(0, mNumPinnedBlocks).boxed().collect(Collectors.toSet()); - Set pinnedBlockIdStr = - pinnedBlocks.stream().map(String::valueOf).collect(Collectors.toSet()); - for (long blockId : pinnedBlocks) { - mEvictor.addPinnedBlock(blockId); - } - runAndInspect(NUM_ACCESSES, mNumBlocks, mNumPages, - () -> mInner.evictMatching( - pageId -> !pinnedBlockIdStr.contains(pageId.getFileId()))); - - for (long blockId : pinnedBlocks) { - mEvictor.removePinnedBlock(blockId); - } - runAndInspect(NUM_ACCESSES, mNumBlocks, mNumPages, () -> mInner.evict()); - } - - private void runAndInspect( - int numAccesses, int numBlocks, int numPages, Supplier truthSupplier) { - Stream accessSequence = randomAccessSequence(numAccesses, numBlocks, numPages); - Iterator it = accessSequence.iterator(); - while (it.hasNext()) { - Access access = it.next(); - switch (access.mType) { - case GET: - mEvictor.updateOnGet(access.mPage); - break; - case PUT: - mEvictor.updateOnPut(access.mPage); - break; - case DELETE: - mEvictor.updateOnDelete(access.mPage); - break; - default: - // no-op - } - assertEquals(truthSupplier.get(), mEvictor.evict()); - } - } - - private static Stream randomAccessSequence(int length, int numBlocks, int numPages) { - Random rng = new Random(); - return IntStream.range(0, length) - .mapToObj(i -> { - PageId page = new PageId(String.valueOf(rng.nextInt(numBlocks)), rng.nextInt(numPages)); - return new Access(AccessType.of(rng.nextInt(AccessType.values().length)), page); - }); - } - - private enum AccessType { - PUT, GET, DELETE; - - static AccessType of(int discriminant) { - switch (discriminant) { - case 0: - return PUT; - case 1: - return GET; - case 2: - return DELETE; - default: - throw new IllegalArgumentException("Unknown access type: " + discriminant); - } - } - } - - private static class Access { - AccessType mType; - PageId mPage; - - Access(AccessType type, PageId pageId) { - mType = type; - mPage = pageId; - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/page/BlockPageIdTest.java b/core/server/worker/src/test/java/alluxio/worker/page/BlockPageIdTest.java deleted file mode 100644 index 29b7c817ffa6..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/page/BlockPageIdTest.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertThrows; - -import alluxio.client.file.cache.PageId; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.curator.shaded.com.google.common.collect.Sets; -import org.junit.Test; - -import java.util.List; -import java.util.Objects; -import java.util.Set; - -public class BlockPageIdTest { - private static final long BLOCK_SIZE = 0; - - @Test - public void equality() { - BlockPageId id1 = new BlockPageId(1, 1, BLOCK_SIZE); - BlockPageId id2 = new BlockPageId("1", 1, BLOCK_SIZE); - BlockPageId id3 = new BlockPageId(1, 1, BLOCK_SIZE); - // reflexive - assertEquals(id1, id1); - assertEquals(id2, id2); - assertEquals(id3, id3); - // symmetric - assertEquals(id1, id2); - assertEquals(id2, id1); - // transitive - assertEquals(id1, id2); - assertEquals(id2, id3); - assertEquals(id3, id1); - } - - @Test - public void equalityWithParentClass() { - BlockPageId id1 = new BlockPageId("1", 1, BLOCK_SIZE); - PageId id2 = new PageId(BlockPageId.fileIdOf(1, BLOCK_SIZE), 1); - assertEquals(id1, id2); - assertEquals(id2, id1); - } - - @Test - public void usedAsKeyInCollections() { - Set pages = ImmutableSet.of( - new PageId(BlockPageId.fileIdOf(1, BLOCK_SIZE), 0), - new BlockPageId(1, 0, BLOCK_SIZE) - ); - assertEquals(1, pages.size()); - } - - @Test - public void inequalityWithOtherSubclass() { - PageId id = new BlockPageId("1", 1, BLOCK_SIZE); - PageId otherSubclassId = new MoreFieldsPageId("1", 1, BLOCK_SIZE); - assertNotEquals(id, otherSubclassId); - assertNotEquals(otherSubclassId, id); - } - - @Test - public void inequality() { - Set pageIds = ImmutableSet.of( - new BlockPageId(1, 1, BLOCK_SIZE), - new BlockPageId(2, 1, BLOCK_SIZE), - new BlockPageId(1, 2, BLOCK_SIZE), - new BlockPageId(2, 2, BLOCK_SIZE), - new Object()); - - for (Set set : Sets.combinations(pageIds, 2)) { - List pair = ImmutableList.copyOf(set); - assertNotEquals(pair.get(0), pair.get(1)); - assertNotEquals(pair.get(1), pair.get(0)); - } - } - - @Test - public void testHashCode() { - PageId id1 = new BlockPageId(1, 1, BLOCK_SIZE); - PageId id2 = new BlockPageId("1", 1, BLOCK_SIZE); - assertEquals(id1, id2); - assertEquals(id1.hashCode(), id2.hashCode()); - - PageId id3 = new PageId(BlockPageId.fileIdOf(1, BLOCK_SIZE), 1); - assertEquals(id1, id3); - assertEquals(id1.hashCode(), id3.hashCode()); - } - - @Test - public void getBlockId() { - long blockId = 2; - BlockPageId pageId = new BlockPageId(blockId, 0, BLOCK_SIZE); - assertEquals(blockId, pageId.getBlockId()); - } - - @Test - public void getBlockSize() { - long blockSize = 42; - BlockPageId pageId = new BlockPageId(1, 0, blockSize); - assertEquals(blockSize, pageId.getBlockSize()); - } - - @Test - public void downcastOk() { - PageId wellFormed = new PageId("paged_block_1234567890abcdef_size_0123cafebabedead", 0); - BlockPageId downcast = BlockPageId.downcast(wellFormed); - assertEquals(wellFormed, downcast); - assertEquals(0x1234_5678_90ab_cdefL, downcast.getBlockId()); - assertEquals(0x0123_cafe_babe_deadL, downcast.getBlockSize()); - - BlockPageId self = new BlockPageId(1234L, 0, BLOCK_SIZE); - assertEquals(self, BlockPageId.downcast(self)); - - PageId negativeBlockId = new PageId("paged_block_ffffffffffffffff_size_0000000000000001", 0); - downcast = BlockPageId.downcast(negativeBlockId); - assertEquals(negativeBlockId, downcast); - assertEquals(-1L, downcast.getBlockId()); - assertEquals(1, downcast.getBlockSize()); - } - - @Test - public void downcastWrong() { - PageId noPrefix = new PageId("_1234567890abcdef_size_0123cafebabedead", 0); - assertThrows(IllegalArgumentException.class, () -> BlockPageId.downcast(noPrefix)); - PageId notEnoughDigits = new PageId("paged_block_size_1234_cafe", 0); - assertThrows(IllegalArgumentException.class, () -> BlockPageId.downcast(notEnoughDigits)); - PageId empty = new PageId("", 0); - assertThrows(IllegalArgumentException.class, () -> BlockPageId.downcast(empty)); - PageId extraSuffix = - new PageId("paged_block_1234567890abcdef_size_0123cafebabedead.parquet", 0); - assertThrows(IllegalArgumentException.class, () -> BlockPageId.downcast(extraSuffix)); - PageId negativeBlockSize = new PageId("paged_block_1234567890abcdef_size_cafebabedead0123", 0); - assertThrows(IllegalArgumentException.class, () -> BlockPageId.downcast(negativeBlockSize)); - } - - private static class MoreFieldsPageId extends PageId { - private final long mSomeField; - - public MoreFieldsPageId(String fileId, long pageIndex, long value) { - super(fileId, pageIndex); - mSomeField = value; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), mSomeField); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - MoreFieldsPageId that = (MoreFieldsPageId) o; - return mSomeField == that.mSomeField; - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java b/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java deleted file mode 100644 index 2c21647ec766..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.CacheManager; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.store.PageReadTargetBuffer; - -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; - -/** - * Implementation of cache manager that stores cached data in byte arrays in memory. - */ -class ByteArrayCacheManager implements CacheManager { - private final Map mPages; - - /** - * Metrics for test validation. - */ - long mPagesServed = 0; - long mPagesCached = 0; - - ByteArrayCacheManager() { - mPages = new HashMap<>(); - } - - @Override - public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { - byte[] data = new byte[page.remaining()]; - page.get(data); - mPages.put(pageId, data); - mPagesCached++; - return true; - } - - @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, - CacheContext cacheContext) { - if (!mPages.containsKey(pageId)) { - return 0; - } - mPagesServed++; - target.writeBytes(mPages.get(pageId), pageOffset, bytesToRead); - return bytesToRead; - } - - @Override - public boolean delete(PageId pageId) { - return mPages.remove(pageId) != null; - } - - @Override - public State state() { - return State.READ_WRITE; - } - - @Override - public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cacheContext) { - return false; - } - - @Override - public void close() throws Exception { - // no-op - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockMetaStoreTest.java b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockMetaStoreTest.java deleted file mode 100644 index 2117db742647..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockMetaStoreTest.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import alluxio.ConfigurationRule; -import alluxio.Constants; -import alluxio.client.file.cache.CacheManagerOptions; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageInfo; -import alluxio.client.file.cache.allocator.Allocator; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.PageNotFoundException; -import alluxio.worker.block.AbstractBlockStoreEventListener; -import alluxio.worker.block.BlockStoreEventListener; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.util.List; -import java.util.Random; - -public class PagedBlockMetaStoreTest { - private static final long PAGE_SIZE = Constants.KB; - private static final long BLOCK_SIZE = Constants.MB; - private static final String PAGE_STORE_TYPE = "MEM"; - @Rule - public ConfigurationRule mConfRule = new ConfigurationRule( - ImmutableMap.of( - PropertyKey.WORKER_PAGE_STORE_SIZES, "1MB,1MB", - PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, String.valueOf(PAGE_SIZE), - PropertyKey.WORKER_PAGE_STORE_DIRS, "/tmp/1,/tmp/2", - PropertyKey.WORKER_PAGE_STORE_TYPE, PAGE_STORE_TYPE - ), - Configuration.modifiableGlobal()); - - private List mDirs; - private PagedBlockMetaStore mMetastore; - - @Before - public void setup() throws Exception { - CacheManagerOptions cachemanagerOptions = - CacheManagerOptions.createForWorker(Configuration.global()); - mDirs = PagedBlockStoreDir.fromPageStoreDirs( - PageStoreDir.createPageStoreDirs(cachemanagerOptions)); - mMetastore = new PagedBlockMetaStore(mDirs); - } - - private static BlockPageId blockPageId(String blockId, long pageIndex) { - return new BlockPageId(blockId, pageIndex, BLOCK_SIZE); - } - - @Test - public void addPage() { - mMetastore.addBlock(new PagedBlockMeta(1, BLOCK_SIZE, mDirs.get(0))); - mMetastore.addBlock(new PagedBlockMeta(2, BLOCK_SIZE, mDirs.get(1))); - addPagesOnDir1(blockPageId("1", 0), blockPageId("1", 1)); - addPagesOnDir2(blockPageId("2", 0)); - - assertTrue(mMetastore.hasBlock(1)); - assertTrue(mMetastore.hasBlock(2)); - assertEquals(mDirs.get(0), mMetastore.getBlock(1).get().getDir()); - assertEquals(mDirs.get(1), mMetastore.getBlock(2).get().getDir()); - assertEquals(2, mDirs.get(0).getBlockCachedPages(1)); - assertEquals(1, mDirs.get(1).getBlockCachedPages(2)); - - PagedBlockStoreMeta storeMeta = mMetastore.getStoreMetaFull(); - assertEquals( - ImmutableMap.of( - mDirs.get(0).getLocation(), - ImmutableList.of(1L), - mDirs.get(1).getLocation(), - ImmutableList.of(2L) - ), - storeMeta.getBlockListByStorageLocation()); - } - - @Test - public void removePage() throws Exception { - mMetastore.addBlock(new PagedBlockMeta(1, BLOCK_SIZE, mDirs.get(0))); - mMetastore.addBlock(new PagedBlockMeta(2, BLOCK_SIZE, mDirs.get(1))); - addPagesOnDir1(blockPageId("1", 0), blockPageId("1", 1)); - addPagesOnDir2(blockPageId("2", 0)); - - mMetastore.removePage(blockPageId("1", 0)); - mMetastore.removePage(blockPageId("1", 1)); - mMetastore.removePage(blockPageId("2", 0)); - - assertFalse(mMetastore.hasBlock(1)); - assertFalse(mMetastore.hasBlock(2)); - assertEquals(0, mDirs.get(0).getBlockCachedPages(1)); - assertEquals(0, mDirs.get(1).getBlockCachedPages(2)); - - PagedBlockStoreMeta storeMeta = mMetastore.getStoreMetaFull(); - assertEquals( - ImmutableMap.of( - mDirs.get(0).getLocation(), - ImmutableList.of(), - mDirs.get(1).getLocation(), - ImmutableList.of() - ), - storeMeta.getBlockListByStorageLocation()); - } - - @Test - public void reset() throws Exception { - mMetastore.addBlock(new PagedBlockMeta(1, BLOCK_SIZE, mDirs.get(0))); - mMetastore.addBlock(new PagedBlockMeta(2, BLOCK_SIZE, mDirs.get(1))); - addPagesOnDir1(blockPageId("1", 0), blockPageId("1", 1)); - addPagesOnDir2(blockPageId("2", 0)); - - mMetastore.reset(); - PagedBlockStoreMeta storeMeta = mMetastore.getStoreMetaFull(); - assertEquals( - ImmutableMap.of( - mDirs.get(0).getLocation(), - ImmutableList.of(), - mDirs.get(1).getLocation(), - ImmutableList.of() - ), - storeMeta.getBlockListByStorageLocation()); - } - - @Test - public void listenerOnRemove() throws PageNotFoundException { - BlockStoreEventListener listener = new AbstractBlockStoreEventListener() { - @Override - public void onRemoveBlock(long blockId, BlockStoreLocation location) { - assertEquals(1, blockId); - assertEquals(mDirs.get(0).getLocation(), location); - } - }; - - mMetastore.registerBlockStoreEventListener(listener); - PageId pageId = blockPageId("1", 0); - mMetastore.addBlock(new PagedBlockMeta(1, BLOCK_SIZE, mDirs.get(0))); - addPagesOnDir1(pageId); - mMetastore.removePage(pageId); - } - - @Test - public void stickyAllocate() { - String blockId = "1"; - long pageSize = 1; - BlockPageId page = blockPageId(blockId, 0); - String fileId = page.getFileId(); - mMetastore = new PagedBlockMetaStore(mDirs, new RandomAllocator(ImmutableList.copyOf(mDirs))); - PageStoreDir dir = mMetastore.allocate(fileId, pageSize); - PageInfo pageInfo = new PageInfo(page, pageSize, dir); - mMetastore.addBlock(new PagedBlockMeta(1, BLOCK_SIZE, (PagedBlockStoreDir) dir)); - mMetastore.addPage(page, pageInfo); - for (int i = 0; i < 100; i++) { - assertEquals(dir, mMetastore.allocate(fileId, pageSize)); - } - } - - @Test - public void allocatePageIsNotAddPage() { - String fileId = BlockPageId.fileIdOf(1, BLOCK_SIZE); - PageStoreDir dir = mMetastore.allocate(fileId, 1); - assertFalse(dir.hasFile(fileId)); - PagedBlockStoreMeta storeMeta = mMetastore.getStoreMeta(); - assertEquals(0, storeMeta.getNumberOfBlocks()); - } - - private void addPagesOnDir1(PageId... pages) { - addPages(0, pages); - } - - private void addPagesOnDir2(PageId... pages) { - addPages(1, pages); - } - - private void addPages(int dirIndex, PageId... pages) { - for (PageId pageId : pages) { - mMetastore.addPage(pageId, new PageInfo(pageId, 0, mDirs.get(dirIndex))); - } - } - - private static class RandomAllocator implements Allocator { - private final List mDirs; - - public RandomAllocator(List dirs) { - mDirs = dirs; - } - - @Override - public PageStoreDir allocate(String fileId, long fileLength) { - Random rng = new Random(); - return mDirs.get(rng.nextInt(mDirs.size())); - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockReaderTest.java b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockReaderTest.java deleted file mode 100644 index 19d1fd222652..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockReaderTest.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import alluxio.AlluxioURI; -import alluxio.ConfigurationRule; -import alluxio.Constants; -import alluxio.client.file.cache.CacheManagerOptions; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.conf.AlluxioProperties; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.master.NoopUfsManager; -import alluxio.underfs.UnderFileSystemConfiguration; -import alluxio.util.io.BufferUtils; -import alluxio.worker.block.UfsInputStreamCache; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.Random; - -@RunWith(Parameterized.class) -public class PagedBlockReaderTest { - private static final long CARDINALITY_BYTE = 256; - // this test uses local file system to create test blocks, so one block is one file - // therefore offset in file is always 0 - private static final long OFFSET_IN_FILE = 0; - private static final long MOUNT_ID = 100; - // use fixed block size and varying page and buffer size - private static final long BLOCK_SIZE = Constants.MB; - private static final int MAX_UFS_READ_CONCURRENCY = 1; - private static final long BLOCK_ID = 0L; - @Rule - public TemporaryFolder mTempFolderRule = new TemporaryFolder(); - @Rule - public ConfigurationRule mConfRule = - new ConfigurationRule(ImmutableMap.of(), Configuration.modifiableGlobal()); - - @Parameterized.Parameters(name = "PageSize: {0}, BufferSize: {1}, Offset: {2}") - public static Collection data() { - Integer[][] params = new Integer[][]{ - /* page size, buffer size */ - {Constants.KB, Constants.KB}, - {Constants.KB, 200}, - {Constants.KB, 201}, - {Constants.MB, Constants.KB}, - {Constants.MB, 200}, - {Constants.MB - 1, Constants.KB}, - {2 * Constants.MB, Constants.KB}, - }; - List offsets = ImmutableList.of(0, 1, (int) (BLOCK_SIZE - 1), (int) BLOCK_SIZE); - - ImmutableList.Builder listBuilder = ImmutableList.builder(); - // cartesian product - for (Integer[] paramPair : params) { - for (int offset : offsets) { - listBuilder.add(new Integer[] {paramPair[0], paramPair[1], offset}); - } - } - return listBuilder.build(); - } - - @Parameterized.Parameter - public long mPageSize; - - @Parameterized.Parameter(1) - public int mBufferSize; - - @Parameterized.Parameter(2) - public int mOffset; - - private PagedBlockReader mReader; - - @Before - public void setup() throws Exception { - mConfRule.set(PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, String.valueOf(mPageSize)); - File tempFolder = mTempFolderRule.newFolder(); - Path blockFilePath = tempFolder.toPath().resolve("block"); - createTempUfsBlock(blockFilePath, BLOCK_SIZE); - - NoopUfsManager ufsManager = new NoopUfsManager(); - ufsManager.addMount(MOUNT_ID, new AlluxioURI(tempFolder.getAbsolutePath()), - new UnderFileSystemConfiguration(new InstancedConfiguration(new AlluxioProperties()), - true)); - - CacheManagerOptions cachemanagerOptions = - CacheManagerOptions.createForWorker(Configuration.global()); - List pagedBlockStoreDirs = PagedBlockStoreDir.fromPageStoreDirs( - PageStoreDir.createPageStoreDirs(cachemanagerOptions)); - final PagedBlockMeta blockMeta = - new PagedBlockMeta(BLOCK_ID, BLOCK_SIZE, pagedBlockStoreDirs.get(0)); - PagedUfsBlockReader ufsBlockReader = new PagedUfsBlockReader( - ufsManager, - new UfsInputStreamCache(), - blockMeta, - mOffset, - createUfsBlockOptions(blockFilePath.toAbsolutePath().toString()), - mPageSize - ); - mReader = new PagedBlockReader( - new ByteArrayCacheManager(), - blockMeta, - mOffset, - Optional.of(ufsBlockReader), - mPageSize - ); - } - - @Test - public void sequentialReadAtOnce() throws Exception { - try (PagedBlockReader reader = mReader) { - ByteBuffer buffer = reader.read(0, BLOCK_SIZE); - assertTrue(BufferUtils.equalIncreasingByteBuffer(0, buffer.remaining(), buffer)); - } - } - - @Test - public void sequentialReadMultipleRounds() throws Exception { - final int bytesToReadPerIter = mBufferSize; - long pos = 0; - try (PagedBlockReader reader = mReader; - ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - while (pos < BLOCK_SIZE) { - int length = (int) Math.min(bytesToReadPerIter, BLOCK_SIZE - pos); - ByteBuffer buffer = reader.read(pos, length); - pos += buffer.remaining(); - byte[] data = new byte[buffer.remaining()]; - buffer.get(data); - baos.write(data); - } - assertTrue(BufferUtils.equalIncreasingByteArray((byte) 0, baos.size(), baos.toByteArray())); - } - } - - @Test - public void randomReadToEnd() throws Exception { - final int bytesToReadPerIter = mBufferSize; - final long initialPos = new Random().nextInt((int) BLOCK_SIZE); - long pos = initialPos; - try (PagedBlockReader reader = mReader; - ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - while (pos < BLOCK_SIZE) { - int length = (int) Math.min(bytesToReadPerIter, BLOCK_SIZE - pos); - ByteBuffer buffer = reader.read(pos, length); - pos += buffer.remaining(); - byte[] data = new byte[buffer.remaining()]; - buffer.get(data); - baos.write(data); - } - assertTrue(BufferUtils.equalIncreasingByteArray( - (byte) (initialPos % CARDINALITY_BYTE), baos.size(), baos.toByteArray())); - } - } - - @Test - public void randomReadJumpAround() throws Exception { - final int bytesToReadPerIter = mBufferSize; - final Random rng = new Random(); - try (PagedBlockReader reader = mReader) { - for (int i = 0; i < 100; i++) { // try 100 random initial positions - final long pos = rng.nextInt((int) BLOCK_SIZE); - final int length = (int) Math.min(bytesToReadPerIter, BLOCK_SIZE - pos); - ByteBuffer buffer = reader.read(pos, length); - - assertTrue(BufferUtils.equalIncreasingByteBuffer( - (byte) (pos % CARDINALITY_BYTE), buffer.remaining(), buffer)); - } - } - } - - @Test - public void sequentialTransferAllAtOnce() throws Exception { - ByteBuffer buffer = ByteBuffer.allocate((int) (BLOCK_SIZE - mOffset)); - buffer.mark(); - ByteBuf wrapped = Unpooled.wrappedBuffer(buffer); - wrapped.clear(); - if (BLOCK_SIZE == mOffset) { - assertEquals(-1, mReader.transferTo(wrapped)); - } else { - assertEquals(BLOCK_SIZE - mOffset, mReader.transferTo(wrapped)); - } - assertTrue(BufferUtils.equalIncreasingByteBuffer( - (byte) (mOffset % CARDINALITY_BYTE), buffer.remaining(), buffer)); - } - - @Test - public void sequentialTransferMultipleTimes() throws Exception { - final int bytesToReadPerIter = mBufferSize; - final int totalReadable = (int) (BLOCK_SIZE - mOffset); - ByteBuffer buffer = ByteBuffer.allocate(totalReadable); - - int bytesRead = 0; - while (bytesRead < totalReadable) { - int bytesToRead = Math.min(bytesToReadPerIter, totalReadable - bytesRead); - ByteBuf buf = Unpooled.buffer(bytesToRead, bytesToRead); - final int actualRead = mReader.transferTo(buf); - if (actualRead == -1) { - break; - } - assertEquals(bytesToRead, actualRead); - bytesRead += actualRead; - buffer.put(buf.nioBuffer()); - } - buffer.flip(); - assertTrue(BufferUtils.equalIncreasingByteBuffer( - (byte) (mOffset % CARDINALITY_BYTE), buffer.remaining(), buffer)); - } - - private static UfsBlockReadOptions createUfsBlockOptions(String ufsPath) { - return new UfsBlockReadOptions(MOUNT_ID, OFFSET_IN_FILE, ufsPath, true); - } - - private static void createTempUfsBlock(Path destPath, long blockSize) throws Exception { - try (OutputStream os = - Files.newOutputStream(destPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { - long bytesWritten = 0; - final int bufSize = Constants.KB; // must be a multiple of cardinality(byte) - byte[] buf = BufferUtils.getIncreasingByteArray(0, bufSize); - while (bytesWritten < blockSize) { - if (bytesWritten + bufSize > blockSize) { - byte[] bytesLeft = new byte[(int) (blockSize - bytesWritten)]; - System.arraycopy(buf, 0, bytesLeft, 0, bytesLeft.length); - os.write(bytesLeft); - break; - } - os.write(buf); - bytesWritten += bufSize; - } - } - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockStoreDirTest.java b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockStoreDirTest.java deleted file mode 100644 index fa8b170f3823..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockStoreDirTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import alluxio.Constants; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageInfo; -import alluxio.client.file.cache.evictor.CacheEvictorOptions; -import alluxio.client.file.cache.evictor.FIFOCacheEvictor; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.client.file.cache.store.PageStoreOptions; -import alluxio.client.file.cache.store.PageStoreType; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.collect.ImmutableList; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.nio.file.Path; - -public class PagedBlockStoreDirTest { - private static final int DIR_INDEX = 0; - private static final long BLOCK_SIZE = Constants.MB; - private PagedBlockStoreDir mDir; - private Path mDirPath; - - @Rule - public TemporaryFolder mTempFolder = new TemporaryFolder(); - - @Before - public void setup() throws Exception { - mDirPath = mTempFolder.newFolder().toPath(); - InstancedConfiguration conf = Configuration.modifiableGlobal(); - conf.set(PropertyKey.WORKER_PAGE_STORE_DIRS, ImmutableList.of(mDirPath)); - conf.set(PropertyKey.WORKER_PAGE_STORE_SIZES, ImmutableList.of(Constants.MB)); - conf.set(PropertyKey.WORKER_PAGE_STORE_TYPE, PageStoreType.LOCAL); - PageStoreDir pageStoreDir = - PageStoreDir.createPageStoreDir( - new CacheEvictorOptions().setEvictorClass(FIFOCacheEvictor.class), - PageStoreOptions.createForWorkerPageStore(conf).get(DIR_INDEX)); - mDir = new PagedBlockStoreDir(pageStoreDir, DIR_INDEX); - } - - private static BlockPageId blockPageId(String blockId, long pageIndex) { - return new BlockPageId(blockId, pageIndex, BLOCK_SIZE); - } - - @Test - public void rootPath() { - assertEquals(mDirPath.resolve(PageStoreType.LOCAL.name()), mDir.getRootPath()); - } - - @Test - public void dirIndex() { - assertEquals(DIR_INDEX, mDir.getDirIndex()); - } - - @Test - public void blockLocation() { - BlockStoreLocation location = mDir.getLocation(); - assertEquals(PagedBlockStoreMeta.DEFAULT_TIER, location.tierAlias()); - assertEquals(PagedBlockStoreMeta.DEFAULT_MEDIUM, location.mediumType()); - assertEquals(DIR_INDEX, location.dir()); - } - - @Test - public void numBlocks() { - assertEquals(0, mDir.getNumBlocks()); - mDir.putPage(new PageInfo(blockPageId("0", 0), 0, mDir)); - assertEquals(1, mDir.getNumBlocks()); - mDir.putPage(new PageInfo(blockPageId("0", 1), 0, mDir)); - assertEquals(1, mDir.getNumBlocks()); - mDir.putPage(new PageInfo(blockPageId("1", 0), 0, mDir)); - assertEquals(2, mDir.getNumBlocks()); - } - - @Test - public void cachedBytes() { - assertEquals(0, mDir.getCachedBytes()); - mDir.putPage(new PageInfo(blockPageId("0", 0), Constants.KB, mDir)); - assertEquals(Constants.KB, mDir.getCachedBytes()); - assertEquals(Constants.KB, mDir.getBlockCachedBytes(0)); - mDir.putPage(new PageInfo(blockPageId("0", 1), Constants.KB, mDir)); - assertEquals(2 * Constants.KB, mDir.getCachedBytes()); - assertEquals(2 * Constants.KB, mDir.getBlockCachedBytes(0)); - mDir.putPage(new PageInfo(blockPageId("1", 0), Constants.KB, mDir)); - assertEquals(3 * Constants.KB, mDir.getCachedBytes()); - assertEquals(Constants.KB, mDir.getBlockCachedBytes(1)); - } - - @Test - public void putPage() throws Exception { - long blockId = 0; - PageId pageId = blockPageId(String.valueOf(blockId), 0); - PageInfo pageInfo = new PageInfo(pageId, Constants.KB, mDir); - mDir.putPage(pageInfo); - assertEquals(1, mDir.getBlockCachedPages(blockId)); - assertTrue(mDir.hasFile(pageId.getFileId())); - - // duplicate adds are ignored - mDir.putPage(pageInfo); - assertEquals(1, mDir.getBlockCachedPages(blockId)); - } - - @Test - public void deletePage() throws Exception { - long blockId = 0; - PageId pageId = blockPageId(String.valueOf(blockId), 0); - PageInfo pageInfo = new PageInfo(pageId, Constants.KB, mDir); - mDir.putPage(pageInfo); - assertEquals(1, mDir.getBlockCachedPages(blockId)); - assertEquals(Constants.KB, mDir.getBlockCachedBytes(blockId)); - - long cachedBytes = mDir.deletePage(pageInfo); - assertEquals(0, mDir.getBlockCachedPages(blockId)); - assertEquals(0, mDir.getBlockCachedBytes(blockId)); - assertEquals(mDir.getCachedBytes(), cachedBytes); - - // deleting a non-existing page is no-op - long cachedBytes2 = mDir.deletePage(pageInfo); - assertEquals(0, mDir.getBlockCachedPages(blockId)); - assertEquals(0, mDir.getBlockCachedBytes(blockId)); - assertEquals(cachedBytes, cachedBytes2); - } - - @Test - public void evictor() throws Exception { - BlockPageEvictor evictor = mDir.getEvictor(); - - long blockId = 0; - PageId pageId0 = blockPageId(String.valueOf(blockId), 0); - PageInfo pageInfo0 = new PageInfo(pageId0, Constants.KB, mDir); - mDir.putPage(pageInfo0); - assertEquals(pageId0, evictor.evict()); - - PageId pageId1 = blockPageId(String.valueOf(blockId), 1); - PageInfo pageInfo1 = new PageInfo(pageId1, Constants.KB, mDir); - mDir.putPage(pageInfo1); - mDir.deletePage(pageInfo0); - assertEquals(pageId1, evictor.evict()); - } - - @Test - public void addTempPage() throws Exception { - long blockId = 0; - PageId pageId0 = blockPageId(String.valueOf(blockId), 0); - final String fileId = pageId0.getFileId(); - PageInfo pageInfo0 = new PageInfo(pageId0, Constants.KB, mDir); - mDir.putTempPage(pageInfo0); - mDir.getPageStore().putTemporary(pageId0, new byte[Constants.KB]); - assertEquals(0, mDir.getBlockCachedPages(blockId)); - assertFalse(mDir.hasFile(fileId)); - assertTrue(mDir.hasTempFile(fileId)); - - mDir.commit(fileId); - assertEquals(1, mDir.getBlockCachedPages(blockId)); - assertTrue(mDir.hasFile(fileId)); - assertFalse(mDir.hasTempFile(fileId)); - } - - @Test - public void abortTempPage() throws Exception { - long blockId = 0; - PageId pageId0 = blockPageId(String.valueOf(blockId), 0); - final String fileId = pageId0.getFileId(); - PageInfo pageInfo0 = new PageInfo(pageId0, Constants.KB, mDir); - mDir.putTempPage(pageInfo0); - mDir.getPageStore().putTemporary(pageId0, new byte[Constants.KB]); - assertEquals(0, mDir.getBlockCachedPages(blockId)); - assertTrue(mDir.hasTempFile(fileId)); - assertFalse(mDir.hasFile(fileId)); - - mDir.abort(fileId); - assertEquals(0, mDir.getBlockCachedPages(blockId)); - assertFalse(mDir.hasTempFile(fileId)); - assertFalse(mDir.hasFile(fileId)); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockStoreMetaTest.java b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockStoreMetaTest.java deleted file mode 100644 index 4edad792be3e..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockStoreMetaTest.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_MEDIUM; -import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_TIER; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import alluxio.ConfigurationRule; -import alluxio.Constants; -import alluxio.client.file.cache.CacheManagerOptions; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageInfo; -import alluxio.client.file.cache.store.PageStoreDir; -import alluxio.collections.Pair; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.util.io.PathUtils; -import alluxio.worker.block.BlockStoreLocation; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.util.List; -import java.util.Map; - -public class PagedBlockStoreMetaTest { - private static final long PAGE_SIZE = Constants.KB; - private static final long BLOCK_SIZE = Constants.MB; - private static final String PAGE_STORE_TYPE = "MEM"; - private static final long PAGE_DIR_CAPACITY_0 = 4 * Constants.KB; - private static final long PAGE_DIR_CAPACITY_1 = 8 * Constants.KB; - private static final String PAGE_DIR_PATH_0 = "/test/page/store/0"; - private static final String PAGE_DIR_PATH_0_FULL = - PathUtils.concatPath(PAGE_DIR_PATH_0, PAGE_STORE_TYPE); - private static final String PAGE_DIR_PATH_1 = "/test/page/store/1"; - private static final String PAGE_DIR_PATH_1_FULL = - PathUtils.concatPath(PAGE_DIR_PATH_1, PAGE_STORE_TYPE); - - private PagedBlockMetaStore mPageMetaStore; - private List mDirs; - - @Rule - public final ConfigurationRule mConfigRule = new ConfigurationRule( - ImmutableMap.of( - PropertyKey.WORKER_PAGE_STORE_SIZES, - String.format("%d,%d", PAGE_DIR_CAPACITY_0, PAGE_DIR_CAPACITY_1), - PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, String.valueOf(PAGE_SIZE), - PropertyKey.WORKER_PAGE_STORE_DIRS, PAGE_DIR_PATH_0 + "," + PAGE_DIR_PATH_1, - PropertyKey.WORKER_PAGE_STORE_TYPE, PAGE_STORE_TYPE - ), - Configuration.modifiableGlobal() - ); - - @Before - public void setup() throws Exception { - CacheManagerOptions cachemanagerOptions = - CacheManagerOptions.createForWorker(Configuration.global()); - mDirs = PagedBlockStoreDir.fromPageStoreDirs( - PageStoreDir.createPageStoreDirs(cachemanagerOptions)); - mPageMetaStore = new PagedBlockMetaStore(mDirs); - } - - private void generatePages(int numPages, long parentBlockId, int dirIndex, long pageSize) { - PagedBlockMeta blockMeta = - new PagedBlockMeta(parentBlockId, numPages * pageSize, mDirs.get(dirIndex)); - mPageMetaStore.addBlock(blockMeta); - for (int i = 0; i < numPages; i++) { - PageId pageId = new BlockPageId(parentBlockId, i, BLOCK_SIZE); - PageInfo pageInfo = new PageInfo(pageId, pageSize, mDirs.get(dirIndex)); - mPageMetaStore.addPage(pageId, pageInfo); - } - } - - @Test - public void blocks() throws Exception { - int numPages = 4; - long pageSize = Constants.KB; - List blockIdsOnDir0 = ImmutableList.of(1L, 2L); - List blockIdsOnDir1 = ImmutableList.of(3L); - for (long blockId : blockIdsOnDir0) { - generatePages(numPages, blockId, 0, pageSize); - } - for (long blockId : blockIdsOnDir1) { - generatePages(numPages, blockId, 1, pageSize); - } - PagedBlockStoreMeta storeMeta = mPageMetaStore.getStoreMetaFull(); - - int numBlocks = blockIdsOnDir0.size() + blockIdsOnDir1.size(); - assertEquals(numBlocks, storeMeta.getNumberOfBlocks()); - - Map> blockList = storeMeta.getBlockList(); - assertNotNull(blockList); - assertEquals(1, blockList.size()); - assertTrue(blockList.containsKey(DEFAULT_TIER)); - assertEquals(numBlocks, blockList.get(DEFAULT_TIER).size()); - for (long blockId : Iterables.concat(blockIdsOnDir0, blockIdsOnDir1)) { - assertTrue(blockList.get(DEFAULT_TIER).contains(blockId)); - } - - Map> blockLocations = storeMeta.getBlockListByStorageLocation(); - assertNotNull(blockLocations); - assertEquals(2, blockLocations.size()); - final BlockStoreLocation blockLocDir0 = new BlockStoreLocation(DEFAULT_TIER, 0, DEFAULT_MEDIUM); - final BlockStoreLocation blockLocDir1 = new BlockStoreLocation(DEFAULT_TIER, 1, DEFAULT_MEDIUM); - assertEquals(blockIdsOnDir0.size(), blockLocations.get(blockLocDir0).size()); - assertEquals(blockIdsOnDir1.size(), blockLocations.get(blockLocDir1).size()); - for (long blockId : blockIdsOnDir0) { - assertTrue(blockLocations.get(blockLocDir0).contains(blockId)); - } - for (long blockId : blockIdsOnDir1) { - assertTrue(blockLocations.get(blockLocDir1).contains(blockId)); - } - } - - @Test - public void noDetailedBlockList() { - int numPages = 4; - long pageSize = Constants.KB; - List blockIds = ImmutableList.of(1L, 2L); - for (long blockId : blockIds) { - generatePages(numPages, blockId, 0, pageSize); - } - PagedBlockStoreMeta storeMeta = mPageMetaStore.getStoreMeta(); - assertEquals(2, storeMeta.getNumberOfBlocks()); - assertNull(storeMeta.getBlockList()); - assertNull(storeMeta.getBlockListByStorageLocation()); - } - - @Test - public void capacity() { - PagedBlockStoreMeta storeMeta = mPageMetaStore.getStoreMetaFull(); - - long totalCapacity = mDirs.stream().map(PageStoreDir::getCapacityBytes).reduce(0L, Long::sum); - assertEquals(totalCapacity, storeMeta.getCapacityBytes()); - assertEquals(totalCapacity, - (long) storeMeta.getCapacityBytesOnTiers().get(DEFAULT_TIER)); - Map, Long> capacityByDirs = storeMeta.getCapacityBytesOnDirs(); - assertEquals(2, capacityByDirs.size()); - assertEquals(mDirs.get(0).getCapacityBytes(), - (long) capacityByDirs.get(new Pair<>(DEFAULT_TIER, PAGE_DIR_PATH_0_FULL))); - assertEquals(mDirs.get(1).getCapacityBytes(), - (long) capacityByDirs.get(new Pair<>(DEFAULT_TIER, PAGE_DIR_PATH_1_FULL))); - } - - @Test - public void directoryPaths() { - PagedBlockStoreMeta storeMeta = mPageMetaStore.getStoreMetaFull(); - Map> dirByTiers = storeMeta.getDirectoryPathsOnTiers(); - assertEquals(1, dirByTiers.size()); - assertEquals( - ImmutableList.of(PAGE_DIR_PATH_0_FULL, PAGE_DIR_PATH_1_FULL), - dirByTiers.get(DEFAULT_TIER)); - } - - @Test - public void lostStorage() { - PagedBlockStoreMeta storeMeta = mPageMetaStore.getStoreMetaFull(); - assertEquals(ImmutableMap.of(), storeMeta.getLostStorage()); - } - - @Test - public void usage() { - long blockId = 0; - int numPages = 4; - long pageSize = Constants.KB; - generatePages(numPages, blockId, 0, pageSize); - PagedBlockStoreMeta storeMeta = mPageMetaStore.getStoreMetaFull(); - - assertEquals(numPages * pageSize, storeMeta.getUsedBytes()); - assertEquals(numPages * pageSize, (long) storeMeta.getUsedBytesOnTiers().get(DEFAULT_TIER)); - Map, Long> usedByDirs = storeMeta.getUsedBytesOnDirs(); - assertEquals(2, usedByDirs.size()); - assertEquals(numPages * pageSize, - (long) usedByDirs.get(new Pair<>(DEFAULT_TIER, PAGE_DIR_PATH_0_FULL))); - assertEquals(0, - (long) usedByDirs.get(new Pair<>(DEFAULT_TIER, PAGE_DIR_PATH_1_FULL))); - } - - @Test - public void storageTierAssoc() { - PagedBlockStoreMeta storeMeta = mPageMetaStore.getStoreMetaFull(); - assertEquals(PagedBlockStoreMeta.DEFAULT_STORAGE_TIER_ASSOC, storeMeta.getStorageTierAssoc()); - } -} diff --git a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java deleted file mode 100644 index fbbc687a5105..000000000000 --- a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.worker.page; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import alluxio.AlluxioTestDirectory; -import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.CacheManager; -import alluxio.client.file.cache.CacheManagerOptions; -import alluxio.client.file.cache.DefaultPageMetaStore; -import alluxio.client.file.cache.LocalCacheManager; -import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.PageMetaStore; -import alluxio.client.file.cache.PageStore; -import alluxio.client.file.cache.evictor.CacheEvictor; -import alluxio.client.file.cache.evictor.FIFOCacheEvictor; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.LocalPageStoreDir; -import alluxio.client.file.cache.store.PageStoreOptions; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.util.CommonUtils; -import alluxio.util.WaitForOptions; -import alluxio.util.io.BufferUtils; - -import com.google.common.collect.ImmutableList; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -@RunWith(Parameterized.class) -public class PagedBlockWriterTest { - private static final long BLOCK_ID = 1L; - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList(new Object[][] { - /*file_length, chuck_size, page_size*/ - {2048, 1024, 128}, - {2049, 1024, 128}, - {2048, 1023, 128}, - {2048, 1024, 129}, - }); - } - - @Parameterized.Parameter - public int mFileLength; - - @Parameterized.Parameter(1) - public int mChunkSize; - - @Parameterized.Parameter(2) - public int mPageSize; - - private LocalCacheManager mCacheManager; - private InstancedConfiguration mConf = Configuration.copyGlobal(); - private PageMetaStore mPageMetaStore; - private CacheEvictor mEvictor; - private CacheManagerOptions mCachemanagerOptions; - private PageStoreOptions mPageStoreOptions; - private PageStore mPageStore; - private LocalPageStoreDir mPageStoreDir; - private PagedBlockWriter mWriter; - - @Rule - public ExpectedException mThrown = ExpectedException.none(); - - @Before - public void before() throws Exception { - mConf.set(PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, mPageSize); - mConf.set(PropertyKey.WORKER_PAGE_STORE_DIRS, ImmutableList.of( - AlluxioTestDirectory.createTemporaryDirectory("page_store").getAbsolutePath())); - mCachemanagerOptions = CacheManagerOptions.createForWorker(mConf); - mPageStoreOptions = mCachemanagerOptions.getPageStoreOptions().get(0); - mPageStore = PageStore.create(mPageStoreOptions); - mEvictor = new FIFOCacheEvictor(mCachemanagerOptions.getCacheEvictorOptions()); - mPageStoreDir = new LocalPageStoreDir(mPageStoreOptions, mPageStore, mEvictor); - mPageStoreDir.reset(); - mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(mPageStoreDir)); - mCacheManager = - LocalCacheManager.create(mCachemanagerOptions, mPageMetaStore); - CommonUtils.waitFor("restore completed", - () -> mCacheManager.state() == CacheManager.State.READ_WRITE, - WaitForOptions.defaults().setTimeoutMs(10000)); - mWriter = new PagedBlockWriter(mCacheManager, BLOCK_ID, mPageSize); - } - - @After - public void after() throws Exception { - mWriter.close(); - } - - @Test - public void appendByteBuf() throws Exception { - for (int offset = 0; offset < mFileLength; offset += mChunkSize) { - int bytesToWrite = Math.min(mChunkSize, mFileLength - offset); - ByteBuf buffer = Unpooled.wrappedBuffer( - BufferUtils.getIncreasingByteBuffer(bytesToWrite)); - assertEquals(bytesToWrite, mWriter.append(buffer)); - } - mWriter.close(); - mPageMetaStore.commitFile(BlockPageId.tempFileIdOf(BLOCK_ID), - BlockPageId.fileIdOf(BLOCK_ID, mFileLength)); - mPageStoreDir.commit(BlockPageId.tempFileIdOf(BLOCK_ID), - BlockPageId.fileIdOf(BLOCK_ID, mFileLength)); - verifyDataInCache(); - } - - @Test - public void append() throws Exception { - for (int offset = 0; offset < mFileLength; offset += mChunkSize) { - int bytesToWrite = Math.min(mChunkSize, mFileLength - offset); - ByteBuffer buffer = - BufferUtils.getIncreasingByteBuffer(bytesToWrite); - assertEquals(bytesToWrite, mWriter.append(buffer)); - } - mWriter.close(); - mPageMetaStore.commitFile(BlockPageId.tempFileIdOf(BLOCK_ID), - BlockPageId.fileIdOf(BLOCK_ID, mFileLength)); - mPageStoreDir.commit(BlockPageId.tempFileIdOf(BLOCK_ID), - BlockPageId.fileIdOf(BLOCK_ID, mFileLength)); - verifyDataInCache(); - } - - private void verifyDataInCache() { - List pageIds = - mCacheManager.getCachedPageIdsByFileId( - BlockPageId.fileIdOf(BLOCK_ID, mFileLength), mFileLength); - assertEquals((int) Math.ceil((double) mFileLength / mPageSize), pageIds.size()); - byte[] dataInCache = new byte[mFileLength]; - for (int i = 0; i < pageIds.size(); i++) { - PageId pageId = pageIds.get(i); - mCacheManager.get(pageId, 0, Math.min(mPageSize, mFileLength - i * mPageSize), - new ByteArrayTargetBuffer(dataInCache, i * mPageSize), - CacheContext.defaults().setTemporary(false)); - } - for (int offset = 0; offset < mFileLength; offset += mChunkSize) { - int chunkLength = Math.min(mChunkSize, mFileLength - offset); - byte[] chunk = new byte[chunkLength]; - System.arraycopy(dataInCache, offset, chunk, 0, chunkLength); - assertTrue( - BufferUtils.equalIncreasingByteArray(chunkLength, chunk)); - } - } -} diff --git a/core/transport/pom.xml b/core/transport/pom.xml deleted file mode 100644 index 0e40c10a1c04..000000000000 --- a/core/transport/pom.xml +++ /dev/null @@ -1,178 +0,0 @@ - - - 4.0.0 - - org.alluxio - alluxio-core - 2.10.0-SNAPSHOT - - alluxio-core-transport - jar - Alluxio Core - Transport - Protobuf shared in Alluxio core modules - - - - - ${project.parent.parent.basedir}/build - false - - alluxio.core.transport - false - - - - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - io.grpc - grpc-core - - - io.grpc - grpc-protobuf - - - io.grpc - grpc-stub - - - - - - generate - - false - - - - - - org.apache.maven.plugins - maven-clean-plugin - - - - src/main/java/alluxio/proto - false - - **/*.java - - - - src/main/java/alluxio/grpc - false - - **/*.java - - - - - - - - - - java11 - - 11 - - - - javax.annotation - javax.annotation-api - - - - - - - - - kr.motd.maven - os-maven-plugin - 1.6.2 - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.6.1 - - com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} - ${skip.protoc} - - - - - compile - compile-custom - - - - - - - - com.salesforce.servicelibs - proto-backwards-compatibility - - ${basedir}/src/main/proto - - - - - backwards-compatibility-check - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - true - - - - - com.mycila - license-maven-plugin - - true - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - - diff --git a/core/transport/src/main/proto/grpc/block_worker.proto b/core/transport/src/main/proto/grpc/block_worker.proto deleted file mode 100644 index 3f5d2a5d6667..000000000000 --- a/core/transport/src/main/proto/grpc/block_worker.proto +++ /dev/null @@ -1,239 +0,0 @@ -syntax = "proto2"; - -option java_multiple_files = true; -option java_package = "alluxio.grpc"; -option java_outer_classname = "BlockWorkerProto"; - -package alluxio.grpc.block; - -import "proto/dataserver/protocol.proto"; -import "grpc/common.proto"; - -// The block worker service -service BlockWorker { - rpc ReadBlock (stream ReadRequest) returns (stream ReadResponse); - rpc WriteBlock (stream WriteRequest) returns (stream WriteResponse); - - // Replaces ShortCircuitBlockReadHandler. - rpc OpenLocalBlock (stream OpenLocalBlockRequest) returns (stream OpenLocalBlockResponse); - - // Replaces ShortCircuitBlockWriteHandler. - rpc CreateLocalBlock (stream CreateLocalBlockRequest) returns (stream CreateLocalBlockResponse); - rpc AsyncCache (AsyncCacheRequest) returns (AsyncCacheResponse); - rpc Cache (CacheRequest) returns (CacheResponse); - rpc Load(LoadRequest)returns (LoadResponse); - rpc RemoveBlock (RemoveBlockRequest) returns (RemoveBlockResponse); - rpc MoveBlock (MoveBlockRequest) returns (MoveBlockResponse); - - // TODO(lu) Move to metrics worker - rpc ClearMetrics (ClearMetricsRequest) returns (ClearMetricsResponse); - - rpc FreeWorker (FreeWorkerRequest) returns (FreeWorkerResponse); -} - -// The check request -message CheckRequest {} - -// The check response -message CheckResponse {} - -// The data chunk. -// next available id: 2 -message Chunk { - optional bytes data = 1; -} - -// The read/write request type. It can either be an Alluxio block operation or a UFS file operation. -// next available id: 3 -enum RequestType { - ALLUXIO_BLOCK = 0; - UFS_FILE = 1; - UFS_FALLBACK_BLOCK = 2; -} - -// The read request. -// next available id: 9 -message ReadRequest { - optional int64 block_id = 1; - optional int64 offset = 2; - optional int64 length = 3; - // Whether the block should be promoted before reading - optional bool promote = 4; - optional int64 chunk_size = 5; - - // This is only set for UFS block read. - optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 6; - - // Read receipt - optional int64 offset_received = 7; - - // Is position read to a small buffer - optional bool position_short = 8; -} - -// The read response. -// next available id: 2 -message ReadResponse { - optional Chunk chunk = 1; -} - -// The write request command. -// next available id: 11 -message WriteRequestCommand { - optional RequestType type = 1; - // The block ID or UFS file ID. - optional int64 id = 2; - optional int64 offset = 3; - // This is only applicable for block write. - optional int32 tier = 4; - optional bool flush = 5; - // Cancel, close and error will be handled by standard gRPC stream APIs. - optional alluxio.proto.dataserver.CreateUfsFileOptions create_ufs_file_options = 6; - optional alluxio.proto.dataserver.CreateUfsBlockOptions create_ufs_block_options = 7; - optional string medium_type = 8; - optional bool pin_on_create = 9; - optional int64 space_to_reserve = 10; -} - -// The write request. -// next available id: 3 -message WriteRequest { - oneof value { - WriteRequestCommand command = 1; - Chunk chunk = 2; - } -} - -// The write response. -// next available id: 2 -message WriteResponse { - optional int64 offset = 1; - // Errors will be handled by standard gRPC stream APIs. -} - -// Request for caching a block asynchronously -// Deprecated and will be removed in v3.0 -message AsyncCacheRequest { - optional int64 block_id = 1; - // TODO(calvin): source host and port should be replace with WorkerNetAddress - optional string source_host = 2; - optional int32 source_port = 3; - optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 4; - optional int64 length = 5; -} - -// Request for caching a block synchronously/asynchronously -// next available id: 7 -message CacheRequest { - optional int64 block_id = 1; - // TODO(calvin): source host and port should be replace with WorkerNetAddress - optional string source_host = 2; - optional int32 source_port = 3; - optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 4; - optional int64 length = 5; - optional bool async = 6; -} - -// Request for load a block into alluxio -// next available id: 3 -message LoadRequest { - repeated Block blocks = 1; - required UfsReadOptions options = 2; -} - -message UfsReadOptions{ - required string tag = 1; - // is position short or not, used for HDFS performance optimization. - // When the client buffer size is large ( > 2MB) and reads are guaranteed to be somewhat - // sequential, the `pread` API to HDFS is not as efficient as simple `read`. - // We introduce a heuristic to choose which API to use. - required bool position_short = 2; - optional int64 bandwidth = 3; -} - - -message Block{ - required int64 block_id = 1; - // The block length. - required int64 length = 2; - optional string ufs_path = 3; - // The offset of the block in within ufs the file. - optional int64 offset_in_file = 4; - optional int64 mountId = 5; -} - -message LoadResponse { - required TaskStatus status = 1; - repeated BlockStatus block_status = 2; -} - -enum TaskStatus { - SUCCESS = 0; - FAILURE = 1; - PARTIAL_FAILURE = 2; -} - -message FreeWorkerRequest{} - -message FreeWorkerResponse{} - -message BlockStatus { - required Block block = 1; - // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. - required int32 code = 2; - // A developer-facing error message - optional string message = 3; - optional bool retryable = 4; -} - -// Response for an async cache request -message AsyncCacheResponse {} - -// Response for an async cache request -message CacheResponse {} - -// next available id: 3 -message OpenLocalBlockRequest { - optional int64 block_id = 1; - optional bool promote = 2; -} - -// next available id: 2 -message OpenLocalBlockResponse { - optional string path = 1; -} - -// next available id: 9 -message CreateLocalBlockRequest { - optional int64 block_id = 1; - optional int32 tier = 3; - optional int64 space_to_reserve = 4; - // If set, only reserve space for the block. - optional bool only_reserve_space = 5; - optional bool cleanup_on_failure = 6; - optional string medium_type = 7; - optional bool pin_on_create = 8; -} - -// next available id: 2 -message CreateLocalBlockResponse { - optional string path = 1; -} - -// next available id: 2 -message RemoveBlockRequest { - optional int64 block_id = 1; -} - -message RemoveBlockResponse {} - -message MoveBlockRequest { - optional int64 block_id = 1; - optional string medium_type = 2; -} - -message MoveBlockResponse {} - -message ClearMetricsRequest {} - -message ClearMetricsResponse {} diff --git a/core/transport/src/main/proto/grpc/raft_journal.proto b/core/transport/src/main/proto/grpc/raft_journal.proto deleted file mode 100644 index c6f6dc5b2001..000000000000 --- a/core/transport/src/main/proto/grpc/raft_journal.proto +++ /dev/null @@ -1,79 +0,0 @@ -syntax = "proto2"; - -option java_multiple_files = true; -option java_package = "alluxio.grpc"; -option java_outer_classname = "RaftJournalProto"; - -package alluxio.grpc.meta; - -import "grpc/common.proto"; - -message JournalQueryRequest { - optional GetSnapshotInfoRequest snapshotInfoRequest = 1; - optional GetSnapshotRequest snapshotRequest = 2; - optional AddQuorumServerRequest addQuorumServerRequest = 3; -} - -message JournalQueryResponse { - optional GetSnapshotInfoResponse snapshotInfoResponse = 1; -} - -message AddQuorumServerRequest { - optional NetAddress serverAddress = 1; -} - -message GetSnapshotInfoRequest { - optional SnapshotMetadata snapshotInfo = 1; -} - -message GetSnapshotInfoResponse { - optional SnapshotMetadata latest = 1; -} - -message GetSnapshotRequest { -} - -message SnapshotMetadata { - optional int64 snapshotTerm = 1; - optional int64 snapshotIndex = 2; -} - -message SnapshotData { - optional int64 snapshotTerm = 1; - optional int64 snapshotIndex = 2; - optional bytes chunk = 3; - optional int64 offset = 4; - optional bool eof = 5; -} - -message UploadSnapshotPRequest { - optional SnapshotData data = 1; -} - -message UploadSnapshotPResponse { - optional int64 offsetReceived = 1; -} - -message DownloadSnapshotPRequest { - optional int64 offsetReceived = 1; -} - -message DownloadSnapshotPResponse { - optional SnapshotData data = 1; -} - -/** - * This interface contains raft service endpoints for Alluxio masters. - */ -service RaftJournalService { - - /** - * Uploads a snapshot to primary master. - */ - rpc UploadSnapshot (stream UploadSnapshotPRequest) returns (stream UploadSnapshotPResponse); - - /** - * Downloads a snapshot from primary master. - */ - rpc DownloadSnapshot (stream DownloadSnapshotPRequest) returns (stream DownloadSnapshotPResponse); -} diff --git a/core/transport/src/main/proto/grpc/version.proto b/core/transport/src/main/proto/grpc/version.proto deleted file mode 100644 index 8126353ecd9e..000000000000 --- a/core/transport/src/main/proto/grpc/version.proto +++ /dev/null @@ -1,49 +0,0 @@ -syntax = "proto2"; - -option java_multiple_files = true; -option java_package = "alluxio.grpc"; -option java_outer_classname = "VersionProto"; - -package alluxio.grpc.version; - - -enum ServiceType { - // FILE_SYSTEM_WORKER_WORKER_SERVICE is replaced by BLOCK_WORKER_CLIENT_SERVICE - // as a clearer type name - reserved 12; - reserved "FILE_SYSTEM_WORKER_WORKER_SERVICE"; - - UNKNOWN_SERVICE = 0; - FILE_SYSTEM_MASTER_CLIENT_SERVICE = 1; - FILE_SYSTEM_MASTER_WORKER_SERVICE = 2; - FILE_SYSTEM_MASTER_JOB_SERVICE = 3; - BLOCK_MASTER_CLIENT_SERVICE = 4; - BLOCK_MASTER_WORKER_SERVICE = 5; - META_MASTER_CONFIG_SERVICE = 6; - META_MASTER_CLIENT_SERVICE = 7; - META_MASTER_MASTER_SERVICE = 8; - METRICS_MASTER_CLIENT_SERVICE = 9; - JOB_MASTER_CLIENT_SERVICE = 10; - JOB_MASTER_WORKER_SERVICE = 11; - JOURNAL_MASTER_CLIENT_SERVICE = 13; - TABLE_MASTER_CLIENT_SERVICE = 14; - META_MASTER_BACKUP_MESSAGING_SERVICE = 15; - RAFT_JOURNAL_SERVICE = 16; - BLOCK_WORKER_CLIENT_SERVICE = 17; -} - -message GetServiceVersionPRequest { - optional ServiceType serviceType = 1; -} -message GetServiceVersionPResponse { - optional int64 version = 1; -} - -service ServiceVersionClientService { - - /** - * Returns the version of the master service. - * NOTE: The version should be updated every time a backwards incompatible API change occurs. - */ - rpc getServiceVersion(GetServiceVersionPRequest) returns (GetServiceVersionPResponse); -} diff --git a/core/transport/src/main/proto/proto/dataserver/protocol.proto b/core/transport/src/main/proto/proto/dataserver/protocol.proto deleted file mode 100644 index 24ab65b0cb15..000000000000 --- a/core/transport/src/main/proto/proto/dataserver/protocol.proto +++ /dev/null @@ -1,56 +0,0 @@ -syntax = "proto2"; - -package alluxio.proto.dataserver; - -import "proto/dataserver/status.proto"; -import "proto/shared/acl.proto"; - -// Options to open a UFS block. -// next available id: 7 -message OpenUfsBlockOptions { - optional string ufs_path = 1; - // The offset of the block in within the file. - optional int64 offset_in_file = 2; - // The block size. - optional int64 block_size = 3; - optional int32 maxUfsReadConcurrency = 4; - optional int64 mountId = 5; - // If set, do not try to cache the block locally when reading the data from the UFS. - optional bool no_cache = 6; - // The client does not need to set this. This is set by the worker. - optional string user = 7; - // If set to true, the block is possibly stored as a UFS block. - optional bool block_in_ufs_tier = 8; -} - -// Options to create a UFS file. -// next available: 6 -message CreateUfsFileOptions { - optional string ufs_path = 1; - optional string owner = 2; - optional string group = 3; - optional int32 mode = 4; - optional int64 mount_id = 5; - optional alluxio.proto.shared.AccessControlList acl = 6; -} - -// Options to create a block file in UFS. -// next available: 3 -message CreateUfsBlockOptions { - // the number of bytes previously written to block store - // zero if no previous temp block created - optional int64 bytes_in_block_store = 1; - // mount ID of the UFS to write to - optional int64 mount_id = 2; - // true if the write request is already a fallback from - // a failed short-circuit write. - optional bool fallback = 3; -} - -// The response. -// next available id: 3 -message Response { - optional status.PStatus status = 1; - optional string message = 2; -} - diff --git a/core/transport/src/main/proto/proto/journal/table.proto b/core/transport/src/main/proto/proto/journal/table.proto deleted file mode 100644 index ed5669191b00..000000000000 --- a/core/transport/src/main/proto/proto/journal/table.proto +++ /dev/null @@ -1,89 +0,0 @@ -syntax = "proto2"; - -package alluxio.proto.journal; - -import "grpc/table/table_master.proto"; - -// Journal entry messages for the table master - -// next available id: 6 -message AttachDbEntry { - optional string udb_type = 1; - optional string udb_connection_uri = 2; - optional string udb_db_name = 3; - optional string db_name = 4; - map config = 5; -} - -// next available id: 2 -message DetachDbEntry { - optional string db_name = 1; -} - -// next available id: 13 -message AddTableEntry { - optional string db_name = 1; - optional string table_name = 2; - optional string owner = 3; - optional alluxio.grpc.table.Schema schema = 4; - optional alluxio.grpc.table.Layout layout = 5; - repeated alluxio.grpc.table.ColumnStatisticsInfo table_stats = 6; - map parameters = 7; - - // partitioning scheme - repeated alluxio.grpc.table.FieldSchema partition_cols = 8; - repeated alluxio.grpc.table.Partition partitions = 9; - - optional int64 previous_version = 10; - optional int64 version = 11; - optional int64 version_creation_time = 12; -} - -// next available id: 5 -message AddTablePartitionsEntry { - optional string db_name = 1; - optional string table_name = 2; - optional int64 version = 3; - repeated alluxio.grpc.table.Partition partitions = 4; -} - -// next available id: 5 -message RemoveTableEntry { - optional string db_name = 1; - optional string table_name = 2; - optional int64 version = 4; -} - -// next available id: 5 -message CompleteTransformTableEntry { - optional string db_name = 1; - optional string table_name = 2; - optional string definition = 3; - // Map from partition spec to transformed layout - map transformed_layouts = 4; -} - -// next available id: 6 -message AddTransformJobInfoEntry { - optional string db_name = 1; - optional string table_name = 2; - optional string definition = 3; - optional int64 job_id = 4; - map transformed_layouts = 5; -} - -// next available id: 3 -message RemoveTransformJobInfoEntry { - optional string db_name = 1; - optional string table_name = 2; -} - -// next available id: 6 -message UpdateDatabaseInfoEntry { - optional string db_name = 1; - optional string location = 2; - map parameter = 3; - optional string owner_name = 4; - optional alluxio.grpc.table.PrincipalType owner_type = 5; - optional string comment = 6; -} diff --git a/dev/jenkins/Dockerfile-jdk11 b/dev/github/Dockerfile-jdk11 similarity index 80% rename from dev/jenkins/Dockerfile-jdk11 rename to dev/github/Dockerfile-jdk11 index d4c9d6cc8509..1e21e7e96531 100644 --- a/dev/jenkins/Dockerfile-jdk11 +++ b/dev/github/Dockerfile-jdk11 @@ -11,7 +11,28 @@ # See https://hub.docker.com/r/alluxio/alluxio-maven for instructions on running the image. -FROM maven:3.6.3-jdk-11 +ARG RUST_VERSION=1.66.1 +# Official rust image from https://github.com/rust-lang/docker-rust/blob/master/Dockerfile-debian.template +# This image installs rustup and cargo into /usr/local/{rustup,cargo} +# we install additional targets and components, +# and copy the toolchains from this image into the maven base image in the next stage +FROM rust:${RUST_VERSION} as rust-installer +RUN rustup target add aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu && \ + rustup component add clippy rustfmt && \ + # move the two directories into an empty directory to save one COPY instruction in the next stage + mkdir /cargo-and-rustup && \ + mv /usr/local/rustup /cargo-and-rustup/rustup && \ + mv /usr/local/cargo /cargo-and-rustup/cargo + +FROM maven:3.8.6-jdk-11 +# ARGs only last for the build phase of a single image. Redefine it to make it available in this stage as well. +# See https://stackoverflow.com/questions/53681522/share-variable-in-multi-stage-dockerfile-arg-before-from-not-substituted +ARG RUST_VERSION +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=${RUST_VERSION} +COPY --from=rust-installer /cargo-and-rustup /usr/local/ # reference: https://github.com/docker-library/openjdk/blob/master/8/jdk/buster/Dockerfile # we need jdk 8 in jdk 11 so that we can compile with jdk 8 and test with jdk 11 @@ -138,7 +159,7 @@ RUN mkdir -p /home/jenkins && \ chmod -R 777 /.config && \ apt-get update -y && \ apt-get upgrade -y ca-certificates && \ - apt-get install -y build-essential fuse3 libfuse3-dev libfuse-dev make ruby ruby-dev + apt-get install -y build-essential docker.io fuse3 libfuse3-dev libfuse-dev make ruby ruby-dev # jekyll for documentation RUN gem install public_suffix:4.0.7 jekyll:4.2.2 bundler:2.3.18 # golang for tooling @@ -153,3 +174,14 @@ RUN ARCH=$(dpkg --print-architecture) && \ wget --quiet https://releases.hashicorp.com/terraform/1.0.1/terraform_1.0.1_linux_${ARCH}.zip && \ unzip -o ./terraform_1.0.1_linux_${ARCH}.zip -d /usr/local/bin/ && \ rm terraform_1.0.1_linux_${ARCH}.zip +# UCX for RDMA +RUN wget https://github.com/openucx/ucx/releases/download/v1.16.0/ucx-1.16.0.tar.gz && \ + tar xzf ucx-1.16.0.tar.gz && \ + cd ucx-1.16.0 && \ + mkdir build && \ + cd build && \ + ../configure --prefix=/usr/local --without-go && \ + make -j4 && \ + make install && \ + echo "/usr/local/lib" | tee /etc/ld.so.conf.d/ucx.conf && \ + ldconfig \ diff --git a/dev/github/Dockerfile-jdk17 b/dev/github/Dockerfile-jdk17 new file mode 100644 index 000000000000..f0b95c4cb75d --- /dev/null +++ b/dev/github/Dockerfile-jdk17 @@ -0,0 +1,192 @@ +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +# See https://hub.docker.com/r/alluxio/alluxio-maven for instructions on running the image. + +ARG RUST_VERSION=1.66.1 +# Official rust image from https://github.com/rust-lang/docker-rust/blob/master/Dockerfile-debian.template +# This image installs rustup and cargo into /usr/local/{rustup,cargo} +# we install additional targets and components, +# and copy the toolchains from this image into the maven base image in the next stage +FROM rust:${RUST_VERSION} as rust-installer +RUN rustup target add aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu && \ + rustup component add clippy rustfmt && \ + # move the two directories into an empty directory to save one COPY instruction in the next stage + mkdir /cargo-and-rustup && \ + mv /usr/local/rustup /cargo-and-rustup/rustup && \ + mv /usr/local/cargo /cargo-and-rustup/cargo + +FROM maven:3.8.6-eclipse-temurin-17 +# ARGs only last for the build phase of a single image. Redefine it to make it available in this stage as well. +# See https://stackoverflow.com/questions/53681522/share-variable-in-multi-stage-dockerfile-arg-before-from-not-substituted +ARG RUST_VERSION +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=${RUST_VERSION} +COPY --from=rust-installer /cargo-and-rustup /usr/local/ + +# reference: https://github.com/docker-library/openjdk/blob/master/8/jdk/buster/Dockerfile +# we need jdk 8 in jdk 17 so that we can compile with jdk 8 and test with jdk 17 + +ENV JAVA_HOME /usr/local/openjdk-8 +ENV PATH $JAVA_HOME/bin:$PATH + +# backwards compatibility shim +RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ] + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + bzip2 \ + unzip \ + xz-utils \ + \ +# java.lang.UnsatisfiedLinkError: /usr/local/openjdk-11/lib/libfontmanager.so: libfreetype.so.6: cannot open shared object file: No such file or directory +# java.lang.NoClassDefFoundError: Could not initialize class sun.awt.X11FontManager +# https://github.com/docker-library/openjdk/pull/235#issuecomment-424466077 + fontconfig libfreetype6 \ + \ +# utilities for keeping Debian and OpenJDK CA certificates in sync + ca-certificates p11-kit \ +# additional packages not installed in slim base image \ + wget gnupg git \ + ; \ + rm -rf /var/lib/apt/lists/* + +# https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor +RUN git config --global safe.directory '*' + +ENV JAVA_HOME /usr/local/openjdk-8 +RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ] # backwards compatibility +ENV PATH $JAVA_HOME/bin:$PATH + +# Default to UTF-8 file.encoding +ENV LANG C.UTF-8 + +# https://adoptopenjdk.net/upstream.html +# > +# > What are these binaries? +# > +# > These binaries are built by Red Hat on their infrastructure on behalf of the OpenJDK jdk8u and jdk11u projects. The binaries are created from the unmodified source code at OpenJDK. Although no formal support agreement is provided, please report any bugs you may find to https://bugs.java.com/. +# > +ENV JAVA_VERSION 8u332 +# https://github.com/docker-library/openjdk/issues/320#issuecomment-494050246 +# > +# > I am the OpenJDK 8 and 11 Updates OpenJDK project lead. +# > ... +# > While it is true that the OpenJDK Governing Board has not sanctioned those releases, they (or rather we, since I am a member) didn't sanction Oracle's OpenJDK releases either. As far as I am aware, the lead of an OpenJDK project is entitled to release binary builds, and there is clearly a need for them. +# > + +RUN set -eux; \ + \ + arch="$(dpkg --print-architecture)"; \ + case "$arch" in \ + 'amd64') \ + downloadUrl='https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u332-b09/OpenJDK8U-jdk_x64_linux_8u332b09.tar.gz'; \ + ;; \ + 'arm64') \ + downloadUrl='https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u332-b09/OpenJDK8U-jdk_aarch64_linux_8u332b09.tar.gz'; \ + ;; \ + *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ + esac; \ + \ + wget --progress=dot:giga -O openjdk.tgz "$downloadUrl"; \ + wget --progress=dot:giga -O openjdk.tgz.asc "$downloadUrl.sign"; \ + \ + export GNUPGHOME="$(mktemp -d)"; \ +# pre-fetch Andrew Haley's (the OpenJDK 8 and 11 Updates OpenJDK project lead) key so we can verify that the OpenJDK key was signed by it +# (https://github.com/docker-library/openjdk/pull/322#discussion_r286839190) +# we pre-fetch this so that the signature it makes on the OpenJDK key can survive "import-clean" in gpg + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys EAC843EBD3EFDB98CC772FADA5CD6035332FA671; \ +# TODO find a good link for users to verify this key is right (https://mail.openjdk.java.net/pipermail/jdk-updates-dev/2019-April/000951.html is one of the only mentions of it I can find); perhaps a note added to https://adoptopenjdk.net/upstream.html would make sense? +# no-self-sigs-only: https://salsa.debian.org/debian/gnupg2/commit/c93ca04a53569916308b369c8b218dad5ae8fe07 + gpg --batch --keyserver keyserver.ubuntu.com --keyserver-options no-self-sigs-only --recv-keys CA5F11C6CE22644D42C6AC4492EF8D39DC13168F; \ + gpg --batch --list-sigs --keyid-format 0xLONG CA5F11C6CE22644D42C6AC4492EF8D39DC13168F \ + | tee /dev/stderr \ + | grep '0xA5CD6035332FA671' \ + | grep 'Andrew Haley'; \ + gpg --batch --verify openjdk.tgz.asc openjdk.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + \ + mkdir -p "$JAVA_HOME"; \ + tar --extract \ + --file openjdk.tgz \ + --directory "$JAVA_HOME" \ + --strip-components 1 \ + --no-same-owner \ + ; \ + rm openjdk.tgz*; \ + \ +# update "cacerts" bundle to use Debian's CA certificates (and make sure it stays up-to-date with changes to Debian's store) +# see https://github.com/docker-library/openjdk/issues/327 +# http://rabexc.org/posts/certificates-not-working-java#comment-4099504075 +# https://salsa.debian.org/java-team/ca-certificates-java/blob/3e51a84e9104823319abeb31f880580e46f45a98/debian/jks-keystore.hook.in +# https://git.alpinelinux.org/aports/tree/community/java-cacerts/APKBUILD?id=761af65f38b4570093461e6546dcf6b179d2b624#n29 + { \ + echo '#!/usr/bin/env bash'; \ + echo 'set -Eeuo pipefail'; \ + echo 'trust extract --overwrite --format=java-cacerts --filter=ca-anchors --purpose=server-auth "$JAVA_HOME/jre/lib/security/cacerts"'; \ + } > /etc/ca-certificates/update.d/docker-openjdk; \ + chmod +x /etc/ca-certificates/update.d/docker-openjdk; \ + /etc/ca-certificates/update.d/docker-openjdk; \ + \ +# https://github.com/docker-library/openjdk/issues/331#issuecomment-498834472 + find "$JAVA_HOME/lib" -name '*.so' -exec dirname '{}' ';' | sort -u > /etc/ld.so.conf.d/docker-openjdk.conf; \ + ldconfig; \ + \ +# basic smoke test to see if the default is jdk 8 + javac -version; \ + java -version + +ENV JAVA_HOME /usr/local/openjdk-17 +ENV PATH $JAVA_HOME/bin:$PATH + +# basic smoke test to see if the default is jdk 17 +RUN set -eux; \ + javac -version; \ + java -version + +# need to create /.config to avoid npm errors +RUN mkdir -p /home/jenkins && \ + chmod -R 777 /home/jenkins && \ + chmod g+w /etc/passwd && \ + mkdir -p /.config && \ + chmod -R 777 /.config && \ + apt-get update -y && \ + apt-get upgrade -y ca-certificates && \ + apt-get install -y build-essential docker.io fuse3 libfuse3-dev libfuse-dev make ruby ruby-dev +# jekyll for documentation +RUN gem install public_suffix:4.0.7 jekyll:4.2.2 bundler:2.3.18 +# golang for tooling +RUN ARCH=$(dpkg --print-architecture) && \ + wget https://go.dev/dl/go1.18.1.linux-${ARCH}.tar.gz && \ + tar -xvf go1.18.1.linux-${ARCH}.tar.gz && \ + mv go /usr/local +ENV GOROOT=/usr/local/go +ENV PATH=$GOROOT/bin:$PATH +# terraform for deployment scripts +RUN ARCH=$(dpkg --print-architecture) && \ + wget --quiet https://releases.hashicorp.com/terraform/1.0.1/terraform_1.0.1_linux_${ARCH}.zip && \ + unzip -o ./terraform_1.0.1_linux_${ARCH}.zip -d /usr/local/bin/ && \ + rm terraform_1.0.1_linux_${ARCH}.zip +# UCX for RDMA +RUN wget https://github.com/openucx/ucx/releases/download/v1.16.0/ucx-1.16.0.tar.gz && \ + tar xzf ucx-1.16.0.tar.gz && \ + cd ucx-1.16.0 && \ + mkdir build && \ + cd build && \ + ../configure --prefix=/usr/local --without-go && \ + make -j4 && \ + make install && \ + echo "/usr/local/lib" | tee /etc/ld.so.conf.d/ucx.conf && \ + ldconfig \ diff --git a/dev/github/Dockerfile-jdk8 b/dev/github/Dockerfile-jdk8 new file mode 100644 index 000000000000..9f37e6149f84 --- /dev/null +++ b/dev/github/Dockerfile-jdk8 @@ -0,0 +1,70 @@ +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +# See https://hub.docker.com/r/alluxio/alluxio-maven for instructions on running the image. + +ARG RUST_VERSION=1.66.1 +# Official rust image from https://github.com/rust-lang/docker-rust/blob/master/Dockerfile-debian.template +# This image installs rustup and cargo into /usr/local/{rustup,cargo} +# we install additional targets and components, +# and copy the toolchains from this image into the maven base image in the next stage +FROM rust:${RUST_VERSION} as rust-installer +RUN rustup target add aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu && \ + rustup component add clippy rustfmt && \ + # move the two directories into an empty directory to save one COPY instruction in the next stage + mkdir /cargo-and-rustup && \ + mv /usr/local/rustup /cargo-and-rustup/rustup && \ + mv /usr/local/cargo /cargo-and-rustup/cargo + +FROM maven:3.8.6-jdk-8 +# ARGs only last for the build phase of a single image. Redefine it to make it available in this stage as well. +# See https://stackoverflow.com/questions/53681522/share-variable-in-multi-stage-dockerfile-arg-before-from-not-substituted +ARG RUST_VERSION +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=${RUST_VERSION} +COPY --from=rust-installer /cargo-and-rustup /usr/local/ + +# need to create /.config to avoid npm errors +RUN mkdir -p /home/jenkins && \ + chmod -R 777 /home/jenkins && \ + chmod g+w /etc/passwd && \ + mkdir -p /.config && \ + chmod -R 777 /.config && \ + apt-get update -y && \ + apt-get upgrade -y ca-certificates && \ + apt-get install -y build-essential docker.io fuse3 libfuse3-dev libfuse-dev make ruby ruby-dev +# jekyll for documentation +RUN gem install public_suffix:4.0.7 jekyll:4.2.2 bundler:2.3.18 +# golang for tooling +RUN ARCH=$(dpkg --print-architecture) && \ + wget https://go.dev/dl/go1.18.1.linux-${ARCH}.tar.gz && \ + tar -xvf go1.18.1.linux-${ARCH}.tar.gz && \ + mv go /usr/local +ENV GOROOT=/usr/local/go +ENV PATH=$GOROOT/bin:$PATH +# terraform for deployment scripts +RUN ARCH=$(dpkg --print-architecture) && \ + wget --quiet https://releases.hashicorp.com/terraform/1.0.1/terraform_1.0.1_linux_${ARCH}.zip && \ + unzip -o ./terraform_1.0.1_linux_${ARCH}.zip -d /usr/local/bin/ && \ + rm terraform_1.0.1_linux_${ARCH}.zip +# UCX for RDMA +RUN wget https://github.com/openucx/ucx/releases/download/v1.16.0/ucx-1.16.0.tar.gz && \ + tar xzf ucx-1.16.0.tar.gz && \ + cd ucx-1.16.0 && \ + mkdir build && \ + cd build && \ + ../configure --prefix=/usr/local --without-go && \ + make -j4 && \ + make install && \ + echo "/usr/local/lib" | tee /etc/ld.so.conf.d/ucx.conf && \ + ldconfig \ diff --git a/dev/github/component_owners.yaml b/dev/github/component_owners.yaml deleted file mode 100644 index 89c1ef46290f..000000000000 --- a/dev/github/component_owners.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# -# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 -# (the "License"). You may not use this work except in compliance with the License, which is -# available at www.apache.org/licenses/LICENSE-2.0 -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied, as more fully set forth in the License. -# -# See the NOTICE file distributed with this work for information regarding copyright ownership. -# - -# Mapping of Alluxio component to Github user(s) that will be automatically assigned for pull requests -# Map -component-owner-mapping: - default_component: # handles components that were not added to this mapping yet - - yuzhu - assembly: - - yuzhu - bin: - - LuQQiu - build: - - yuzhu - conf: - - rongrong - core/client/fs/src/main/java/alluxio/client/file/cache/: - - beinan - core: - - tcrain - dev: - - Xenorith - docs: - - apc999 - examples: - - LuQQiu - integration/csi: - - ssz1997 - integration/jnifuse: - - LuQQiu - integration: - - ssz1997 - job: - - rongrong - libexec: - - yuzhu - minicluster: - - tcrain - shell: - - LuQQiu - stress: - - dbw9580 - table: - - HelloHorizon - tests: - - tcrain - underfs: - - fuzhengjia - webui: - - tieujason330 diff --git a/dev/github/run_checks.sh b/dev/github/run_checks.sh index c59b13f35087..8d8e1d4d0ac8 100755 --- a/dev/github/run_checks.sh +++ b/dev/github/run_checks.sh @@ -17,25 +17,21 @@ set -ex if [ -n "${ALLUXIO_GIT_CLEAN}" ] then + # https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor + git config --global --add safe.directory '*' git clean -fdx fi -mvn_args="" -if [ -n "${ALLUXIO_MVN_RUNTOEND}" ] -then - mvn_args+=" -fn -DfailIfNoTests=false --fail-at-end" -fi - export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" # Always use java 8 to compile the source code -JAVA_HOME_BACKUP=${JAVA_HOME} -PATH_BACKUP=${PATH} JAVA_HOME=/usr/local/openjdk-8 PATH=$JAVA_HOME/bin:$PATH -mvn -Duser.home=/home/jenkins -T 4C clean install -Pdeveloper -DskipTests -Dmaven.javadoc.skip \ --Dsurefire.forkCount=2 ${mvn_args} +mvn -Duser.home=/home/jenkins -T 4C clean install -DskipTests -./dev/scripts/check-docs.sh -./dev/scripts/generate-tarballs checkUfsVersions +# compile go cli +./build/cli/build-cli.sh +./dev/scripts/check-docs.sh +./dev/scripts/build-artifact.sh ufsVersionCheck +./dev/scripts/build-artifact.sh tarball --dryRun diff --git a/dev/github/run_docker.sh b/dev/github/run_docker.sh index c4ca078dee12..dea462ec10ea 100755 --- a/dev/github/run_docker.sh +++ b/dev/github/run_docker.sh @@ -13,51 +13,38 @@ set -ex function main { - if [ -z "${ALLUXIO_DOCKER_ID}" ] - then + if [ -z "${ALLUXIO_DOCKER_ID}" ]; then ALLUXIO_DOCKER_ID="$(id -u)" fi - if [ -z "${ALLUXIO_DOCKER_M2}" ] - then + + if [ -z "${ALLUXIO_DOCKER_M2}" ]; then ALLUXIO_DOCKER_M2="${HOME}/.m2" fi - if [ -z "${ALLUXIO_DOCKER_IMAGE}" ] - then - ALLUXIO_DOCKER_IMAGE="alluxio/alluxio-maven:0.0.8-jdk8" + + if [ -z "${ALLUXIO_DOCKER_IMAGE}" ]; then + ALLUXIO_DOCKER_IMAGE="alluxio/alluxio-maven:0.1.3-jdk8" fi local run_args="--rm" - if [ -z ${ALLUXIO_DOCKER_NO_TTY} ] - then + if [ -z "${ALLUXIO_DOCKER_NO_TTY}" ]; then run_args+=" -it" fi - if [ -n "${ALLUXIO_DOCKER_FORK_COUNT}" ] - then + if [ -n "${ALLUXIO_DOCKER_FORK_COUNT}" ]; then run_args+=" -e ALLUXIO_FORK_COUNT=${ALLUXIO_DOCKER_FORK_COUNT}" fi - if [ -n "${ALLUXIO_DOCKER_GIT_CLEAN}" ] - then + if [ -n "${ALLUXIO_DOCKER_GIT_CLEAN}" ]; then run_args+=" -e ALLUXIO_GIT_CLEAN=true" fi - if [ -n "${ALLUXIO_DOCKER_MVN_RUNTOEND}" ] - then - run_args+=" -e ALLUXIO_MVN_RUNTOEND=true" - fi - - if [ -n "${ALLUXIO_DOCKER_MVN_PROJECT_LIST}" ] - then + if [ -n "${ALLUXIO_DOCKER_MVN_PROJECT_LIST}" ]; then run_args+=" -e ALLUXIO_MVN_PROJECT_LIST=${ALLUXIO_DOCKER_MVN_PROJECT_LIST}" fi - if [ -n "${ALLUXIO_SONAR_ARGS}" ] - then - # write out to a file, in case there are spaces in the args - echo "ALLUXIO_SONAR_ARGS=${ALLUXIO_SONAR_ARGS}" > /tmp/.env - run_args+=" --env-file /tmp/.env" + if [ -n "${ALLUXIO_DOCKER_MVN_TESTS}" ]; then + run_args+=" -e ALLUXIO_MVN_TESTS=${ALLUXIO_DOCKER_MVN_TESTS}" fi local home="/home/jenkins" @@ -78,20 +65,20 @@ function main { # configure anything that's relative to ${HOME}. run_args+=" -e HOME=${home}" run_args+=" -v ${ALLUXIO_DOCKER_M2}:${home}/.m2" + run_args+=" -v /var/run/docker.sock:/var/run/docker.sock" run_args+=" -e npm_config_cache=${home}/.npm" run_args+=" -e MAVEN_CONFIG=${home}/.m2" run_args+=" -e ALLUXIO_USE_FIXED_TEST_PORTS=true" run_args+=" -e ALLUXIO_PORT_COORDINATION_DIR=${home}" - if [ -n "${ALLUXIO_CHECKSTYLE}" ] - then + if [ -n "${ALLUXIO_CHECKSTYLE}" ]; then run_args+=" --entrypoint=dev/github/run_checks.sh" else run_args+=" --entrypoint=dev/github/run_tests.sh" fi - docker run ${run_args} ${ALLUXIO_DOCKER_IMAGE} $@ + docker run ${run_args} ${ALLUXIO_DOCKER_IMAGE} } main "$@" diff --git a/dev/github/run_tests.sh b/dev/github/run_tests.sh index 13446db8d0ed..6ced0beafed3 100755 --- a/dev/github/run_tests.sh +++ b/dev/github/run_tests.sh @@ -15,26 +15,19 @@ # set -ex -if [ -z "${ALLUXIO_FORK_COUNT}" ] -then +if [ -z "${ALLUXIO_FORK_COUNT}" ]; then ALLUXIO_FORK_COUNT=2 fi -if [ -n "${ALLUXIO_GIT_CLEAN}" ] -then +if [ -n "${ALLUXIO_GIT_CLEAN}" ]; then + # https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor + git config --global --add safe.directory '*' git clean -fdx fi mvn_args="" -if [ -n "${ALLUXIO_MVN_RUNTOEND}" ] -then - mvn_args+=" -fn -DfailIfNoTests=false --fail-at-end" -fi - -mvn_project_list="" -if [ -n "${ALLUXIO_MVN_PROJECT_LIST}" ] -then - mvn_project_list+=" -pl ${ALLUXIO_MVN_PROJECT_LIST}" +if [ -n "${ALLUXIO_MVN_PROJECT_LIST}" ]; then + mvn_args+="-pl ${ALLUXIO_MVN_PROJECT_LIST}" fi export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" @@ -44,20 +37,23 @@ JAVA_HOME_BACKUP=${JAVA_HOME} PATH_BACKUP=${PATH} JAVA_HOME=/usr/local/openjdk-8 PATH=$JAVA_HOME/bin:$PATH -mvn -Duser.home=/home/jenkins -T 4C clean install -Pdeveloper -Dfindbugs.skip -Dcheckstyle.skip -DskipTests -Dmaven.javadoc.skip \ --Dlicense.skip -Dsurefire.forkCount=2 ${mvn_args} +mvn -Duser.home=/home/jenkins -T 4C clean install -Dfindbugs.skip -Dcheckstyle.skip -DskipTests -Dmaven.javadoc.skip \ +-Dlicense.skip -Dsort.skip -am ${mvn_args} # Set things up so that the current user has a real name and can authenticate. myuid=$(id -u) mygid=$(id -g) echo "$myuid:x:$myuid:$mygid:anonymous uid:/home/jenkins:/bin/false" >> /etc/passwd -export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" - # Revert back to the image default java version to run the test JAVA_HOME=${JAVA_HOME_BACKUP} PATH=${PATH_BACKUP} +mvn_args+=" -fn -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false --fail-at-end" +if [ -n "${ALLUXIO_MVN_TESTS}" ]; then + mvn_args+=" -Dtest=${ALLUXIO_MVN_TESTS}" +fi + # Run tests -mvn -Duser.home=/home/jenkins test -Pdeveloper -Dmaven.main.skip -Dskip.protoc=true -Dmaven.javadoc.skip -Dlicense.skip=true \ --Dcheckstyle.skip=true -Dfindbugs.skip=true -Dsurefire.forkCount=${ALLUXIO_FORK_COUNT} ${mvn_args} $@ +mvn -Duser.home=/home/jenkins test -Dmaven.main.skip -Dskip.protoc=true -Dmaven.javadoc.skip -Dlicense.skip=true \ +-Dcheckstyle.skip=true -Dfindbugs.skip=true -Dsort.skip -Dsurefire.forkCount=${ALLUXIO_FORK_COUNT} ${mvn_args} diff --git a/dev/intellij/runConfigurations/AlluxioDoraWorker_0.xml b/dev/intellij/runConfigurations/AlluxioDoraWorker_0.xml new file mode 100644 index 000000000000..83846186c4b7 --- /dev/null +++ b/dev/intellij/runConfigurations/AlluxioDoraWorker_0.xml @@ -0,0 +1,34 @@ + + + + + diff --git a/dev/intellij/runConfigurations/AlluxioDoraWorker_1.xml b/dev/intellij/runConfigurations/AlluxioDoraWorker_1.xml new file mode 100644 index 000000000000..0a6721b2d680 --- /dev/null +++ b/dev/intellij/runConfigurations/AlluxioDoraWorker_1.xml @@ -0,0 +1,34 @@ + + + + + diff --git a/dev/intellij/runConfigurations/AlluxioFuse.xml b/dev/intellij/runConfigurations/AlluxioFuse.xml index 8932ba8179ff..5c0325a6b1fe 100644 --- a/dev/intellij/runConfigurations/AlluxioFuse.xml +++ b/dev/intellij/runConfigurations/AlluxioFuse.xml @@ -15,7 +15,7 @@