merge 08-28

LICENSE.txt

@@ -202,6 +202,14 @@
 
 --------------------------------------------------------------------------------
 
+be/src/common/status.* : BSD-style license
+
+  Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+  Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+--------------------------------------------------------------------------------
+
 be/src/gutil/atomicops-internals-x86.h : Apache 2.0 License
 
 Copyright 2003 Google Inc.
@@ -220,7 +228,7 @@ Copyright 2007 Google Inc.
 
 --------------------------------------------------------------------------------
 
-be/src/gutil/utf: licensed under the following terms:
+be/src/gutil/utf/*: licensed under the following terms:
 
   UTF-8 Library
 
@@ -368,35 +376,17 @@ License.
 
 --------------------------------------------------------------------------------
 
-be/src/util/coding.*, be/src/util/status.*: 3-clause BSD
-
-Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+be/src/util/coding.*: this code is licensed under both GPLv2 and Apache 2.0 License.
+                      Doris chooses Apache 2.0 License.
+
+  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+  This source code is licensed under both the GPLv2 (found in the
+  COPYING file in the root directory) and Apache 2.0 License
+  (found in the LICENSE.Apache file in the root directory).
+
+  Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+  Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 --------------------------------------------------------------------------------
 
@@ -498,6 +488,32 @@ webroot/static/jquery.js : MIT license
 
 --------------------------------------------------------------------------------
 
+Sizzle in webroot/static/jquery.js : MIT license
+
+  Copyright 2013 jQuery Foundation and other contributors
+  http://jquery.com/
+
+  Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+
+  The above copyright notice and this permission notice shall be
+  included in all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
 webroot/static/jquery.dataTables.js and webroot/static/datatables*: MIT license
 
   Copyright (C) 2008-2015, SpryMedia Ltd.

be/src/agent/status.h

@@ -41,6 +41,7 @@ enum AgentStatus {
     DORIS_PUSH_HAD_LOADED = -504,
     DORIS_TIMEOUT = -901,
     DORIS_INTERNAL_ERROR = -902,
+    DORIS_DISK_REACH_CAPACITY_LIMIT = -903,
 };
 }  // namespace doris
 #endif  // DORIS_BE_SRC_AGENT_STATUS_H

be/src/agent/task_worker_pool.cpp

@@ -433,21 +433,35 @@ void* TaskWorkerPool::_create_tablet_worker_thread_callback(void* arg_this) {
         vector<string> error_msgs;
         TStatus task_status;
 
-        OLAPStatus create_status =
-            worker_pool_this->_env->storage_engine()->create_tablet(create_tablet_req);
+        std::vector<TTabletInfo> finish_tablet_infos;
+        OLAPStatus create_status = worker_pool_this->_env->storage_engine()->create_tablet(create_tablet_req);
         if (create_status != OLAPStatus::OLAP_SUCCESS) {
             OLAP_LOG_WARNING("create table failed. status: %d, signature: %ld",
                              create_status, agent_task_req.signature);
             // TODO liutao09 distinguish the OLAPStatus
             status_code = TStatusCode::RUNTIME_ERROR;
         } else {
             ++_s_report_version;
+            // get path hash of the created tablet
+            TabletSharedPtr tablet = StorageEngine::instance()->tablet_manager()->get_tablet(
+                create_tablet_req.tablet_id, create_tablet_req.tablet_schema.schema_hash);
+            DCHECK(tablet != nullptr);
+            TTabletInfo tablet_info;
+            tablet_info.tablet_id = tablet->table_id(); 
+            tablet_info.schema_hash = tablet->schema_hash();
+            tablet_info.version = create_tablet_req.version;
+            tablet_info.version_hash = create_tablet_req.version_hash;
+            tablet_info.row_count = 0;
+            tablet_info.data_size = 0;
+            tablet_info.__set_path_hash(tablet->data_dir()->path_hash());
+            finish_tablet_infos.push_back(tablet_info);
         }
 
         task_status.__set_status_code(status_code);
         task_status.__set_error_msgs(error_msgs);
 
         TFinishTaskRequest finish_task_request;
+        finish_task_request.__set_finish_tablet_infos(finish_tablet_infos);
         finish_task_request.__set_backend(worker_pool_this->_backend);
         finish_task_request.__set_report_version(_s_report_version);
         finish_task_request.__set_task_type(agent_task_req.task_type);
@@ -1252,7 +1266,7 @@ void* TaskWorkerPool::_report_disk_state_worker_thread_callback(void* arg_this)
         }
 #endif
         vector<DataDirInfo> data_dir_infos;
-        worker_pool_this->_env->storage_engine()->get_all_data_dir_info(&data_dir_infos);
+        worker_pool_this->_env->storage_engine()->get_all_data_dir_info(&data_dir_infos, true /* update */);
 
         map<string, TDisk> disks;
         for (auto& root_path_info : data_dir_infos) {

be/src/common/config.h

@@ -212,12 +212,11 @@ namespace config {
     // inc_rowset expired interval
     CONF_Int32(inc_rowset_expired_sec, "1800");
     // garbage sweep policy
-    CONF_Int32(max_garbage_sweep_interval, "14400");
+    CONF_Int32(max_garbage_sweep_interval, "3600");
     CONF_Int32(min_garbage_sweep_interval, "180");
     CONF_Int32(snapshot_expire_time_sec, "172800");
     // 仅仅是建议值，当磁盘空间不足时，trash下的文件保存期可不遵守这个参数
     CONF_Int32(trash_file_expire_time_sec, "259200");
-    CONF_Int32(disk_capacity_insufficient_percentage, "90");
     // check row nums for BE/CE and schema change. true is open, false is closed.
     CONF_Bool(row_nums_check, "true")
     //file descriptors cache, by default, cache 30720 descriptors
@@ -439,6 +438,13 @@ namespace config {
     CONF_Int32(path_gc_check_step, "1000");
     CONF_Int32(path_gc_check_step_interval_ms, "10");
     CONF_Int32(path_scan_interval_second, "86400");
+
+    // The following 2 configs limit the max usage of disk capacity of a data dir.
+    // If both of these 2 threshold reached, no more data can be writen into that data dir.
+    // The percent of max used capacity of a data dir
+    CONF_Int32(storage_flood_stage_usage_percent, "95");    // 95%
+    // The min bytes that should be left of a data dir
+    CONF_Int64(storage_flood_stage_left_capacity_bytes, "1073741824")   // 1GB
 } // namespace config
 
 } // namespace doris

be/src/common/status.cpp

@@ -1,20 +1,3 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

be/src/common/status.h

@@ -1,20 +1,3 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

be/src/exec/es/es_scroll_parser.cpp

@@ -149,7 +149,7 @@ Status ScrollParser::parse(const std::string& scroll_result) {
     VLOG(1) << "es_scan_reader total hits: " << _total << " documents";
     const rapidjson::Value &inner_hits_node = outer_hits_node[FIELD_INNER_HITS];
     if (!inner_hits_node.IsArray()) {
-        LOG(WARNING) << "errors while parse scroll reponse:" << scroll_result;
+        LOG(WARNING) << "exception maybe happend on es cluster, reponse:" << scroll_result;
         return Status::InternalError("inner hits node is not an array");
     }
 

be/src/exec/es/es_scroll_query.cpp

@@ -96,6 +96,7 @@ std::string ESScrollQueryBuilder::build(const std::map<std::string, std::string>
     rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
     es_query_dsl.Accept(writer);
     std::string es_query_dsl_json = buffer.GetString();
+    LOG(INFO) << "Generated ES queryDSL [ " << es_query_dsl_json << " ]";
     return es_query_dsl_json;                
 
 }

be/src/exec/text_converter.cpp

@@ -37,7 +37,7 @@ void TextConverter::unescape_string(StringValue* value, MemPool* pool) {
     value->ptr = new_data;
 }
 
-void TextConverter::unescape_string(const char* src, char* dest, int* len) {
+void TextConverter::unescape_string(const char* src, char* dest, size_t* len) {
     char* dest_ptr = dest;
     const char* end = src + *len;
     bool escape_next_char = false;

be/src/exec/text_converter.h

@@ -51,7 +51,7 @@ class TextConverter {
     // Removes escape characters from len characters of the null-terminated string src,
     // and copies the unescaped string into dest, changing *len to the unescaped length.
     // No null-terminator is added to dest.
-    void unescape_string(const char* src, char* dest, int* len);
+    void unescape_string(const char* src, char* dest, size_t* len);
 
     // Removes escape characters from 'str', allocating a new string from pool.
     // 'str' is updated with the new ptr and length.

be/src/olap/compaction.h

@@ -32,6 +32,7 @@
 
 namespace doris {
 
+class DataDir;
 class Merger;
 
 // This class is a base class for compaction.

be/src/olap/data_dir.cpp

@@ -59,14 +59,14 @@ DataDir::DataDir(const std::string& path, int64_t capacity_bytes,
         TabletManager* tablet_manager, TxnManager* txn_manager)
         : _path(path),
         _capacity_bytes(capacity_bytes),
+        _available_bytes(0),
+        _disk_capacity_bytes(0),
+        _is_used(false),
         _tablet_manager(tablet_manager),
         _txn_manager(txn_manager),
         _cluster_id(-1),
-        _available_bytes(0),
-        _used_bytes(0),
-        _current_shard(0),
-        _is_used(false),
         _to_be_deleted(false),
+        _current_shard(0),
         _test_file_read_buf(nullptr),
         _test_file_write_buf(nullptr),
         _meta(nullptr) {
@@ -100,6 +100,7 @@ Status DataDir::init() {
         return Status::InternalError("invalid root path: ");
     }
 
+    RETURN_IF_ERROR(update_capacity());
     RETURN_IF_ERROR(_init_cluster_id());
     RETURN_IF_ERROR(_init_extension_and_capacity());
     RETURN_IF_ERROR(_init_file_system());
@@ -1057,4 +1058,35 @@ void DataDir::_remove_check_paths_no_lock(const std::set<std::string>& paths) {
     }
 }
 
+Status DataDir::update_capacity() {
+    try {
+        boost::filesystem::path path_name(_path);
+        boost::filesystem::space_info path_info = boost::filesystem::space(path_name);
+        _available_bytes = path_info.available;
+        if (_disk_capacity_bytes == 0) {
+            // disk capacity only need to be set once
+            _disk_capacity_bytes = path_info.capacity;
+        }
+    } catch (boost::filesystem::filesystem_error& e) {
+        LOG(WARNING) << "get space info failed. path: " << _path << " erro:" << e.what();
+        return Status::InternalError("get path available capacity failed");
+    }
+    LOG(INFO) << "path: " << _path << " total capacity: " << _disk_capacity_bytes
+            << ", available capacity: " << _available_bytes;
+
+    return Status::OK();
+}
+
+bool DataDir::reach_capacity_limit(int64_t incoming_data_size) {
+    double used_pct = (_available_bytes + incoming_data_size) / (double) _disk_capacity_bytes;
+    int64_t left_bytes = _disk_capacity_bytes - _available_bytes - incoming_data_size;
+
+    if (used_pct >= config::storage_flood_stage_usage_percent / 100.0
+        && left_bytes <= config::storage_flood_stage_left_capacity_bytes) {
+        LOG(WARNING) << "reach capacity limit. used pct: " << used_pct << ", left bytes: " << left_bytes
+                << ", path: " << _path;
+        return true;
+    }
+    return false;
+}
 } // namespace doris