chore(license): add license checker for copyright and fix license fil…

…es (cherry-pick #1674) (#1677)

#1676

There are several tasks/steps for this PR:

- develop a script to check the consistency between `.licenserc.yaml` and all files of the project.
- according to the check result, fix `.licenserc.yaml`.
- according to the fixed `.licenserc.yaml`, amend `LICENSE`.

License checker could be run just by `python3 scripts/check_license.py`.

This PR is to cherry-pick #1674 into v2.5 to solve issue #1676.

.licenserc.yaml

@@ -20,80 +20,104 @@ header:
     copyright-owner: Apache Software Foundation
 
   paths-ignore:
+    # Configuration files that could not be added with copyright info (otherwise would lead to error).
+    - '.rat-excludes'
+    - '**/*.csv'
+    - '**/*.json'
     # All the type of licenses of this project should be added to LICENSE.
+    - 'DISCLAIMER-WIP'
     - 'LICENSE'
     - 'NOTICE'
-    - '.github/pull_request_template.md'
+    # Template files of issues and pull requests for Github.
     - '.github/ISSUE_TEMPLATE/bug_report.md'
     - '.github/ISSUE_TEMPLATE/feature-request.md'
     - '.github/ISSUE_TEMPLATE/general_question.md'
-    - '.rat-excludes'
-    - 'DISCLAIMER-WIP'
+    - '.github/pull_request_template.md'
+    # Image files for docs.
+    - '**/*.jpg'
+    - '**/*.png'
+    # Files in pdf format.
+    - '**/*.pdf'
+    # Special files for golang.
+    - '**/go.sum'
+    # TODO(wangdan): Generated files for go client, could generate dynamically?
+    - 'go-client/idl/base/GoUnusedProtection__.go'
+    - 'go-client/idl/base/dsn_err_string.go'
+    - 'go-client/idl/base/rocskdb_err_string.go'
+    # Special files for nodejs.
+    - '**/.npmigonre'
+    # Special files for python.
     - 'python-client/requirement.txt'
-    - '.devcontainer/devcontainer.json'
+    # Text files used for tests and could not be added with copyright info (otherwise would lead to error).
+    - 'src/aio/test/copy_source.txt'
+    - 'src/runtime/test/command.txt'
+    - 'src/failure_detector/test/gtest.filter'
+    - 'src/meta/test/meta_state/gtest.filter'
+    - 'src/meta/test/suite1'
+    - 'src/meta/test/suite2'
+    - 'src/nfs/test/nfs_test_file1'
+    - 'src/nfs/test/nfs_test_file2'
+    - 'src/runtime/test/gtest.filter'
+    # Used for tests and should be empty, or ignore all comment lines (otherwise would lead to error).
+    - 'src/utils/test/config-empty.ini'
+    # Binary files used for tests and could not be added with copyright info (otherwise would lead to error).
+    - 'src/replica/duplication/test/log.1.0.handle_real_private_log'
+    - 'src/replica/duplication/test/log.1.0.handle_real_private_log2'
+    - 'src/replica/duplication/test/log.1.0.all_loaded_are_write_empties'
+    # Used for patches for thirdparties.
+    - 'thirdparty/fix_fds_for_macos.patch'
+    - 'thirdparty/fix_jemalloc_for_m1_on_macos.patch'
+    - 'thirdparty/fix_libevent_for_macos.patch'
+    - 'thirdparty/fix_prometheus-cpp_limits.patch'
+    - 'thirdparty/fix_s2_for_aarch64.patch'
+    - 'thirdparty/fix_thrift_for_cpp11.patch'
+    - 'thirdparty/rocksdb_fix_atomic_flush_0879c240.patch'
     # TODO(yingchun): shell/* files are import from thirdparties, we can move them to thirdparty later.
+    # Copyright (c) 2016, Adi Shavit
     - 'src/shell/argh.h'
+    # Copyright (c) 2010-2016, Salvatore Sanfilippo, etc.
     - 'src/shell/linenoise/linenoise.c'
+    # Copyright (c) 2010-2014, Salvatore Sanfilippo, etc.
     - 'src/shell/linenoise/linenoise.h'
+    # Copyright (c) 2006-2015, Salvatore Sanfilippo, etc.
     - 'src/shell/sds/sds.c'
     - 'src/shell/sds/sds.h'
     - 'src/shell/sds/sdsalloc.h'
-    - '**/go.sum'
-    - '**/*.csv'
-    - '**/*.json'
-    - 'go-client/idl/base/dsn_err_string.go'
-    - 'go-client/idl/base/rocskdb_err_string.go'
-    - 'go-client/idl/base/GoUnusedProtection__.go'
-    - '**/.npmigonre'
     # Copyright (c) Facebook, Inc
     - 'src/utils/TokenBucket.h'
     - 'src/utils/test/TokenBucketTest.cpp'
     - 'src/utils/test/TokenBucketTest.h'
+    # https://github.com/preshing/modern-cpp-threading/blob/master/LICENSE
+    - 'src/utils/hpc_locks/autoreseteventcondvar.h'
     # https://github.com/preshing/cpp11-on-multicore/blob/master/LICENSE
     - 'src/utils/hpc_locks/autoresetevent.h'
-    - 'src/utils/hpc_locks/autoreseteventcondvar.h'
     - 'src/utils/hpc_locks/benaphore.h'
     - 'src/utils/hpc_locks/bitfield.h'
     - 'src/utils/hpc_locks/readme.txt'
     - 'src/utils/hpc_locks/rwlock.h'
     - 'src/utils/hpc_locks/sema.h'
-    # Copyright (c) xxxx The Chromium Authors
+    # Copyright (c) 2011 The Chromium Authors
     - 'src/utils/safe_strerror_posix.h'
+    # Copyright (c) 2012 The Chromium Authors
     - 'src/runtime/build_config.h'
     - 'src/utils/test/autoref_ptr_test.cpp'
+    # Copyright (c) 2006-2009 The Chromium Authors
     - 'src/utils/safe_strerror_posix.cpp'
     # Copyright 2017 The Abseil Authors
     - 'src/utils/absl/base/internal/invoke.h'
     - 'src/utils/absl/utility/utility.h'
+    - 'src/utils/memutil.h'
     - 'src/utils/smart_pointers.h'
+    - 'src/utils/string_view.cpp'
     - 'src/utils/string_view.h'
     - 'src/utils/test/memutil_test.cpp'
-    - 'src/utils/test/string_view_test.cpp'
     - 'src/utils/test/smart_pointers_test.cpp'
-    - 'src/utils/memutil.h'
-    - 'src/utils/string_view.cpp'
+    - 'src/utils/test/string_view_test.cpp'
     # Copyright (c) 2010-2011, Rob Jansen
     - 'cmake_modules/FindRT.cmake'
     - 'cmake_modules/FindDL.cmake'
     # Copyright (c) 2017 Guillaume Papin
     - 'scripts/run-clang-format.py'
-    # need manual fix
-    - 'src/failure_detector/test/gtest.filter'
-    - 'src/meta/test/meta_state/gtest.filter'
-    - 'src/meta/test/suite1'
-    - 'src/meta/test/suite2'
-    - 'src/nfs/test/nfs_test_file1'
-    - 'src/nfs/test/nfs_test_file2'
-    - 'src/runtime/test/gtest.filter'
-    - 'thirdparty/fix_fds_for_macos.patch'
-    - 'thirdparty/fix_jemalloc_for_m1_on_macos.patch'
-    - 'thirdparty/fix_libevent_for_macos.patch'
-    - 'thirdparty/fix_prometheus-cpp_limits.patch'
-    - 'thirdparty/fix_s2_for_aarch64.patch'
-    - 'thirdparty/fix_thrift_for_cpp11.patch'
-    - 'thirdparty/rocksdb_fix_atomic_flush_0879c240.patch'
-    # should be empty, or ignore all comment lines
-    - 'src/utils/test/config-empty.ini'
     # The MIT License (MIT), Copyright (c) 2015 Microsoft Corporation
     - 'cmake_modules/BaseFunctions.cmake'
     - 'docs/rdsn-README.md'
@@ -105,15 +129,13 @@ header:
     - 'idl/replica_admin.thrift'
     - 'scripts/compile_thrift.py'
     - 'scripts/learn_stat.py'
-    - 'src/common/api_common.h'
     - 'src/runtime/api_layer1.h'
     - 'src/runtime/api_task.h'
     - 'src/utils/api_utilities.h'
     - 'src/runtime/app_model.h'
     - 'src/common/json_helper.h'
     - 'src/runtime/rpc/rpc_stream.h'
     - 'src/runtime/rpc/serialization.h'
-    - 'src/common/serialization_helper/dsn.layer2_types.h'
     - 'src/common/serialization_helper/dsn_types.h'
     - 'src/common/serialization_helper/thrift_helper.h'
     - 'src/runtime/serverlet.h'
@@ -131,7 +153,6 @@ header:
     - 'src/client/partition_resolver.h'
     - 'src/replica/replica_base.h'
     - 'src/common/replica_envs.h'
-    - 'src/replica/replica_test_utils.h'
     - 'src/common/replication.codes.h'
     - 'src/replica/replication_app_base.h'
     - 'src/client/replication_ddl_client.h'
@@ -187,7 +208,6 @@ header:
     - 'src/utils/configuration.h'
     - 'src/utils/crc.h'
     - 'src/utils/customizable_id.h'
-    - 'src/utils/dlib.h'
     - 'src/utils/enum_helper.h'
     - 'src/utils/error_code.h'
     - 'src/utils/errors.h'
@@ -220,7 +240,6 @@ header:
     - 'src/aio/test/aio.cpp'
     - 'src/aio/test/clear.sh'
     - 'src/aio/test/config.ini'
-    - 'src/aio/test/copy_source.txt'
     - 'src/aio/test/run.sh'
     - 'src/block_service/test/config-test.ini'
     - 'src/client/CMakeLists.txt'
@@ -363,7 +382,6 @@ header:
     - 'src/replica/replica_learn.cpp'
     - 'src/replica/replica_stub.cpp'
     - 'src/replica/replica_stub.h'
-    - 'src/replica/replica_test_utils.cpp'
     - 'src/replica/replication_app_base.cpp'
     - 'src/replica/replication_service_app.cpp'
     - 'src/replica/split/test/config-test.ini'
@@ -514,7 +532,6 @@ header:
     - 'src/replica/test/run.sh'
     - 'src/runtime/CMakeLists.txt'
     - 'src/runtime/core_main.cpp'
-    - 'src/runtime/dsn.layer2_types.cpp'
     - 'src/runtime/env.sim.cpp'
     - 'src/runtime/env.sim.h'
     - 'src/runtime/fault_injector.cpp'
@@ -569,7 +586,6 @@ header:
     - 'src/runtime/test/address_test.cpp'
     - 'src/runtime/test/async_call.cpp'
     - 'src/runtime/test/clear.sh'
-    - 'src/runtime/test/command.txt'
     - 'src/runtime/test/config-test-corrupt-message.ini'
     - 'src/runtime/test/config-test-sim.ini'
     - 'src/runtime/test/config-test.ini'

LICENSE

@@ -231,7 +231,8 @@ limitations under the License.
 
 --------------------------------------------------------------------------------
 
-src/shell/linenoise/* - BSD-2-Clause License
+src/shell/linenoise/linenoise.h - BSD-2-Clause License
+src/shell/linenoise/LICENSE
 
   Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
   Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
@@ -263,6 +264,38 @@ src/shell/linenoise/* - BSD-2-Clause License
 
 --------------------------------------------------------------------------------
 
+src/shell/linenoise/linenoise.c - BSD-2-Clause License
+
+  Copyright (c) 2010-2016, Salvatore Sanfilippo <antirez at gmail dot com>
+  Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are
+  met:
+
+   *  Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   *  Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
 src/shell/sds/* - BSD-2-Clause License
 
   Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
@@ -327,12 +360,12 @@ src/shell/argh.h - BSD-3-Clause License
 
 --------------------------------------------------------------------------------
 
-src/utils/smart_pointers.h - Apache 2.0 License
-src/utils/string_view.h
-src/utils/absl/base/internal/invoke.h
+src/utils/absl/base/internal/invoke.h - Apache 2.0 License
 src/utils/absl/utility/utility.h
 src/utils/memutil.h
+src/utils/smart_pointers.h
 src/utils/string_view.cpp
+src/utils/string_view.h
 src/utils/test/memutil_test.cpp
 src/utils/test/smart_pointers_test.cpp
 src/utils/test/string_view_test.cpp
@@ -406,11 +439,12 @@ limitations under the License.
 --------------------------------------------------------------------------------
 
 src/utils/hpc_locks/autoreseteventcondvar.h - zlib License
-src/utils/hpc_locks/rwlock.h
 src/utils/hpc_locks/autoresetevent.h
-src/utils/hpc_locks/sema.h
-src/utils/hpc_locks/bitfield.h
 src/utils/hpc_locks/benaphore.h
+src/utils/hpc_locks/bitfield.h
+src/utils/hpc_locks/readme.txt
+src/utils/hpc_locks/rwlock.h
+src/utils/hpc_locks/sema.h
 
 Copyright (c) 2015 Jeff Preshing