Merge pull request #41046 from azat/build/llvm-15

alexey-milovidov · web-flow · commit da01982652d6 · 2022-09-16T07:31:06.000+03:00
Switch to llvm/clang 15
diff --git a/.clang-tidy b/.clang-tidy
@@ -22,6 +22,8 @@ Checks: '*,
     -bugprone-implicit-widening-of-multiplication-result,
     -bugprone-narrowing-conversions,
     -bugprone-not-null-terminated-result,
+    -bugprone-unchecked-optional-access,
+    -bugprone-assignment-in-if-condition,
 
     -cert-dcl16-c,
     -cert-err58-cpp,
@@ -103,6 +105,7 @@ Checks: '*,
 
     -misc-no-recursion,
     -misc-non-private-member-variables-in-classes,
+    -misc-const-correctness,
 
     -modernize-avoid-c-arrays,
     -modernize-concat-nested-namespaces,
@@ -114,6 +117,7 @@ Checks: '*,
     -modernize-use-nodiscard,
     -modernize-use-override,
     -modernize-use-trailing-return-type,
+    -modernize-macro-to-enum,
 
     -performance-inefficient-string-concatenation,
     -performance-no-int-to-ptr,
@@ -135,6 +139,7 @@ Checks: '*,
     -readability-suspicious-call-argument,
     -readability-uppercase-literal-suffix,
     -readability-use-anyofallof,
+    -readability-simplify-boolean-expr,
 
     -zirkon-*,
 '
diff --git a/cmake/clang_tidy.cmake b/cmake/clang_tidy.cmake
@@ -3,7 +3,7 @@ option (ENABLE_CLANG_TIDY "Use clang-tidy static analyzer" OFF)
 
 if (ENABLE_CLANG_TIDY)
 
-    find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12")
+    find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12")
 
     if (CLANG_TIDY_PATH)
         message(STATUS
diff --git a/cmake/target.cmake b/cmake/target.cmake
@@ -45,6 +45,7 @@ if (CMAKE_CROSSCOMPILING)
     endif ()
 
     if (USE_MUSL)
+        # use of undeclared identifier 'PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP'
         set (ENABLE_SENTRY OFF CACHE INTERNAL "")
         set (ENABLE_ODBC OFF CACHE INTERNAL "")
         set (ENABLE_GRPC OFF CACHE INTERNAL "")
diff --git a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -415,7 +415,7 @@
 /*
  * Defined if strerror_r returns char * if _GNU_SOURCE is defined.
  */
-#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE
+/* #undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE */
 
 /* Performs additional safety checks when defined. */
 /* #undef JEMALLOC_OPT_SAFETY_CHECKS */
diff --git a/contrib/krb5-cmake/autoconf_linux.h b/contrib/krb5-cmake/autoconf_linux.h
@@ -440,7 +440,9 @@
 #define HAVE_STRERROR 1
 
 /* Define to 1 if you have the `strerror_r' function. */
+#ifndef USE_MUSL
 #define HAVE_STRERROR_R 1
+#endif
 
 /* Define to 1 if you have the <strings.h> header file. */
 #define HAVE_STRINGS_H 1
diff --git a/contrib/libcpuid b/contrib/libcpuid
@@ -1 +1 @@
-Subproject commit 8db3b8d2d32d22437f063ce692a1b9bb15e42d18
+Subproject commit 503083acb77edf9fbce22a05826307dff2ce96e6
diff --git a/contrib/libpq-cmake/CMakeLists.txt b/contrib/libpq-cmake/CMakeLists.txt
@@ -63,6 +63,13 @@ target_include_directories (_libpq SYSTEM PUBLIC ${LIBPQ_SOURCE_DIR})
 target_include_directories (_libpq SYSTEM PUBLIC "${LIBPQ_SOURCE_DIR}/include")
 target_include_directories (_libpq SYSTEM PRIVATE "${LIBPQ_SOURCE_DIR}/configs")
 
+# NOTE: this is a dirty hack to avoid and instead pg_config.h should be shipped
+# for different OS'es like for jemalloc, not one generic for all OS'es like
+# now.
+if (OS_DARWIN OR OS_FREEBSD OR USE_MUSL)
+    target_compile_definitions(_libpq PRIVATE -DSTRERROR_R_INT=1)
+endif()
+
 target_link_libraries (_libpq PRIVATE OpenSSL::SSL)
 
 add_library(ch_contrib::libpq ALIAS _libpq)
diff --git a/contrib/librdkafka b/contrib/librdkafka
@@ -1 +1 @@
-Subproject commit ff32b4e9eeafd0b276f010ee969179e4e9e6d0b2
+Subproject commit 6f3b483426a8c8ec950e27e446bec175cf8b553f
diff --git a/contrib/llvm b/contrib/llvm
@@ -1 +1 @@
-Subproject commit 20607e61728e97c969e536644c3c0c1bb1a50672
+Subproject commit 0db5bf5bd2452cd8f1283a1fcdc04845af705bfc
diff --git a/contrib/sentry-native b/contrib/sentry-native
@@ -1 +1 @@
-Subproject commit f431047ac8da13179c488018dddf1c0d0771a997
+Subproject commit ae10fb8c224c3f41571446e1ed7fd57b9e5e366b
diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
@@ -67,24 +67,5 @@ ENV GOCACHE=/workdir/
 RUN mkdir /workdir && chmod 777 /workdir
 WORKDIR /workdir
 
-# NOTE: thread sanitizer is broken in clang-14, we have to build it with clang-15
-# https://github.com/ClickHouse/ClickHouse/pull/39450
-# https://github.com/google/sanitizers/issues/1540
-# https://github.com/google/sanitizers/issues/1552
-
-RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-15 main" >> \
-        /etc/apt/sources.list.d/clang.list \
-    && apt-get update \
-    && apt-get install \
-        clang-15 \
-        llvm-15 \
-        clang-tidy-15 \
-        --yes --no-install-recommends \
-    && apt-get clean
-
-# for external_symbolizer_path
-RUN ln -s /usr/bin/llvm-symbolizer-15 /usr/bin/llvm-symbolizer
-
 COPY build.sh /
 CMD ["bash", "-c", "/build.sh 2>&1"]
diff --git a/docker/packager/packager b/docker/packager/packager
@@ -339,17 +339,16 @@ if __name__ == "__main__":
     parser.add_argument(
         "--compiler",
         choices=(
-            "clang-15",  # For TSAN builds, see #39450
-            "clang-14",
-            "clang-14-darwin",
-            "clang-14-darwin-aarch64",
-            "clang-14-aarch64",
-            "clang-14-ppc64le",
-            "clang-14-amd64sse2",
-            "clang-14-freebsd",
+            "clang-15",
+            "clang-15-darwin",
+            "clang-15-darwin-aarch64",
+            "clang-15-aarch64",
+            "clang-15-ppc64le",
+            "clang-15-amd64sse2",
+            "clang-15-freebsd",
             "gcc-11",
         ),
-        default="clang-14",
+        default="clang-15",
         help="a compiler to use",
     )
     parser.add_argument(
diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile
@@ -16,11 +16,10 @@ RUN apt-get update \
 # and MEMORY_LIMIT_EXCEEDED exceptions in Functional tests (total memory limit in Functional tests is ~55.24 GiB).
 # TSAN will flush shadow memory when reaching this limit.
 # It may cause false-negatives, but it's better than OOM.
-RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7 memory_limit_mb=46080'" >> /etc/environment; \
-  echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \
-  echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment; \
-  echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment; \
-  ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer;
+RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7 memory_limit_mb=46080'" >> /etc/environment
+RUN echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment
+RUN echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment
+RUN echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment
 # Sanitizer options for current shell (not current, but the one that will be spawned on "docker run")
 # (but w/o verbosity for TSAN, otherwise test.reference will not match)
 ENV TSAN_OPTIONS='halt_on_error=1 history_size=7 memory_limit_mb=46080'
diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile
@@ -8,16 +8,41 @@ FROM clickhouse/binary-builder:$FROM_TAG
 ARG apt_archive="http://archive.ubuntu.com"
 RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
 
-RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-14 libllvm14 libclang-14-dev libmlir-14-dev
+RUN apt-get update && apt-get --yes --allow-unauthenticated install libclang-${LLVM_VERSION}-dev libmlir-${LLVM_VERSION}-dev
+
+# libclang-15-dev does not contain proper symlink:
+#
+# This is what cmake will search for:
+#
+#     # readlink -f /usr/lib/llvm-15/lib/libclang-15.so.1
+#     /usr/lib/x86_64-linux-gnu/libclang-15.so.1
+#
+# This is what exists:
+#
+#     # ls -l /usr/lib/x86_64-linux-gnu/libclang-15*
+#     lrwxrwxrwx 1 root root       16 Sep  5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so -> libclang-15.so.1
+#     lrwxrwxrwx 1 root root       21 Sep  5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so.15 -> libclang-15.so.15.0.0
+#     -rw-r--r-- 1 root root 31835760 Sep  5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so.15.0.0
+#
+ARG TARGETARCH
+RUN arch=${TARGETARCH:-amd64} \
+    && case $arch in \
+        amd64) rarch=x86_64 ;; \
+        arm64) rarch=aarch64 ;; \
+        *) exit 1 ;; \
+    esac \
+    && ln -rsf /usr/lib/$rarch-linux-gnu/libclang-15.so.15 /usr/lib/$rarch-linux-gnu/libclang-15.so.1
 
 # repo versions doesn't work correctly with C++17
 # also we push reports to s3, so we add index.html to subfolder urls
 # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
 # TODO: remove branch in a few weeks after merge, e.g. in May or June 2022
-RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser --branch llvm-14 \
+#
+# FIXME: update location of a repo
+RUN git clone https://github.com/azat/woboq_codebrowser --branch llvm-15 \
   && cd woboq_codebrowser \
-  && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-14 -DCMAKE_C_COMPILER=clang-14 \
-  && make -j \
+  && cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} \
+  && ninja \
   && cd .. \
   && rm -rf woboq_codebrowser
 
@@ -32,7 +57,7 @@ ENV SHA=nosha
 ENV DATA="https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data"
 
 CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \
-    cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-14 -DCMAKE_C_COMPILER=/usr/bin/clang-14 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
+    cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=/usr/bin/clang-${LLVM_VERSION} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
     mkdir -p $HTML_RESULT_DIRECTORY && \
     $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \
     cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
@@ -19,7 +19,7 @@ stage=${stage:-}
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 echo "$script_dir"
 repo_dir=ch
-BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-14_debug_none_unsplitted_disable_False_binary"}
+BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-15_debug_none_unsplitted_disable_False_binary"}
 BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
 
 function clone
diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh
@@ -2,7 +2,7 @@
 set -euo pipefail
 
 
-CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-14_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
+CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-15_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
 CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}
 
 
diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile
@@ -5,7 +5,7 @@ FROM ubuntu:20.04
 ARG apt_archive="http://archive.ubuntu.com"
 RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
 
-ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
+ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=15
 
 RUN apt-get update \
     && apt-get install \
@@ -56,6 +56,8 @@ RUN apt-get update \
 
 # This symlink required by gcc to find lld compiler
 RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
+# for external_symbolizer_path
+RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer
 
 ARG CCACHE_VERSION=4.6.1
 RUN mkdir /tmp/ccache \
diff --git a/src/Common/ArrayCache.h b/src/Common/ArrayCache.h
@@ -722,5 +722,3 @@ class ArrayCache : private boost::noncopyable
         return res;
     }
 };
-
-template <typename Key, typename Payload> constexpr size_t ArrayCache<Key, Payload>::min_chunk_size;
diff --git a/src/Common/MemoryStatisticsOS.cpp b/src/Common/MemoryStatisticsOS.cpp
@@ -135,7 +135,7 @@ MemoryStatisticsOS::Data MemoryStatisticsOS::get() const
     struct kinfo_proc kp;
     size_t len = sizeof(struct kinfo_proc);
 
-    if (-1 == ::sysctl(mib, 4, &kp, &len, NULL, 0))
+    if (-1 == ::sysctl(mib, 4, &kp, &len, nullptr, 0))
         throwFromErrno("Cannot sysctl(kern.proc.pid." + std::to_string(self) + ")", ErrorCodes::SYSTEM_ERROR);
 
     if (sizeof(struct kinfo_proc) != len)
diff --git a/src/Core/examples/coro.cpp b/src/Core/examples/coro.cpp
@@ -14,7 +14,7 @@
 
 namespace std // NOLINT(cert-dcl58-cpp)
 {
-    using namespace experimental::coroutines_v1;
+    using namespace experimental::coroutines_v1; // NOLINT(cert-dcl58-cpp)
 }
 
 #if __has_warning("-Wdeprecated-experimental-coroutine")
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
@@ -530,6 +530,7 @@ String FormatFactory::getFormatFromFileDescriptor(int fd)
         return getFormatFromFileName(file_path, false);
     return "";
 #else
+    (void)fd;
     return "";
 #endif
 }
diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp
@@ -233,7 +233,7 @@ void ReadBufferFromFileDescriptor::rewind()
 
 
 /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout.
-bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds)
+bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds) const
 {
     fd_set fds;
     FD_ZERO(&fds);
diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h
@@ -66,7 +66,7 @@ class ReadBufferFromFileDescriptor : public ReadBufferFromFileBase
 
 private:
     /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout.
-    bool poll(size_t timeout_microseconds);
+    bool poll(size_t timeout_microseconds) const;
 };
 
 
diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp
@@ -739,7 +739,10 @@ CompiledAggregateFunctions compileAggregateFunctions(CHJIT & jit, const std::vec
     {
         compileCreateAggregateStatesFunctions(module, functions, create_aggregate_states_functions_name);
         compileAddIntoAggregateStatesFunctions(module, functions, add_aggregate_states_functions_name);
-        compileAddIntoAggregateStatesFunctionsSinglePlace(module, functions, add_aggregate_states_functions_name_single_place);
+        /// FIXME: this leads to use-of-uninitialized-value in llvm
+        /// But for now, it is safe, since it is not used by Aggregator anyway
+        (void)compileAddIntoAggregateStatesFunctionsSinglePlace;
+        /// compileAddIntoAggregateStatesFunctionsSinglePlace(module, functions, add_aggregate_states_functions_name_single_place);
         compileMergeAggregatesStates(module, functions, merge_aggregate_states_functions_name);
         compileInsertAggregatesIntoResultColumns(module, functions, insert_aggregate_states_functions_name);
     });
@@ -752,7 +755,7 @@ CompiledAggregateFunctions compileAggregateFunctions(CHJIT & jit, const std::vec
 
     assert(create_aggregate_states_function);
     assert(add_into_aggregate_states_function);
-    assert(add_into_aggregate_states_function_single_place);
+    /// assert(add_into_aggregate_states_function_single_place); /// FIXME
     assert(merge_aggregate_states_function);
     assert(insert_aggregate_states_function);
 
diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
@@ -2,6 +2,15 @@
 
 #if USE_MSGPACK
 
+/// FIXME: there is some issue with clang-15, that incorrectly detect a
+/// "Attempt to free released memory" in msgpack::unpack(), because of delete
+/// operator for zone (from msgpack/v1/detail/cpp11_zone.hpp), hence NOLINT
+///
+/// NOTE: that I was not able to suppress it locally, only with
+/// NOLINTBEGIN/NOLINTEND
+//
+// NOLINTBEGIN(clang-analyzer-cplusplus.NewDelete)
+
 #include <cstdlib>
 #include <Common/assert_cast.h>
 #include <IO/ReadHelpers.h>
@@ -553,6 +562,8 @@ void registerMsgPackSchemaReader(FormatFactory & factory)
 
 }
 
+// NOLINTEND(clang-analyzer-cplusplus.NewDelete)
+
 #else
 
 namespace DB
diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp
@@ -255,7 +255,7 @@ class HDFSSource::DisclosedGlobIterator::Impl
 class HDFSSource::URISIterator::Impl
 {
 public:
-    explicit Impl(const std::vector<const String> & uris_, ContextPtr context)
+    explicit Impl(const std::vector<String> & uris_, ContextPtr context)
     {
         auto path_and_uri = getPathFromUriAndUriWithoutPath(uris_[0]);
         HDFSBuilderWrapper builder = createHDFSBuilder(path_and_uri.second + "/", context->getGlobalContext()->getConfigRef());
@@ -293,7 +293,7 @@ String HDFSSource::DisclosedGlobIterator::next()
     return pimpl->next();
 }
 
-HDFSSource::URISIterator::URISIterator(const std::vector<const String> & uris_, ContextPtr context)
+HDFSSource::URISIterator::URISIterator(const std::vector<String> & uris_, ContextPtr context)
     : pimpl(std::make_shared<HDFSSource::URISIterator::Impl>(uris_, context))
 {
 }
diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h
@@ -86,7 +86,7 @@ class StorageHDFS final : public IStorage, WithContext
         const String & format_name,
         const ContextPtr & ctx);
 
-    std::vector<const String> uris;
+    std::vector<String> uris;
     String format_name;
     String compression_method;
     const bool distributed_processing;
@@ -116,7 +116,7 @@ class HDFSSource : public ISource, WithContext
     class URISIterator
     {
         public:
-            URISIterator(const std::vector<const String> & uris_, ContextPtr context);
+            URISIterator(const std::vector<String> & uris_, ContextPtr context);
             String next();
         private:
             class Impl;
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py
