Remove source_hash.txtes from git (#3460)

jleibs · web-flow · commit 874fb6544815 · 2023-09-25T20:29:20.000+02:00
### What * Resolves: #3334 Main changes: * Removes the `source_hash.txt` from git and adds it to gitignore. * Doesn't run the codegen during `build.rs` unless `source_hash.txt` is present * Makes `cargo codegen` produce the `source_hash.txt` * Removes the printlns from the source_hash generation when run as part of cargo codegen. * Adds the formatting dependencies to the docker image * Adds a new CI check that codegen produces no diffs. Confirmed that we catch it when codegen would produce changes: https://github.com/rerun-io/rerun/actions/runs/6302436017/job/17109617144?pr=3461 ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/3460) (if applicable) - [PR Build Summary](https://build.rerun.io/pr/3460) - [Docs preview](https://rerun.io/preview/d4eaa3fb30cd2fad8b518bf8829286a3762a64fb/docs)  - [Examples preview](https://rerun.io/preview/d4eaa3fb30cd2fad8b518bf8829286a3762a64fb/examples)  - [Recent benchmark results](https://ref.rerun.io/dev/bench/) - [Wasm size tracking](https://ref.rerun.io/dev/sizes/)
diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml
@@ -56,7 +56,7 @@ jobs:
     runs-on: ubuntu-latest-16-cores
 
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
 
     steps:
       - uses: actions/checkout@v4
diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml
@@ -74,7 +74,7 @@ jobs:
               runner="ubuntu-latest-16-cores"
               target="x86_64-unknown-linux-gnu"
               run_tests="true"
-              container="{'image': 'rerunio/ci_docker:0.9.1'}"
+              container="{'image': 'rerunio/ci_docker:0.10.0'}"
               ;;
             windows)
               runner="windows-latest-8-cores"
diff --git a/.github/workflows/reusable_build_and_upload_rerun_c.yml b/.github/workflows/reusable_build_and_upload_rerun_c.yml
@@ -94,7 +94,7 @@ jobs:
               runner="ubuntu-latest-16-cores"
               target="x86_64-unknown-linux-gnu"
               run_tests="true"
-              container="{'image': 'rerunio/ci_docker:0.9.1'}"
+              container="{'image': 'rerunio/ci_docker:0.10.0'}"
               lib_name="librerun_c.a"
               ;;
             windows)
diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml
@@ -38,7 +38,7 @@ jobs:
     runs-on: ubuntu-latest-16-cores
 
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
 
     steps:
       - uses: actions/checkout@v4
diff --git a/.github/workflows/reusable_build_web_demo.yml b/.github/workflows/reusable_build_web_demo.yml
@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest-16-cores
 
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
 
     steps:
       - uses: actions/checkout@v4
diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml
@@ -109,11 +109,41 @@ jobs:
 
   # ---------------------------------------------------------------------------
 
+  no-codegen-changes:
+    name: Check if running codegen would produce any changes
+    runs-on: ubuntu-latest-16-cores
+    container:
+      image: rerunio/ci_docker:0.10.0
+    env:
+      RUSTC_WRAPPER: "sccache"
+    steps:
+      # Note: We explicitly don't override `ref` here. We need to see if changes would be made
+      # in a context where we have merged with main. Otherwise we might miss changes such as one
+      # PR introduces a new type and another PR changes the codegen.
+      - uses: actions/checkout@v4
+
+      - name: Set up Rust
+        uses: ./.github/actions/setup-rust
+        with:
+          cache_key: "build-linux"
+          save_cache: true
+          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
+
+      - name: Codegen
+        uses: actions-rs/cargo@v1
+        with:
+          command: codegen
+
+      - name: No Diffs From Running Codegen
+        run: |
+          git diff --exit-code
+
   rs-lints:
     name: Rust lints (fmt, check, cranky, tests, doc)
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
     env:
       RUSTC_WRAPPER: "sccache"
     steps:
@@ -213,7 +243,7 @@ jobs:
     name: Check Rust web build (wasm32 + wasm-bindgen)
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
     env:
       RUSTC_WRAPPER: "sccache"
     steps:
@@ -325,7 +355,7 @@ jobs:
     name: Cargo Deny
     runs-on: ubuntu-latest
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
     steps:
       - uses: actions/checkout@v4
         with:
@@ -357,7 +387,7 @@ jobs:
     name: C++ tests
     runs-on: ubuntu-latest
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
     env:
       RUSTC_WRAPPER: "sccache"
     steps:
diff --git a/.github/workflows/reusable_deploy_docs.yml b/.github/workflows/reusable_deploy_docs.yml
@@ -106,7 +106,7 @@ jobs:
     name: Rust
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
     steps:
       - name: Show context
         run: |
diff --git a/.github/workflows/reusable_run_notebook.yml b/.github/workflows/reusable_run_notebook.yml
@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
 
     steps:
       - uses: actions/checkout@v4
diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml
@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: rerunio/ci_docker:0.9.1
+      image: rerunio/ci_docker:0.10.0
 
     permissions:
       contents: "read"
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,9 @@
 .DS_Store
 
+# Codegen stuff:
+crates/re_types/source_hash.txt
+crates/re_types_builder/source_hash.txt
+
 # C++ and CMake stuff:
 *.bin
 *.o
diff --git a/ci_docker/Dockerfile b/ci_docker/Dockerfile
@@ -2,7 +2,7 @@ FROM ubuntu:20.04
 LABEL maintainer="opensource@rerun.io"
 # Remember to update the version in publish.sh
 # TODO(jleibs) use this version in the publish.sh script and below in the CACHE_KEY
-LABEL version="0.9.1"
+LABEL version="0.10.0"
 LABEL description="Docker image used for the CI of https://github.com/rerun-io/rerun"
 
 # Install the ubuntu package dependencies
@@ -17,7 +17,6 @@ RUN set -eux; \
     apt-get install -y --no-install-recommends \
     libarrow-dev \
     build-essential \
-    clang-format \
     cmake \
     curl \
     git \
@@ -33,12 +32,32 @@ RUN set -eux; \
     libxkbcommon-dev \
     lsb-release \
     python3-pip \
+    software-properties-common \
     sudo; \
     rm -rf /var/lib/apt/lists/*;
 
 # Need a more recent pip for manylinux packages to work properly
 RUN python3 -m pip install -U pip
 
+# Need a more recent clang-format for codegen
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \
+    apt update -y && \
+    apt upgrade -y && \
+    wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh 15 && \
+    apt-get install clang-format-15 && \
+    ln -sf clang-format-15 /usr/bin/clang-format && \
+    rm -rf /var/lib/apt/lists/*;
+
+# Need a more recent flatbuffers for codegen
+RUN git clone --depth 1 --branch v23.5.9 https://github.com/google/flatbuffers.git && \
+    cd flatbuffers && \
+    cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release && \
+    make -j4 && \
+    make install && \
+    cd .. && rm -rf flatbuffers
+
 # We need a more modern patchelf than ships on ubuntu-20.04
 RUN curl -L https://github.com/NixOS/patchelf/releases/download/0.17.2/patchelf-0.17.2-x86_64.tar.gz | tar -xz ./bin/patchelf
 
@@ -75,6 +94,8 @@ RUN cargo install cargo-benchcmp
 # Install the python build dependencies
 ADD rerun_py/requirements-build.txt requirements-build.txt
 RUN pip install -r requirements-build.txt
+ADD rerun_py/requirements-lint.txt requirements-lint.txt
+RUN pip install -r requirements-lint.txt
 
 # Install tools from setup_web.sh
 ADD scripts/setup_web.sh setup_web.sh
@@ -84,7 +105,7 @@ RUN ./setup_web.sh || true
 RUN curl -L https://github.com/WebAssembly/binaryen/releases/download/version_112/binaryen-version_112-x86_64-linux.tar.gz | tar xzk --strip-components 1
 
 # Increment this to invalidate cache
-ENV CACHE_KEY=rerun_docker_v0.9.1
+ENV CACHE_KEY=rerun_docker_v0.10.0
 
 # See: https://github.com/actions/runner-images/issues/6775#issuecomment-1410270956
 RUN git config --system --add safe.directory '*'
diff --git a/ci_docker/publish.sh b/ci_docker/publish.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -eux
 
-VERSION=0.9.1 # Bump on each new version. Remember to update the version in the Dockerfile too.
+VERSION=0.10.0 # Bump on each new version. Remember to update the version in the Dockerfile too.
 
 # The build needs to run from top of repo to access the requirements.txt
 cd `git rev-parse --show-toplevel`
diff --git a/crates/re_build_tools/src/lib.rs b/crates/re_build_tools/src/lib.rs
@@ -6,6 +6,7 @@
 
 use anyhow::Context as _;
 
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::{path::PathBuf, process::Command};
 
 mod hashing;
@@ -22,6 +23,19 @@ pub use self::rebuild_detector::{
     rerun_if_changed_glob, rerun_if_changed_or_doesnt_exist, write_file_if_necessary,
 };
 
+/// Atomic bool indicating whether or not to print the cargo build instructions
+pub(crate) static OUTPUT_CARGO_BUILD_INSTRUCTIONS: AtomicBool = AtomicBool::new(true);
+
+/// Change whether or not this library should output cargo build instructions
+pub fn set_output_cargo_build_instructions(output_instructions: bool) {
+    OUTPUT_CARGO_BUILD_INSTRUCTIONS.store(output_instructions, Ordering::Relaxed);
+}
+
+/// Helper to check whether or not cargo build instructions should be printed.
+pub(crate) fn should_output_cargo_build_instructions() -> bool {
+    OUTPUT_CARGO_BUILD_INSTRUCTIONS.load(Ordering::Relaxed)
+}
+
 // Situations to consider
 // ----------------------
 //
@@ -98,7 +112,9 @@ pub fn export_env_vars() {
 }
 
 fn set_env(name: &str, value: &str) {
-    println!("cargo:rustc-env={name}={value}");
+    if should_output_cargo_build_instructions() {
+        println!("cargo:rustc-env={name}={value}");
+    }
 }
 
 fn run_command(cmd: &str, args: &[&str]) -> anyhow::Result<String> {
diff --git a/crates/re_build_tools/src/rebuild_detector.rs b/crates/re_build_tools/src/rebuild_detector.rs
@@ -7,6 +7,8 @@ use std::{
 
 use cargo_metadata::{CargoOpt, Metadata, MetadataCommand, Package, PackageId};
 
+use crate::should_output_cargo_build_instructions;
+
 /// Call from `build.rs` to trigger a rebuild whenever any source file of the given package
 /// _or any of its dependencies_ changes, recursively.
 ///
@@ -40,7 +42,9 @@ pub fn rebuild_if_crate_changed(pkg_name: &str) {
 
 /// Call from `build.rs` to trigger a rebuild whenever an environment variable changes.
 pub fn get_and_track_env_var(env_var_name: &str) -> Result<String, std::env::VarError> {
-    println!("cargo:rerun-if-env-changed={env_var_name}");
+    if should_output_cargo_build_instructions() {
+        println!("cargo:rerun-if-env-changed={env_var_name}");
+    }
     std::env::var(env_var_name)
 }
 
@@ -58,14 +62,18 @@ pub fn rerun_if_changed(path: impl AsRef<Path>) {
     let path = path.as_ref();
     // Make sure the file exists, otherwise we'll be rebuilding all the time.
     assert!(path.exists(), "Failed to find {path:?}");
-    println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
+    if should_output_cargo_build_instructions() {
+        println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
+    }
 }
 
 /// Call from `build.rs` to trigger a rebuild whenever the file at `path` changes, or it doesn't
 /// exist.
 pub fn rerun_if_changed_or_doesnt_exist(path: impl AsRef<Path>) {
     let path = path.as_ref();
-    println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
+    if should_output_cargo_build_instructions() {
+        println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
+    }
 }
 
 /// Call from `build.rs` to trigger a rebuild whenever any of the files identified by the given
diff --git a/crates/re_types/build.rs b/crates/re_types/build.rs
@@ -1,10 +1,11 @@
 //! Generates Rust & Python code from flatbuffers definitions.
 
+use std::path::Path;
+
 use re_build_tools::{
-    compute_crate_hash, compute_dir_filtered_hash, compute_dir_hash, compute_strings_hash,
-    is_tracked_env_var_set, iter_dir, read_versioning_hash, rerun_if_changed,
-    rerun_if_changed_or_doesnt_exist, write_versioning_hash,
+    is_tracked_env_var_set, iter_dir, read_versioning_hash, rerun_if_changed, write_versioning_hash,
 };
+use re_types_builder::{compute_re_types_hash, SourceLocations};
 
 // ---
 
@@ -33,41 +34,28 @@ fn main() {
         return;
     }
 
-    rerun_if_changed_or_doesnt_exist(SOURCE_HASH_PATH);
+    // Only re-build if source-hash exists
+    if !Path::new(SOURCE_HASH_PATH).exists() {
+        return;
+    }
+
+    rerun_if_changed(SOURCE_HASH_PATH);
     for path in iter_dir(DEFINITIONS_DIR_PATH, Some(&["fbs"])) {
         rerun_if_changed(&path);
     }
 
     // NOTE: We need to hash both the flatbuffers definitions as well as the source code of the
     // code generator itself!
     let cur_hash = read_versioning_hash(SOURCE_HASH_PATH);
-    let re_types_builder_hash = compute_crate_hash("re_types_builder");
-    let definitions_hash = compute_dir_hash(DEFINITIONS_DIR_PATH, Some(&["fbs"]));
-    let doc_examples_hash = compute_dir_hash(DOC_EXAMPLES_DIR_PATH, Some(&["rs", "py", "cpp"]));
-    let python_extensions_hash = compute_dir_filtered_hash(PYTHON_OUTPUT_DIR_PATH, |path| {
-        path.to_str().unwrap().ends_with("_ext.py")
-    });
-    let cpp_extensions_hash = compute_dir_filtered_hash(CPP_OUTPUT_DIR_PATH, |path| {
-        path.to_str().unwrap().ends_with("_ext.cpp")
-    });
-
-    let new_hash = compute_strings_hash(&[
-        &re_types_builder_hash,
-        &definitions_hash,
-        &doc_examples_hash,
-        &python_extensions_hash,
-        &cpp_extensions_hash,
-    ]);
-
-    // Leave these be please, very useful when debugging.
-    eprintln!("re_types_builder_hash: {re_types_builder_hash:?}");
-    eprintln!("definitions_hash: {definitions_hash:?}");
-    eprintln!("doc_examples_hash: {doc_examples_hash:?}");
-    eprintln!("python_extensions_hash: {python_extensions_hash:?}");
-    eprintln!("cpp_extensions_hash: {cpp_extensions_hash:?}");
-    eprintln!("new_hash: {new_hash:?}");
     eprintln!("cur_hash: {cur_hash:?}");
 
+    let new_hash = compute_re_types_hash(&SourceLocations {
+        definitions_dir: DEFINITIONS_DIR_PATH,
+        doc_examples_dir: DOC_EXAMPLES_DIR_PATH,
+        python_output_dir: PYTHON_OUTPUT_DIR_PATH,
+        cpp_output_dir: CPP_OUTPUT_DIR_PATH,
+    });
+
     if let Some(cur_hash) = cur_hash {
         if cur_hash == new_hash {
             // Neither the source of the code generator nor the IDL definitions have changed, no need
diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt
diff --git a/crates/re_types_builder/Cargo.toml b/crates/re_types_builder/Cargo.toml
@@ -18,6 +18,7 @@ all-features = true
 
 
 [dependencies]
+re_build_tools.workspace = true
 re_error.workspace = true
 re_log.workspace = true
 re_tracing = { workspace = true, features = ["server"] }
diff --git a/crates/re_types_builder/build.rs b/crates/re_types_builder/build.rs
diff --git a/crates/re_types_builder/source_hash.txt b/crates/re_types_builder/source_hash.txt
diff --git a/crates/re_types_builder/src/bin/build_re_types.rs b/crates/re_types_builder/src/bin/build_re_types.rs
diff --git a/crates/re_types_builder/src/lib.rs b/crates/re_types_builder/src/lib.rs
