add likely to hot path in non-simd sto-alg so "Branch Mispredict" goes down by 2% (#1034)

MacroModel · web-flow · commit 2f72d9968320 · 2024-12-17T06:59:19.000-05:00
* add likely to hot path in non-simd sto-alg so "Branch Mispredict" goes down by 2%

* add likely to happy path
diff --git a/include/fast_io_core_impl/integers/sto/sto_contiguous.h b/include/fast_io_core_impl/integers/sto/sto_contiguous.h
@@ -571,7 +571,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 						constexpr ::std::uint_least64_t mask{0x000000FF000000FF};
 						constexpr ::std::uint_least64_t fullmask{baseval * 0x80};
 
-						while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least64_t))
+						while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least64_t)) [[likely]]
 						{
 							::std::uint_least64_t val;
 							::fast_io::freestanding::my_memcpy(__builtin_addressof(val), first, sizeof(::std::uint_least64_t));
@@ -581,7 +581,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 								val = ::fast_io::little_endian(val);
 							}
 
-							if (::std::uint_least64_t const cval{((val + first_bound) | (val - zero_lower_bound)) & fullmask}; cval)
+							if (::std::uint_least64_t const cval{((val + first_bound) | (val - zero_lower_bound)) & fullmask}; cval) [[likely]]
 							{
 								auto ctrz_cval{::std::countr_zero(cval)};
 								auto const valid_bits{ctrz_cval & -8};
@@ -636,7 +636,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 								val = ::fast_io::little_endian(val);
 							}
 
-							if (::std::uint_least32_t const cval{((val + first_bound) | (val - 0x30303030)) & 0x80808080}; cval)
+							if (::std::uint_least32_t const cval{((val + first_bound) | (val - 0x30303030)) & 0x80808080}; cval) [[likely]]
 							{
 								auto ctrz_cval{::std::countr_zero(cval)};
 								auto const valid_bits{ctrz_cval & -8};
@@ -686,7 +686,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 					constexpr ::std::uint_least64_t first_bound{0x7fc67fc67fc67fc6 + 0x0001000100010001 * (10 - base)};
 					if constexpr (max_size >= u64_size_of_c16)
 					{
-						while (static_cast<::std::size_t>(first_phase_last - first) >= u64_size_of_c16)
+						while (static_cast<::std::size_t>(first_phase_last - first) >= u64_size_of_c16) [[likely]]
 						{
 							::std::uint_least64_t val;
 							::fast_io::freestanding::my_memcpy(__builtin_addressof(val), first, sizeof(::std::uint_least64_t));
@@ -696,7 +696,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 								val = ::fast_io::little_endian(val);
 							}
 
-							if (::std::uint_least64_t const cval{((val + first_bound) | (val - 0x0030003000300030)) & 0x8000800080008000}; cval)
+							if (::std::uint_least64_t const cval{((val + first_bound) | (val - 0x0030003000300030)) & 0x8000800080008000}; cval) [[likely]]
 							{
 								auto ctrz_cval{::std::countr_zero(cval)};
 								auto const valid_bits{ctrz_cval & -16};
@@ -751,7 +751,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 						constexpr ::std::uint_least64_t mask{0x000000FF000000FF};
 						constexpr ::std::uint_least64_t mul1{pow_base_sizeof_base_2 + (pow_base_sizeof_base_6 << 32)};
 						constexpr ::std::uint_least64_t mul2{1 + (pow_base_sizeof_base_4 << 32)};
-						while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least64_t))
+						while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least64_t)) [[likely]]
 						{
 							::std::uint_least64_t val;
 							::fast_io::freestanding::my_memcpy(__builtin_addressof(val), first, sizeof(::std::uint_least64_t));
@@ -767,7 +767,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 																  ~(((val + 0x3f3f3f3f3f3f3f3f) | (val - 0x4040404040404040)) &
 																	((val + 0x1f1f1f1f1f1f1f1f) | (val - 0x6060606060606060)))) &
 																 0x8080808080808080};
-								cval)
+								cval) [[likely]]
 							{
 								auto ctrz_cval{::std::countr_zero(cval)};
 								auto const valid_bits{ctrz_cval & -8};
@@ -824,14 +824,14 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 
 						constexpr ::std::uint_least32_t pow_base_sizeof_base_2{::fast_io::details::compile_pow_n<::std::uint_least32_t, base_char_type, 2>};
 						constexpr ::std::uint_least32_t mask{0x000000FF};
-						while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least32_t))
+						while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least32_t)) [[likely]]
 						{
 							::std::uint_least32_t val;
 							::fast_io::freestanding::my_memcpy(__builtin_addressof(val), first, sizeof(::std::uint_least32_t));
 
 							val = ::fast_io::little_endian(val);
 
-							if (::std::uint_least32_t const cval{((val + first_bound) | (val - 0x30303030)) & 0x80808080}; cval) [[unlikely]]
+							if (::std::uint_least32_t const cval{((val + first_bound) | (val - 0x30303030)) & 0x80808080}; cval) [[likely]]
 							{
 								auto ctrz_cval{::std::countr_zero(cval)};
 								auto const valid_bits{ctrz_cval & -8};
@@ -874,7 +874,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi
 		}
 	}
 
-	for (; first != first_phase_last; ++first)
+	for (; first != first_phase_last; ++first) [[likely]]
 	{
 		unsigned_char_type ch{static_cast<unsigned_char_type>(*first)};
 		if (char_digit_to_literal<base, char_type>(ch)) [[unlikely]]
@@ -949,57 +949,57 @@ template <char8_t base, ::std::integral char_type>
 inline constexpr parse_result<char_type const *> scan_shbase_impl(char_type const *first,
 																  char_type const *last) noexcept
 {
-	if (first == last || *first != char_literal_v<u8'0', char_type>)
+	if (first == last || *first != char_literal_v<u8'0', char_type>) [[unlikely]]
 	{
 		return {first, parse_code::invalid};
 	}
-	if ((++first) == last)
+	if ((++first) == last) [[unlikely]]
 	{
 		return {first, parse_code::invalid};
 	}
 	if constexpr (base == 2 || base == 3 || base == 16)
 	{
 		auto ch{*first};
 		if ((ch != char_literal_v<(base == 2 ? u8'B' : (base == 3 ? u8't' : u8'X')), char_type>)&(
-				ch != char_literal_v<(base == 2 ? u8'b' : (base == 3 ? u8't' : u8'x')), char_type>))
+				ch != char_literal_v<(base == 2 ? u8'b' : (base == 3 ? u8't' : u8'x')), char_type>)) [[unlikely]]
 		{
 			return {first, parse_code::invalid};
 		}
 		++first;
 	}
 	else
 	{
-		if (*first != char_literal_v<u8'[', char_type>)
+		if (*first != char_literal_v<u8'[', char_type>) [[unlikely]]
 		{
 			return {first, parse_code::invalid};
 		}
 		++first;
-		if ((++first) == last)
+		if ((++first) == last) [[unlikely]]
 		{
 			return {first, parse_code::invalid};
 		}
 		constexpr auto digit0{char_literal_v<u8'0' + (base < 10 ? base : base / 10), char_type>};
-		if (*first != digit0)
+		if (*first != digit0) [[unlikely]]
 		{
 			return {first, parse_code::invalid};
 		}
-		if ((++first) == last)
+		if ((++first) == last) [[unlikely]]
 		{
 			return {first, parse_code::invalid};
 		}
 		if constexpr (10 < base)
 		{
 			constexpr auto digit1{char_literal_v<u8'0' + (base % 10), char_type>};
-			if (*first != digit1)
+			if (*first != digit1) [[unlikely]]
 			{
 				return {first, parse_code::invalid};
 			}
-			if ((++first) == last)
+			if ((++first) == last) [[unlikely]]
 			{
 				return {first, parse_code::invalid};
 			}
 		}
-		if (*first != char_literal_v<u8']', char_type>)
+		if (*first != char_literal_v<u8']', char_type>) [[unlikely]]
 		{
 			return {first, parse_code::invalid};
 		}
@@ -1018,7 +1018,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 	[[maybe_unused]] bool sign{};
 	if constexpr (my_signed_integral<T>)
 	{
-		if (first == last)
+		if (first == last) [[unlikely]]
 		{
 			return {first, parse_code::invalid};
 		}
@@ -1031,7 +1031,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 		{
 			if constexpr (base == 8)
 			{
-				if (first == last || *first != char_literal_v<u8'0', char_type>)
+				if (first == last || *first != char_literal_v<u8'0', char_type>) [[unlikely]]
 				{
 					return {first, parse_code::invalid};
 				}
@@ -1040,7 +1040,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 			else
 			{
 				auto phase_ret = scan_shbase_impl<base>(first, last);
-				if (phase_ret.code != ongoing_parse_code)
+				if (phase_ret.code != ongoing_parse_code) [[unlikely]]
 				{
 					return phase_ret;
 				}
@@ -1062,7 +1062,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 			{
 				++first;
 				first = ::fast_io::details::find_none_zero_simd_impl(first, last);
-				if (first == last)
+				if (first == last) [[likely]]
 				{
 					t = 0;
 					return {first, parse_code::ok};
@@ -1071,7 +1071,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 			else
 			{
 				++first;
-				if ((first == last) || (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first))))
+				if ((first == last) || (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first)))) [[likely]]
 				{
 					t = {};
 					return {first, parse_code::ok};
@@ -1104,7 +1104,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 			if constexpr (smaller_than_uint64)
 			{
 				constexpr unsigned_type umax{static_cast<unsigned_type>(-1)};
-				if (temp > umax)
+				if (temp > umax) [[unlikely]]
 				{
 					return {it, parse_code::overflow};
 				}
@@ -1118,7 +1118,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 		else [[unlikely]]
 		{
 			auto [it2, ec] = scan_int_contiguous_none_simd_space_part_define_impl<base>(first, last, res);
-			if (ec != parse_code::ok)
+			if (ec != parse_code::ok) [[unlikely]]
 			{
 				return {it2, ec};
 			}
@@ -1129,7 +1129,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 #endif
 	{
 		auto [it2, ec] = scan_int_contiguous_none_simd_space_part_define_impl<base>(first, last, res);
-		if (ec != parse_code::ok)
+		if (ec != parse_code::ok) [[unlikely]]
 		{
 			return {it2, ec};
 		}
@@ -1139,7 +1139,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ
 	{
 		constexpr unsigned_type umax{static_cast<unsigned_type>(-1)};
 		constexpr unsigned_type imax{umax >> 1};
-		if (res > (static_cast<my_make_unsigned_t<T>>(imax) + sign))
+		if (res > (static_cast<my_make_unsigned_t<T>>(imax) + sign)) [[unlikely]]
 		{
 			return {it, parse_code::overflow};
 		}
@@ -1177,7 +1177,7 @@ inline constexpr parse_result<char_type const *> scan_int_contiguous_define_impl
 		{
 			if constexpr (base == 8)
 			{
-				if (first == last || *first != char_literal_v<u8'0', char_type>)
+				if (first == last || *first != char_literal_v<u8'0', char_type>) [[unlikely]]
 				{
 					return {first, parse_code::invalid};
 				}
@@ -1186,7 +1186,7 @@ inline constexpr parse_result<char_type const *> scan_int_contiguous_define_impl
 			else
 			{
 				auto phase_ret = scan_shbase_impl<base>(first, last);
-				if (phase_ret.code != ongoing_parse_code)
+				if (phase_ret.code != ongoing_parse_code) [[unlikely]]
 				{
 					return phase_ret;
 				}
@@ -1310,7 +1310,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t
 	}
 	if constexpr (base == 8)
 	{
-		if (*first != char_literal_v<u8'0', char_type>)
+		if (*first != char_literal_v<u8'0', char_type>) [[unlikely]]
 		{
 			return {first, parse_code::invalid};
 		}
@@ -1320,7 +1320,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t
 		::std::uint_least8_t size_cache{sz};
 		if (size_cache == 0)
 		{
-			if (*first != char_literal_v<u8'0', char_type>)
+			if (*first != char_literal_v<u8'0', char_type>) [[unlikely]]
 			{
 				return {first, parse_code::invalid};
 			}
@@ -1338,13 +1338,13 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t
 		{
 			auto ch{*first};
 			if ((ch == char_literal_v<(base == 2 ? u8'B' : (base == 3 ? u8't' : u8'X')), char_type>) |
-				(ch == char_literal_v<(base == 2 ? u8'b' : (base == 3 ? u8't' : u8'x')), char_type>))
+				(ch == char_literal_v<(base == 2 ? u8'b' : (base == 3 ? u8't' : u8'x')), char_type>)) [[likely]]
 			{
 				sz = 0;
 				++first;
 				return {first, ongoing_parse_code};
 			}
-			else
+			else [[unlikely]]
 			{
 				return {first, parse_code::invalid};
 			}
@@ -1353,7 +1353,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t
 		{
 			if (size_cache == 1)
 			{
-				if (*first != char_literal_v<u8'[', char_type>)
+				if (*first != char_literal_v<u8'[', char_type>) [[unlikely]]
 				{
 					return {first, parse_code::invalid};
 				}
@@ -1366,7 +1366,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t
 			constexpr auto digit0{char_literal_v<u8'0' + (base < 10 ? base : base / 10), char_type>};
 			if (size_cache == 2)
 			{
-				if (*first != digit0)
+				if (*first != digit0) [[unlikely]]
 				{
 					return {first, parse_code::invalid};
 				}
@@ -1381,7 +1381,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t
 				constexpr auto digit1{char_literal_v<u8'0' + (base % 10), char_type>};
 				if (size_cache == 3)
 				{
-					if (*first != digit1)
+					if (*first != digit1) [[unlikely]]
 					{
 						return {first, parse_code::invalid};
 					}
@@ -1395,7 +1395,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t
 			constexpr ::std::uint_least8_t last_index{base < 10 ? 3 : 4};
 			if (size_cache == last_index)
 			{
-				if (*first != char_literal_v<u8']', char_type>)
+				if (*first != char_literal_v<u8']', char_type>) [[unlikely]]
 				{
 					return {first, parse_code::invalid};
 				}
@@ -1443,7 +1443,7 @@ inline constexpr parse_result<char_type const *> sc_int_ctx_zero_phase(scan_inte
 			}
 			return {first, parse_code::partial};
 		}
-		if (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first)))
+		if (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first))) [[likely]]
 		{
 			return {first, parse_code::ok};
 		}
@@ -1472,7 +1472,7 @@ inline constexpr parse_result<char_type const *> sc_int_ctx_digit_phase(State &s
 			st.integer_phase = scan_integral_context_phase::digit;
 			return {it, parse_code::partial};
 		}
-		if (st.size == 0)
+		if (st.size == 0) [[likely]]
 		{
 			t = {};
 			return {it, parse_code::ok};
@@ -1487,7 +1487,7 @@ inline constexpr parse_result<char_type const *> sc_int_ctx_digit_phase(State &s
 			st.integer_phase = scan_integral_context_phase::overflow;
 			return {it, parse_code::partial};
 		}
-		else
+		else [[unlikely]]
 		{
 			return {it, parse_code::overflow};
 		}
@@ -1504,7 +1504,7 @@ inline constexpr parse_result<char_type const *> sc_int_ctx_zero_invalid_phase(c
 		return {first, parse_code::partial};
 	}
 	++first;
-	if (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first)))
+	if (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first))) [[likely]]
 	{
 		return {first, parse_code::ok};
 	}
@@ -1554,7 +1554,7 @@ inline constexpr parse_result<char_type const *> scan_context_define_parse_impl(
 		if constexpr (!noskipws)
 		{
 			auto phase_ret = sc_int_ctx_space_phase(first, last);
-			if (phase_ret.code != ongoing_parse_code)
+			if (phase_ret.code != ongoing_parse_code) [[unlikely]]
 			{
 				return phase_ret;
 			}
@@ -1567,7 +1567,7 @@ inline constexpr parse_result<char_type const *> scan_context_define_parse_impl(
 		if constexpr (my_signed_integral<T>)
 		{
 			auto phase_ret = sc_int_ctx_sign_phase<true, false>(st, first, last);
-			if (phase_ret.code != ongoing_parse_code)
+			if (phase_ret.code != ongoing_parse_code) [[unlikely]]
 			{
 				return phase_ret;
 			}
@@ -1581,7 +1581,7 @@ inline constexpr parse_result<char_type const *> scan_context_define_parse_impl(
 		{
 			st.integer_phase = scan_integral_context_phase::prefix;
 			auto phase_ret = sc_int_ctx_prefix_phase<base>(st.size, first, last);
-			if (phase_ret.code != ongoing_parse_code)
+			if (phase_ret.code != ongoing_parse_code) [[unlikely]]
 			{
 				return phase_ret;
 			}
@@ -1673,9 +1673,7 @@ inline constexpr parse_code scan_context_eof_define_parse_impl(State &st, T &t)
 		}
 	}
 	case scan_integral_context_phase::digit:
-		return scan_int_contiguous_none_space_part_define_impl<base>(st.buffer.data(), st.buffer.data() + st.size,
-																	 t)
-			.code;
+		return scan_int_contiguous_none_space_part_define_impl<base>(st.buffer.data(), st.buffer.data() + st.size, t).code;
 	case scan_integral_context_phase::overflow:
 		return parse_code::overflow;
 	case scan_integral_context_phase::zero_skip:

Original file line number	Diff line number	Diff line change
`@@ -571,7 +571,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`571`	`571`	`constexpr ::std::uint_least64_t mask{0x000000FF000000FF};`
`572`	`572`	`constexpr ::std::uint_least64_t fullmask{baseval * 0x80};`
`573`	`573`
`574`		`- while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least64_t))`
	`574`	`+ while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least64_t)) [[likely]]`
`575`	`575`	`{`
`576`	`576`	`::std::uint_least64_t val;`
`577`	`577`	`::fast_io::freestanding::my_memcpy(__builtin_addressof(val), first, sizeof(::std::uint_least64_t));`
`@@ -581,7 +581,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`581`	`581`	`val = ::fast_io::little_endian(val);`
`582`	`582`	`}`
`583`	`583`
`584`		`- if (::std::uint_least64_t const cval{((val + first_bound) \| (val - zero_lower_bound)) & fullmask}; cval)`
	`584`	`+ if (::std::uint_least64_t const cval{((val + first_bound) \| (val - zero_lower_bound)) & fullmask}; cval) [[likely]]`
`585`	`585`	`{`
`586`	`586`	`auto ctrz_cval{::std::countr_zero(cval)};`
`587`	`587`	`auto const valid_bits{ctrz_cval & -8};`
`@@ -636,7 +636,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`636`	`636`	`val = ::fast_io::little_endian(val);`
`637`	`637`	`}`
`638`	`638`
`639`		`- if (::std::uint_least32_t const cval{((val + first_bound) \| (val - 0x30303030)) & 0x80808080}; cval)`
	`639`	`+ if (::std::uint_least32_t const cval{((val + first_bound) \| (val - 0x30303030)) & 0x80808080}; cval) [[likely]]`
`640`	`640`	`{`
`641`	`641`	`auto ctrz_cval{::std::countr_zero(cval)};`
`642`	`642`	`auto const valid_bits{ctrz_cval & -8};`
`@@ -686,7 +686,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`686`	`686`	`constexpr ::std::uint_least64_t first_bound{0x7fc67fc67fc67fc6 + 0x0001000100010001 * (10 - base)};`
`687`	`687`	`if constexpr (max_size >= u64_size_of_c16)`
`688`	`688`	`{`
`689`		`- while (static_cast<::std::size_t>(first_phase_last - first) >= u64_size_of_c16)`
	`689`	`+ while (static_cast<::std::size_t>(first_phase_last - first) >= u64_size_of_c16) [[likely]]`
`690`	`690`	`{`
`691`	`691`	`::std::uint_least64_t val;`
`692`	`692`	`::fast_io::freestanding::my_memcpy(__builtin_addressof(val), first, sizeof(::std::uint_least64_t));`
`@@ -696,7 +696,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`696`	`696`	`val = ::fast_io::little_endian(val);`
`697`	`697`	`}`
`698`	`698`
`699`		`- if (::std::uint_least64_t const cval{((val + first_bound) \| (val - 0x0030003000300030)) & 0x8000800080008000}; cval)`
	`699`	`+ if (::std::uint_least64_t const cval{((val + first_bound) \| (val - 0x0030003000300030)) & 0x8000800080008000}; cval) [[likely]]`
`700`	`700`	`{`
`701`	`701`	`auto ctrz_cval{::std::countr_zero(cval)};`
`702`	`702`	`auto const valid_bits{ctrz_cval & -16};`
`@@ -751,7 +751,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`751`	`751`	`constexpr ::std::uint_least64_t mask{0x000000FF000000FF};`
`752`	`752`	`constexpr ::std::uint_least64_t mul1{pow_base_sizeof_base_2 + (pow_base_sizeof_base_6 << 32)};`
`753`	`753`	`constexpr ::std::uint_least64_t mul2{1 + (pow_base_sizeof_base_4 << 32)};`
`754`		`- while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least64_t))`
	`754`	`+ while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least64_t)) [[likely]]`
`755`	`755`	`{`
`756`	`756`	`::std::uint_least64_t val;`
`757`	`757`	`::fast_io::freestanding::my_memcpy(__builtin_addressof(val), first, sizeof(::std::uint_least64_t));`
`@@ -767,7 +767,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`767`	`767`	`~(((val + 0x3f3f3f3f3f3f3f3f) \| (val - 0x4040404040404040)) &`
`768`	`768`	`((val + 0x1f1f1f1f1f1f1f1f) \| (val - 0x6060606060606060)))) &`
`769`	`769`	`0x8080808080808080};`
`770`		`- cval)`
	`770`	`+ cval) [[likely]]`
`771`	`771`	`{`
`772`	`772`	`auto ctrz_cval{::std::countr_zero(cval)};`
`773`	`773`	`auto const valid_bits{ctrz_cval & -8};`
`@@ -824,14 +824,14 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`824`	`824`
`825`	`825`	`constexpr ::std::uint_least32_t pow_base_sizeof_base_2{::fast_io::details::compile_pow_n<::std::uint_least32_t, base_char_type, 2>};`
`826`	`826`	`constexpr ::std::uint_least32_t mask{0x000000FF};`
`827`		`- while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least32_t))`
	`827`	`+ while (static_cast<::std::size_t>(first_phase_last - first) >= sizeof(::std::uint_least32_t)) [[likely]]`
`828`	`828`	`{`
`829`	`829`	`::std::uint_least32_t val;`
`830`	`830`	`::fast_io::freestanding::my_memcpy(__builtin_addressof(val), first, sizeof(::std::uint_least32_t));`
`831`	`831`
`832`	`832`	`val = ::fast_io::little_endian(val);`
`833`	`833`
`834`		`- if (::std::uint_least32_t const cval{((val + first_bound) \| (val - 0x30303030)) & 0x80808080}; cval) [[unlikely]]`
	`834`	`+ if (::std::uint_least32_t const cval{((val + first_bound) \| (val - 0x30303030)) & 0x80808080}; cval) [[likely]]`
`835`	`835`	`{`
`836`	`836`	`auto ctrz_cval{::std::countr_zero(cval)};`
`837`	`837`	`auto const valid_bits{ctrz_cval & -8};`
`@@ -874,7 +874,7 @@ runtime_scan_int_contiguous_none_simd_space_part_define_impl(char_type const *fi`
`874`	`874`	`}`
`875`	`875`	`}`
`876`	`876`
`877`		`- for (; first != first_phase_last; ++first)`
	`877`	`+ for (; first != first_phase_last; ++first) [[likely]]`
`878`	`878`	`{`
`879`	`879`	`unsigned_char_type ch{static_cast<unsigned_char_type>(*first)};`
`880`	`880`	`if (char_digit_to_literal<base, char_type>(ch)) [[unlikely]]`
`@@ -949,57 +949,57 @@ template <char8_t base, ::std::integral char_type>`
`949`	`949`	`inline constexpr parse_result<char_type const > scan_shbase_impl(char_type const first,`
`950`	`950`	`char_type const *last) noexcept`
`951`	`951`	`{`
`952`		`- if (first == last \|\| *first != char_literal_v<u8'0', char_type>)`
	`952`	`+ if (first == last \|\| *first != char_literal_v<u8'0', char_type>) [[unlikely]]`
`953`	`953`	`{`
`954`	`954`	`return {first, parse_code::invalid};`
`955`	`955`	`}`
`956`		`- if ((++first) == last)`
	`956`	`+ if ((++first) == last) [[unlikely]]`
`957`	`957`	`{`
`958`	`958`	`return {first, parse_code::invalid};`
`959`	`959`	`}`
`960`	`960`	`if constexpr (base == 2 \|\| base == 3 \|\| base == 16)`
`961`	`961`	`{`
`962`	`962`	`auto ch{*first};`
`963`	`963`	`if ((ch != char_literal_v<(base == 2 ? u8'B' : (base == 3 ? u8't' : u8'X')), char_type>)&(`
`964`		`- ch != char_literal_v<(base == 2 ? u8'b' : (base == 3 ? u8't' : u8'x')), char_type>))`
	`964`	`+ ch != char_literal_v<(base == 2 ? u8'b' : (base == 3 ? u8't' : u8'x')), char_type>)) [[unlikely]]`
`965`	`965`	`{`
`966`	`966`	`return {first, parse_code::invalid};`
`967`	`967`	`}`
`968`	`968`	`++first;`
`969`	`969`	`}`
`970`	`970`	`else`
`971`	`971`	`{`
`972`		`- if (*first != char_literal_v<u8'[', char_type>)`
	`972`	`+ if (*first != char_literal_v<u8'[', char_type>) [[unlikely]]`
`973`	`973`	`{`
`974`	`974`	`return {first, parse_code::invalid};`
`975`	`975`	`}`
`976`	`976`	`++first;`
`977`		`- if ((++first) == last)`
	`977`	`+ if ((++first) == last) [[unlikely]]`
`978`	`978`	`{`
`979`	`979`	`return {first, parse_code::invalid};`
`980`	`980`	`}`
`981`	`981`	`constexpr auto digit0{char_literal_v<u8'0' + (base < 10 ? base : base / 10), char_type>};`
`982`		`- if (*first != digit0)`
	`982`	`+ if (*first != digit0) [[unlikely]]`
`983`	`983`	`{`
`984`	`984`	`return {first, parse_code::invalid};`
`985`	`985`	`}`
`986`		`- if ((++first) == last)`
	`986`	`+ if ((++first) == last) [[unlikely]]`
`987`	`987`	`{`
`988`	`988`	`return {first, parse_code::invalid};`
`989`	`989`	`}`
`990`	`990`	`if constexpr (10 < base)`
`991`	`991`	`{`
`992`	`992`	`constexpr auto digit1{char_literal_v<u8'0' + (base % 10), char_type>};`
`993`		`- if (*first != digit1)`
	`993`	`+ if (*first != digit1) [[unlikely]]`
`994`	`994`	`{`
`995`	`995`	`return {first, parse_code::invalid};`
`996`	`996`	`}`
`997`		`- if ((++first) == last)`
	`997`	`+ if ((++first) == last) [[unlikely]]`
`998`	`998`	`{`
`999`	`999`	`return {first, parse_code::invalid};`
`1000`	`1000`	`}`
`1001`	`1001`	`}`
`1002`		`- if (*first != char_literal_v<u8']', char_type>)`
	`1002`	`+ if (*first != char_literal_v<u8']', char_type>) [[unlikely]]`
`1003`	`1003`	`{`
`1004`	`1004`	`return {first, parse_code::invalid};`
`1005`	`1005`	`}`
`@@ -1018,7 +1018,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1018`	`1018`	`[[maybe_unused]] bool sign{};`
`1019`	`1019`	`if constexpr (my_signed_integral<T>)`
`1020`	`1020`	`{`
`1021`		`- if (first == last)`
	`1021`	`+ if (first == last) [[unlikely]]`
`1022`	`1022`	`{`
`1023`	`1023`	`return {first, parse_code::invalid};`
`1024`	`1024`	`}`
`@@ -1031,7 +1031,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1031`	`1031`	`{`
`1032`	`1032`	`if constexpr (base == 8)`
`1033`	`1033`	`{`
`1034`		`- if (first == last \|\| *first != char_literal_v<u8'0', char_type>)`
	`1034`	`+ if (first == last \|\| *first != char_literal_v<u8'0', char_type>) [[unlikely]]`
`1035`	`1035`	`{`
`1036`	`1036`	`return {first, parse_code::invalid};`
`1037`	`1037`	`}`
`@@ -1040,7 +1040,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1040`	`1040`	`else`
`1041`	`1041`	`{`
`1042`	`1042`	`auto phase_ret = scan_shbase_impl<base>(first, last);`
`1043`		`- if (phase_ret.code != ongoing_parse_code)`
	`1043`	`+ if (phase_ret.code != ongoing_parse_code) [[unlikely]]`
`1044`	`1044`	`{`
`1045`	`1045`	`return phase_ret;`
`1046`	`1046`	`}`
`@@ -1062,7 +1062,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1062`	`1062`	`{`
`1063`	`1063`	`++first;`
`1064`	`1064`	`first = ::fast_io::details::find_none_zero_simd_impl(first, last);`
`1065`		`- if (first == last)`
	`1065`	`+ if (first == last) [[likely]]`
`1066`	`1066`	`{`
`1067`	`1067`	`t = 0;`
`1068`	`1068`	`return {first, parse_code::ok};`
`@@ -1071,7 +1071,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1071`	`1071`	`else`
`1072`	`1072`	`{`
`1073`	`1073`	`++first;`
`1074`		`- if ((first == last) \|\| (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first))))`
	`1074`	`+ if ((first == last) \|\| (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first)))) [[likely]]`
`1075`	`1075`	`{`
`1076`	`1076`	`t = {};`
`1077`	`1077`	`return {first, parse_code::ok};`
`@@ -1104,7 +1104,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1104`	`1104`	`if constexpr (smaller_than_uint64)`
`1105`	`1105`	`{`
`1106`	`1106`	`constexpr unsigned_type umax{static_cast<unsigned_type>(-1)};`
`1107`		`- if (temp > umax)`
	`1107`	`+ if (temp > umax) [[unlikely]]`
`1108`	`1108`	`{`
`1109`	`1109`	`return {it, parse_code::overflow};`
`1110`	`1110`	`}`
`@@ -1118,7 +1118,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1118`	`1118`	`else [[unlikely]]`
`1119`	`1119`	`{`
`1120`	`1120`	`auto [it2, ec] = scan_int_contiguous_none_simd_space_part_define_impl<base>(first, last, res);`
`1121`		`- if (ec != parse_code::ok)`
	`1121`	`+ if (ec != parse_code::ok) [[unlikely]]`
`1122`	`1122`	`{`
`1123`	`1123`	`return {it2, ec};`
`1124`	`1124`	`}`
`@@ -1129,7 +1129,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1129`	`1129`	`#endif`
`1130`	`1130`	`{`
`1131`	`1131`	`auto [it2, ec] = scan_int_contiguous_none_simd_space_part_define_impl<base>(first, last, res);`
`1132`		`- if (ec != parse_code::ok)`
	`1132`	`+ if (ec != parse_code::ok) [[unlikely]]`
`1133`	`1133`	`{`
`1134`	`1134`	`return {it2, ec};`
`1135`	`1135`	`}`
`@@ -1139,7 +1139,7 @@ scan_int_contiguous_none_space_part_define_impl(char_type const *first, char_typ`
`1139`	`1139`	`{`
`1140`	`1140`	`constexpr unsigned_type umax{static_cast<unsigned_type>(-1)};`
`1141`	`1141`	`constexpr unsigned_type imax{umax >> 1};`
`1142`		`- if (res > (static_cast<my_make_unsigned_t<T>>(imax) + sign))`
	`1142`	`+ if (res > (static_cast<my_make_unsigned_t<T>>(imax) + sign)) [[unlikely]]`
`1143`	`1143`	`{`
`1144`	`1144`	`return {it, parse_code::overflow};`
`1145`	`1145`	`}`
`@@ -1177,7 +1177,7 @@ inline constexpr parse_result<char_type const *> scan_int_contiguous_define_impl`
`1177`	`1177`	`{`
`1178`	`1178`	`if constexpr (base == 8)`
`1179`	`1179`	`{`
`1180`		`- if (first == last \|\| *first != char_literal_v<u8'0', char_type>)`
	`1180`	`+ if (first == last \|\| *first != char_literal_v<u8'0', char_type>) [[unlikely]]`
`1181`	`1181`	`{`
`1182`	`1182`	`return {first, parse_code::invalid};`
`1183`	`1183`	`}`
`@@ -1186,7 +1186,7 @@ inline constexpr parse_result<char_type const *> scan_int_contiguous_define_impl`
`1186`	`1186`	`else`
`1187`	`1187`	`{`
`1188`	`1188`	`auto phase_ret = scan_shbase_impl<base>(first, last);`
`1189`		`- if (phase_ret.code != ongoing_parse_code)`
	`1189`	`+ if (phase_ret.code != ongoing_parse_code) [[unlikely]]`
`1190`	`1190`	`{`
`1191`	`1191`	`return phase_ret;`
`1192`	`1192`	`}`
`@@ -1310,7 +1310,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t`
`1310`	`1310`	`}`
`1311`	`1311`	`if constexpr (base == 8)`
`1312`	`1312`	`{`
`1313`		`- if (*first != char_literal_v<u8'0', char_type>)`
	`1313`	`+ if (*first != char_literal_v<u8'0', char_type>) [[unlikely]]`
`1314`	`1314`	`{`
`1315`	`1315`	`return {first, parse_code::invalid};`
`1316`	`1316`	`}`
`@@ -1320,7 +1320,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t`
`1320`	`1320`	`::std::uint_least8_t size_cache{sz};`
`1321`	`1321`	`if (size_cache == 0)`
`1322`	`1322`	`{`
`1323`		`- if (*first != char_literal_v<u8'0', char_type>)`
	`1323`	`+ if (*first != char_literal_v<u8'0', char_type>) [[unlikely]]`
`1324`	`1324`	`{`
`1325`	`1325`	`return {first, parse_code::invalid};`
`1326`	`1326`	`}`
`@@ -1338,13 +1338,13 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t`
`1338`	`1338`	`{`
`1339`	`1339`	`auto ch{*first};`
`1340`	`1340`	`if ((ch == char_literal_v<(base == 2 ? u8'B' : (base == 3 ? u8't' : u8'X')), char_type>) \|`
`1341`		`- (ch == char_literal_v<(base == 2 ? u8'b' : (base == 3 ? u8't' : u8'x')), char_type>))`
	`1341`	`+ (ch == char_literal_v<(base == 2 ? u8'b' : (base == 3 ? u8't' : u8'x')), char_type>)) [[likely]]`
`1342`	`1342`	`{`
`1343`	`1343`	`sz = 0;`
`1344`	`1344`	`++first;`
`1345`	`1345`	`return {first, ongoing_parse_code};`
`1346`	`1346`	`}`
`1347`		`- else`
	`1347`	`+ else [[unlikely]]`
`1348`	`1348`	`{`
`1349`	`1349`	`return {first, parse_code::invalid};`
`1350`	`1350`	`}`
`@@ -1353,7 +1353,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t`
`1353`	`1353`	`{`
`1354`	`1354`	`if (size_cache == 1)`
`1355`	`1355`	`{`
`1356`		`- if (*first != char_literal_v<u8'[', char_type>)`
	`1356`	`+ if (*first != char_literal_v<u8'[', char_type>) [[unlikely]]`
`1357`	`1357`	`{`
`1358`	`1358`	`return {first, parse_code::invalid};`
`1359`	`1359`	`}`
`@@ -1366,7 +1366,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t`
`1366`	`1366`	`constexpr auto digit0{char_literal_v<u8'0' + (base < 10 ? base : base / 10), char_type>};`
`1367`	`1367`	`if (size_cache == 2)`
`1368`	`1368`	`{`
`1369`		`- if (*first != digit0)`
	`1369`	`+ if (*first != digit0) [[unlikely]]`
`1370`	`1370`	`{`
`1371`	`1371`	`return {first, parse_code::invalid};`
`1372`	`1372`	`}`
`@@ -1381,7 +1381,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t`
`1381`	`1381`	`constexpr auto digit1{char_literal_v<u8'0' + (base % 10), char_type>};`
`1382`	`1382`	`if (size_cache == 3)`
`1383`	`1383`	`{`
`1384`		`- if (*first != digit1)`
	`1384`	`+ if (*first != digit1) [[unlikely]]`
`1385`	`1385`	`{`
`1386`	`1386`	`return {first, parse_code::invalid};`
`1387`	`1387`	`}`
`@@ -1395,7 +1395,7 @@ sc_int_ctx_prefix_phase(::std::uint_least8_t &sz, char_type const *first, char_t`
`1395`	`1395`	`constexpr ::std::uint_least8_t last_index{base < 10 ? 3 : 4};`
`1396`	`1396`	`if (size_cache == last_index)`
`1397`	`1397`	`{`
`1398`		`- if (*first != char_literal_v<u8']', char_type>)`
	`1398`	`+ if (*first != char_literal_v<u8']', char_type>) [[unlikely]]`
`1399`	`1399`	`{`
`1400`	`1400`	`return {first, parse_code::invalid};`
`1401`	`1401`	`}`
`@@ -1443,7 +1443,7 @@ inline constexpr parse_result<char_type const *> sc_int_ctx_zero_phase(scan_inte`
`1443`	`1443`	`}`
`1444`	`1444`	`return {first, parse_code::partial};`
`1445`	`1445`	`}`
`1446`		`- if (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first)))`
	`1446`	`+ if (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first))) [[likely]]`
`1447`	`1447`	`{`
`1448`	`1448`	`return {first, parse_code::ok};`
`1449`	`1449`	`}`
`@@ -1472,7 +1472,7 @@ inline constexpr parse_result<char_type const *> sc_int_ctx_digit_phase(State &s`
`1472`	`1472`	`st.integer_phase = scan_integral_context_phase::digit;`
`1473`	`1473`	`return {it, parse_code::partial};`
`1474`	`1474`	`}`
`1475`		`- if (st.size == 0)`
	`1475`	`+ if (st.size == 0) [[likely]]`
`1476`	`1476`	`{`
`1477`	`1477`	`t = {};`
`1478`	`1478`	`return {it, parse_code::ok};`
`@@ -1487,7 +1487,7 @@ inline constexpr parse_result<char_type const *> sc_int_ctx_digit_phase(State &s`
`1487`	`1487`	`st.integer_phase = scan_integral_context_phase::overflow;`
`1488`	`1488`	`return {it, parse_code::partial};`
`1489`	`1489`	`}`
`1490`		`- else`
	`1490`	`+ else [[unlikely]]`
`1491`	`1491`	`{`
`1492`	`1492`	`return {it, parse_code::overflow};`
`1493`	`1493`	`}`
`@@ -1504,7 +1504,7 @@ inline constexpr parse_result<char_type const *> sc_int_ctx_zero_invalid_phase(c`
`1504`	`1504`	`return {first, parse_code::partial};`
`1505`	`1505`	`}`
`1506`	`1506`	`++first;`
`1507`		`- if (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first)))`
	`1507`	`+ if (!char_is_digit<base, char_type>(static_cast<unsigned_char_type>(*first))) [[likely]]`
`1508`	`1508`	`{`
`1509`	`1509`	`return {first, parse_code::ok};`
`1510`	`1510`	`}`
`@@ -1554,7 +1554,7 @@ inline constexpr parse_result<char_type const *> scan_context_define_parse_impl(`
`1554`	`1554`	`if constexpr (!noskipws)`
`1555`	`1555`	`{`
`1556`	`1556`	`auto phase_ret = sc_int_ctx_space_phase(first, last);`
`1557`		`- if (phase_ret.code != ongoing_parse_code)`
	`1557`	`+ if (phase_ret.code != ongoing_parse_code) [[unlikely]]`
`1558`	`1558`	`{`
`1559`	`1559`	`return phase_ret;`
`1560`	`1560`	`}`
`@@ -1567,7 +1567,7 @@ inline constexpr parse_result<char_type const *> scan_context_define_parse_impl(`
`1567`	`1567`	`if constexpr (my_signed_integral<T>)`
`1568`	`1568`	`{`
`1569`	`1569`	`auto phase_ret = sc_int_ctx_sign_phase<true, false>(st, first, last);`
`1570`		`- if (phase_ret.code != ongoing_parse_code)`
	`1570`	`+ if (phase_ret.code != ongoing_parse_code) [[unlikely]]`
`1571`	`1571`	`{`
`1572`	`1572`	`return phase_ret;`
`1573`	`1573`	`}`
`@@ -1581,7 +1581,7 @@ inline constexpr parse_result<char_type const *> scan_context_define_parse_impl(`
`1581`	`1581`	`{`
`1582`	`1582`	`st.integer_phase = scan_integral_context_phase::prefix;`
`1583`	`1583`	`auto phase_ret = sc_int_ctx_prefix_phase<base>(st.size, first, last);`
`1584`		`- if (phase_ret.code != ongoing_parse_code)`
	`1584`	`+ if (phase_ret.code != ongoing_parse_code) [[unlikely]]`
`1585`	`1585`	`{`
`1586`	`1586`	`return phase_ret;`
`1587`	`1587`	`}`
`@@ -1673,9 +1673,7 @@ inline constexpr parse_code scan_context_eof_define_parse_impl(State &st, T &t)`
`1673`	`1673`	`}`
`1674`	`1674`	`}`
`1675`	`1675`	`case scan_integral_context_phase::digit:`
`1676`		`- return scan_int_contiguous_none_space_part_define_impl<base>(st.buffer.data(), st.buffer.data() + st.size,`
`1677`		`- t)`
`1678`		`- .code;`
	`1676`	`+ return scan_int_contiguous_none_space_part_define_impl<base>(st.buffer.data(), st.buffer.data() + st.size, t).code;`
`1679`	`1677`	`case scan_integral_context_phase::overflow:`
`1680`	`1678`	`return parse_code::overflow;`
`1681`	`1679`	`case scan_integral_context_phase::zero_skip:`