always get OOM during upload to s3

clickhouse-backup --version
Version: 2.0.0
Git Commit: 93f6c2f
Build Date: 2022-09-04
uname -a
Linux clickhouse01.sj-dev.local 3.10.0-1160.49.1.el7.x86_64 #1 SMP Tue Nov 30 15:51:32 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
cat /etc/os-release
NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="https://www.centos.org/"
BUG_REPORT_URL="https://bugs.centos.org/"

CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"

Oct 22 10:59:06 clickhouse01 kernel: clickhouse-back invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
Oct 22 10:59:06 clickhouse01 kernel: clickhouse-back cpuset=/ mems_allowed=0-1
Oct 22 10:59:06 clickhouse01 kernel: CPU: 26 PID: 399012 Comm: clickhouse-back Kdump: loaded Not tainted 3.10.0-1160.49.1.el7.x86_64 #1
Oct 22 10:59:06 clickhouse01 kernel: Hardware name: ASUSTeK COMPUTER INC. RS520-E9-RS12-E/Z11PR-D16 Series, BIOS 5501 04/18/2019
Oct 22 10:59:06 clickhouse01 kernel: Call Trace:
Oct 22 10:59:06 clickhouse01 kernel: [] dump_stack+0x19/0x1b
Oct 22 10:59:06 clickhouse01 kernel: [] dump_header+0x90/0x229
Oct 22 10:59:06 clickhouse01 kernel: [] ? ktime_get_ts64+0x52/0xf0
Oct 22 10:59:06 clickhouse01 kernel: [] oom_kill_process+0x2cd/0x490
Oct 22 10:59:06 clickhouse01 kernel: [] ? oom_unkillable_task+0xcd/0x120
Oct 22 10:59:06 clickhouse01 kernel: [] out_of_memory+0x31a/0x500
Oct 22 10:59:06 clickhouse01 kernel: [] __alloc_pages_nodemask+0xad4/0xbe0
Oct 22 10:59:06 clickhouse01 kernel: [] alloc_pages_current+0x98/0x110
Oct 22 10:59:06 clickhouse01 kernel: [] __page_cache_alloc+0x97/0xb0
Oct 22 10:59:06 clickhouse01 kernel: [] filemap_fault+0x270/0x420
Oct 22 10:59:06 clickhouse01 kernel: [] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
Oct 22 10:59:06 clickhouse01 kernel: [] xfs_filemap_fault+0x2c/0x30 [xfs]
Oct 22 10:59:06 clickhouse01 kernel: [] __do_fault.isra.61+0x8a/0x100
Oct 22 10:59:06 clickhouse01 kernel: [] do_read_fault.isra.63+0x4c/0x1b0
Oct 22 10:59:06 clickhouse01 kernel: [] ? recalc_sigpending+0x1b/0x70
Oct 22 10:59:06 clickhouse01 kernel: [] handle_mm_fault+0xa20/0xfb0
Oct 22 10:59:06 clickhouse01 kernel: [] ? signal_delivered+0x6f/0xa0
Oct 22 10:59:06 clickhouse01 kernel: [] __do_page_fault+0x213/0x500
Oct 22 10:59:06 clickhouse01 kernel: [] do_page_fault+0x35/0x90
Oct 22 10:59:06 clickhouse01 kernel: [] page_fault+0x28/0x30
Oct 22 10:59:06 clickhouse01 kernel: Mem-Info:
Oct 22 10:59:06 clickhouse01 kernel: active_anon:14676774 inactive_anon:951461 isolated_anon:96#012 active_file:1314 inactive_file:1180 isolated_file:47#012 unevictable:0 dirty:2 writeback:8 unstable:0#012 slab_reclaimable:71380 slab_unreclaimable:75905#012 mapped:1380 shmem:1093 pagetables:239133 bounce:0#012 free:56749 free_pcp:366 free_cma:0
Oct 22 10:59:06 clickhouse01 kernel: Node 0 DMA free:15900kB min:20kB low:24kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15984kB managed:15900kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
Oct 22 10:59:06 clickhouse01 kernel: lowmem_reserve[]: 0 1418 31608 31608
Oct 22 10:59:06 clickhouse01 kernel: Node 0 DMA32 free:122808kB min:2000kB low:2500kB high:3000kB active_anon:918668kB inactive_anon:308540kB active_file:208kB inactive_file:156kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:1686056kB managed:1452552kB mlocked:0kB dirty:0kB writeback:0kB mapped:436kB shmem:172kB slab_reclaimable:35092kB slab_unreclaimable:26344kB kernel_stack:4688kB pagetables:12320kB unstable:0kB bounce:0kB free_pcp:4kB local_pcp:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no
Oct 22 10:59:06 clickhouse01 kernel: lowmem_reserve[]: 0 0 30190 30190
Oct 22 10:59:06 clickhouse01 kernel: Node 0 Normal free:42848kB min:42612kB low:53264kB high:63916kB active_anon:28044132kB inactive_anon:1749984kB active_file:3860kB inactive_file:3980kB unevictable:0kB isolated(anon):128kB isolated(file):188kB present:31457280kB managed:30914724kB mlocked:0kB dirty:8kB writeback:16kB mapped:3364kB shmem:2960kB slab_reclaimable:104484kB slab_unreclaimable:140728kB kernel_stack:15872kB pagetables:428756kB unstable:0kB bounce:0kB free_pcp:1396kB local_pcp:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:16051 all_unreclaimable? yes
Oct 22 10:59:06 clickhouse01 kernel: lowmem_reserve[]: 0 0 0 0
Oct 22 10:59:06 clickhouse01 kernel: Node 1 Normal free:45440kB min:45472kB low:56840kB high:68208kB active_anon:29744296kB inactive_anon:1747320kB active_file:1188kB inactive_file:584kB unevictable:0kB isolated(anon):256kB isolated(file):0kB present:33554432kB managed:32991508kB mlocked:0kB dirty:0kB writeback:16kB mapped:1720kB shmem:1240kB slab_reclaimable:145944kB slab_unreclaimable:136548kB kernel_stack:16208kB pagetables:515456kB unstable:0kB bounce:0kB free_pcp:56kB local_pcp:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:9886 all_unreclaimable? yes
Oct 22 10:59:06 clickhouse01 kernel: lowmem_reserve[]: 0 0 0 0
Oct 22 10:59:06 clickhouse01 kernel: Node 0 DMA: 14kB (U) 18kB (U) 116kB (U) 032kB 264kB (U) 1128kB (U) 1256kB (U) 0512kB 11024kB (U) 12048kB (M) 34096kB (M) = 15900kB
Oct 22 10:59:06 clickhouse01 kernel: Node 0 DMA32: 6584kB (UEM) 5138kB (UEM) 30516kB (UEM) 62632kB (UEM) 47364kB (UEM) 276128kB (UEM) 80256kB (UEM) 8512kB (UEM) 11024kB (M) 02048kB 04096kB = 122848kB
Oct 22 10:59:06 clickhouse01 kernel: Node 0 Normal: 61104kB (UE) 24848kB (U) 016kB 032kB 064kB 0128kB 0256kB 0512kB 01024kB 02048kB 04096kB = 44312kB
Oct 22 10:59:06 clickhouse01 kernel: Node 1 Normal: 105554kB (UEM) 6708kB (UM) 1416kB (UM) 032kB 064kB 0128kB 0256kB 0512kB 01024kB 02048kB 04096kB = 47804kB
Oct 22 10:59:06 clickhouse01 kernel: Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB
Oct 22 10:59:06 clickhouse01 kernel: Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
Oct 22 10:59:07 clickhouse01 kernel: Node 1 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB
Oct 22 10:59:07 clickhouse01 kernel: Node 1 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
Oct 22 10:59:07 clickhouse01 kernel: 6336 total pagecache pages
Oct 22 10:59:07 clickhouse01 kernel: 2451 pages in swap cache
Oct 22 10:59:07 clickhouse01 kernel: Swap cache stats: add 17476952, delete 17474779, find 136621503/139510410
Oct 22 10:59:07 clickhouse01 kernel: Free swap = 0kB
Oct 22 10:59:07 clickhouse01 kernel: Total swap = 8388604kB
Oct 22 10:59:07 clickhouse01 kernel: 16678438 pages RAM
Oct 22 10:59:07 clickhouse01 kernel: 0 pages HighMem/MovableOnly
Oct 22 10:59:07 clickhouse01 kernel: 334767 pages reserved
Oct 22 10:59:07 clickhouse01 kernel: [ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
Oct 22 10:59:07 clickhouse01 kernel: [ 1050] 0 1050 36493 50 71 44 0 systemd-journal
Oct 22 10:59:07 clickhouse01 kernel: [ 1073] 0 1073 11735 2 24 502 -1000 systemd-udevd
Oct 22 10:59:07 clickhouse01 kernel: [ 1078] 0 1078 68089 28 32 98 0 lvmetad
Oct 22 10:59:07 clickhouse01 kernel: [ 1515] 0 1515 13896 20 27 93 -1000 auditd
Oct 22 10:59:07 clickhouse01 kernel: [ 1538] 81 1538 15096 107 36 81 -900 dbus-daemon
Oct 22 10:59:07 clickhouse01 kernel: [ 1549] 28 1549 642815 93 103 449 0 nscd
Oct 22 10:59:07 clickhouse01 kernel: [ 1597] 0 1597 13233 44 32 168 0 smartd
Oct 22 10:59:07 clickhouse01 kernel: [ 1601] 0 1601 5490 111 15 46 0 irqbalance
Oct 22 10:59:07 clickhouse01 kernel: [ 1626] 0 1626 1304 1 8 31 0 mcelog
Oct 22 10:59:07 clickhouse01 kernel: [ 1640] 998 1640 23642 30 17 50 0 chronyd
Oct 22 10:59:07 clickhouse01 kernel: [ 1959] 0 1959 57076 650 65 782 0 snmpd
Oct 22 10:59:07 clickhouse01 kernel: [ 1964] 0 1964 187315 75 209 1321 0 rsyslogd
Oct 22 10:59:07 clickhouse01 kernel: [ 1966] 0 1966 28323 0 12 55 0 clickhouse-jdbc
Oct 22 10:59:07 clickhouse01 kernel: [ 1967] 0 1967 6858 15 19 49 0 xinetd
Oct 22 10:59:07 clickhouse01 kernel: [ 2044] 0 2044 2492506 25870 485 85764 0 java
Oct 22 10:59:07 clickhouse01 kernel: [ 2072] 0 2072 7787 0 20 247 0 nginx
Oct 22 10:59:07 clickhouse01 kernel: [ 2073] 996 2073 7894 0 20 356 0 nginx
Oct 22 10:59:07 clickhouse01 kernel: [ 2087] 993 2087 29911 8 44 244 0 zabbix_agentd
Oct 22 10:59:07 clickhouse01 kernel: [ 2106] 993 2106 29911 881 47 230 0 zabbix_agentd
Oct 22 10:59:07 clickhouse01 kernel: [ 2107] 993 2107 29995 34 44 253 0 zabbix_agentd
Oct 22 10:59:07 clickhouse01 kernel: [ 2109] 993 2109 29995 33 44 254 0 zabbix_agentd
Oct 22 10:59:07 clickhouse01 kernel: [ 2110] 993 2110 29995 35 44 252 0 zabbix_agentd
Oct 22 10:59:07 clickhouse01 kernel: [ 2111] 993 2111 29997 70 46 257 0 zabbix_agentd
Oct 22 10:59:07 clickhouse01 kernel: [ 2115] 0 2115 65011 106 26 2880 0 lsi_mrdsnmpagen
Oct 22 10:59:07 clickhouse01 kernel: [ 2139] 0 2139 28374 1 11 45 0 sh
Oct 22 10:59:07 clickhouse01 kernel: [ 2141] 0 2141 6112543 6281 434 95361 0 java
Oct 22 10:59:07 clickhouse01 kernel: [ 2255] 0 2255 5442 9 12 106 0 lsi_mrdsnmpagen
Oct 22 10:59:07 clickhouse01 kernel: [ 2324] 2 2324 5789 16 16 79 0 slpd
Oct 22 10:59:07 clickhouse01 kernel: [ 2604] 0 2604 112890 58 56 548 0 nslcd
Oct 22 10:59:07 clickhouse01 kernel: [ 2614] 0 2614 31141 25 16 136 0 crond
Oct 22 10:59:07 clickhouse01 kernel: [ 2648] 0 2648 27074 1 8 27 0 agetty
Oct 22 10:59:07 clickhouse01 kernel: [ 2662] 0 2662 22351 44 45 434 0 sendmail
Oct 22 10:59:07 clickhouse01 kernel: [ 2679] 51 2679 20680 36 40 348 0 sendmail
Oct 22 10:59:07 clickhouse01 kernel: [ 2715] 1000 2715 28321 2 12 48 0 agent.sh
Oct 22 10:59:07 clickhouse01 kernel: [ 2745] 1000 2745 149233 116 43 6277 0 python
Oct 22 10:59:07 clickhouse01 kernel: [ 2853] 0 2853 5925390 42 280 33159 0 java
Oct 22 10:59:07 clickhouse01 kernel: [ 3925] 999 3925 152671 107 62 1804 0 polkitd
Oct 22 10:59:07 clickhouse01 kernel: [ 4860] 997 4860 139960 521 118 6455 0 clckhouse-watch
Oct 22 10:59:07 clickhouse01 kernel: [ 4861] 997 4861 120464330 355534 203563 1302192 0 clickhouse-serv
Oct 22 10:59:07 clickhouse01 kernel: [197036] 1000 197036 27143 2 8 32 0 rsync
Oct 22 10:59:07 clickhouse01 kernel: [160210] 0 160210 6131 43 18 38 0 systemd-logind
Oct 22 10:59:07 clickhouse01 kernel: [56264] 0 56264 27834 24 57 232 -1000 sshd
Oct 22 10:59:07 clickhouse01 kernel: [180766] 10578 180766 32666 1 20 193 0 screen
Oct 22 10:59:07 clickhouse01 kernel: [180767] 10578 180767 40098 347 35 756 0 zsh
Oct 22 10:59:07 clickhouse01 kernel: [180812] 10578 180812 40194 23 33 678 0 zsh
Oct 22 10:59:07 clickhouse01 kernel: [180814] 10578 180814 40194 51 33 650 0 zsh
Oct 22 10:59:07 clickhouse01 kernel: [180816] 10578 180816 39738 57 32 691 0 zsh
Oct 22 10:59:07 clickhouse01 kernel: [180821] 10578 180821 1701 5 6 52 0 gitstatusd-linu
Oct 22 10:59:07 clickhouse01 kernel: [393077] 0 393077 39913 5 82 328 0 sshd
Oct 22 10:59:07 clickhouse01 kernel: [393085] 10578 393085 40332 246 80 535 0 sshd
Oct 22 10:59:07 clickhouse01 kernel: [393086] 10578 393086 28409 0 14 97 0 bash
Oct 22 10:59:07 clickhouse01 kernel: [393118] 0 393118 61516 1 72 297 0 sudo
Oct 22 10:59:07 clickhouse01 kernel: [393119] 0 393119 28442 88 13 45 0 bash
Oct 22 10:59:07 clickhouse01 kernel: [398998] 0 398998 19930814 15220492 31906 565298 0 clickhouse-back
Oct 22 10:59:07 clickhouse01 kernel: [399314] 0 399314 39913 328 80 5 0 sshd
Oct 22 10:59:07 clickhouse01 kernel: [399322] 10578 399322 39913 330 79 4 0 sshd
Oct 22 10:59:07 clickhouse01 kernel: [399323] 10578 399323 28409 96 13 0 0 bash
Oct 22 10:59:07 clickhouse01 kernel: [399346] 0 399346 61516 297 75 1 0 sudo
Oct 22 10:59:07 clickhouse01 kernel: [399347] 0 399347 28442 122 12 2 0 bash
Oct 22 10:59:07 clickhouse01 kernel: [399738] 0 399738 27024 23 9 0 0 tail
Oct 22 10:59:07 clickhouse01 kernel: [400092] 0 400092 50760 168 53 112 0 crond
Oct 22 10:59:07 clickhouse01 kernel: [400143] 0 400143 28321 45 12 0 0 sh
Oct 22 10:59:07 clickhouse01 kernel: [400144] 0 400144 28321 45 12 0 0 clickhouse-metr
Oct 22 10:59:07 clickhouse01 kernel: [400145] 0 400145 11287 353 26 0 0 nc
Oct 22 10:59:07 clickhouse01 kernel: [400147] 0 400147 28321 46 9 0 0 clickhouse-metr
Oct 22 10:59:07 clickhouse01 kernel: [400148] 0 400148 28411 37 12 0 0 awk
Oct 22 10:59:07 clickhouse01 kernel: [400149] 0 400149 110656 545 73 2 0 clickhouse-clie
Oct 22 10:59:07 clickhouse01 kernel: [400151] 0 400151 28411 38 12 0 0 awk
Oct 22 10:59:07 clickhouse01 kernel: [400169] 0 400169 31141 35 14 126 0 crond
Oct 22 10:59:07 clickhouse01 kernel: Out of memory: Kill process 398998 (clickhouse-back) score 832 or sacrifice child
Oct 22 10:59:07 clickhouse01 kernel: Killed process 398998 (clickhouse-back), UID 0, total-vm:79723256kB, anon-rss:60881960kB, file-rss:0kB, shmem-rss:0kB

during upload and after oom
[root@clickhouse01 backup]# free -mt
total used free shared buff/cache available
Mem: 63842 62385 251 4 1204 967
Swap: 8191 8191 0
Total: 72034 70577 251
[root@clickhouse01 backup]# free -mt
total used free shared buff/cache available
Mem: 63842 4743 58254 4 843 58608
Swap: 8191 5772 2419
Total: 72034 10516 60674