Skip to content

Commit 38afcd7

Browse files
var77Ngalstyan4
var77
authored and
Ngalstyan4
committed
CI improvements, codecov comments, formatting check, github releases
1 parent ffcfe79 commit 38afcd7

9 files changed

Lines changed: 53 additions & 137 deletions

File tree

‎.github/workflows/build-linux.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@ on:
88
branches:
99
- main
1010
- dev
11+
workflow_dispatch:
12+
inputs:
13+
create_release:
14+
type: boolean
15+
description: "Create GitHub release for current version"
16+
required: false
17+
default: false
1118
jobs:
1219
ubuntu-build:
1320
runs-on: ${{ matrix.os }}
@@ -69,3 +76,12 @@ jobs:
6976
- uses: geekyeggo/delete-artifact@v2
7077
with:
7178
name: lanterndb-package
79+
- name: Create GitHub release
80+
uses: softprops/action-gh-release@v1
81+
id: create_release
82+
if: ${{ github.event_name == 'workflow_dispatch' && inputs.create_release }}
83+
with:
84+
name: LanternDB v${{ steps.package.outputs.package_version }}
85+
tag_name: v${{ steps.package.outputs.package_version }}
86+
files: ${{ steps.package.outputs.package_path }}
87+
generate_release_notes: true

‎.github/workflows/test-linux.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,5 @@ jobs:
6969
name: codecov-lanterndb
7070
fail_ci_if_error: true
7171
files: /tmp/coverage.xml
72+
- name: Check clang formatting
73+
run: "cd /tmp/lanterndb/build && make format_check"

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# CHANGELOG

‎CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,13 @@ if (CLANG_FORMAT)
185185
COMMENT "Formatting code with clang-format"
186186
VERBATIM
187187
)
188+
# Add format check target
189+
add_custom_target(
190+
format_check
191+
COMMAND ${CLANG_FORMAT} --dry-run -Werror ${SOURCES}
192+
COMMENT "Checking code formatting with clang-format"
193+
VERBATIM
194+
)
188195
endif()
189196

190197
add_custom_target(

‎ci/scripts/build-linux.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ echo "LANG=en_US.UTF-8" > /etc/locale.conf && \
4040
apt update -y && apt install locales -y && \
4141
locale-gen en_US.UTF-8 && \
4242
# Install required packages for build
43-
apt install lsb-core build-essential automake cmake wget git dpkg-dev gcovr -y && \
43+
apt install lsb-core build-essential automake cmake wget git dpkg-dev gcovr clang-format -y && \
4444
# Add postgresql apt repo
4545
export ARCH=$(dpkg-architecture -q DEB_BUILD_ARCH) && \
4646
sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
@@ -57,7 +57,7 @@ if [ -z ${USE_SOURCE} ]; then
5757
cd /tmp && git clone --recursive https://github.com/lanterndata/lanterndb.git -b $BRANCH
5858
else
5959
# Use already checkouted code
60-
mkdir -p /tmp/lanterndb && cp -r ./* /tmp/lanterndb/
60+
shopt -s dotglob && mkdir -p /tmp/lanterndb && cp -r ./* /tmp/lanterndb/
6161
fi
6262

6363
cd /tmp/lanterndb && mkdir build && cd build && \

‎ci/scripts/universal-package.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,4 @@ done
3737
cd /tmp && tar cf ${PACKAGE_NAME}.tar $PACKAGE_NAME
3838
echo "package_name=${PACKAGE_NAME}.tar" >> $GITHUB_OUTPUT
3939
echo "package_path=/tmp/${PACKAGE_NAME}.tar" >> $GITHUB_OUTPUT
40+
echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_OUTPUT

‎codecov.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
comment: # this is a top-level key
2+
layout: " diff, flags, files"
3+
behavior: default
4+
require_changes: false # if true: only post the comment if coverage changes
5+
require_base: false # [true :: must have a base report to post]
6+
require_head: true # [true :: must have a head report to post]

‎src/hnsw/external_index.c‎

Lines changed: 17 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,7 @@ void ldb_wal_retriever_area_init(int size)
337337
#else
338338
takenbuffers = palloc0(sizeof(Buffer) * TAKENBUFFERS_MAX);
339339
if(takenbuffers_next > 0) {
340-
elog(ERROR, "takenbuffers_next > 0 %d", takenbuffers_next);
340+
elog(ERROR, "takenbuffers_next > 0 %d", takenbuffers_next);
341341
}
342342
#endif
343343

@@ -564,11 +564,11 @@ void ldb_wal_retriever_area_reset()
564564
wal_retriever_area_offset = 0;
565565
#else
566566
for(int i = 0; i < TAKENBUFFERS_MAX; i++) {
567-
if(takenbuffers[ i ] == InvalidBuffer) {
568-
continue;
567+
if(takenbuffers[ i ] == InvalidBuffer) {
568+
continue;
569569
}
570-
ReleaseBuffer(takenbuffers[ i ]);
571-
takenbuffers[ i ] = InvalidBuffer;
570+
ReleaseBuffer(takenbuffers[ i ]);
571+
takenbuffers[ i ] = InvalidBuffer;
572572
}
573573
takenbuffers_next = 0;
574574
#endif
@@ -585,134 +585,17 @@ void ldb_wal_retriever_area_free()
585585
wal_retriever_area_offset = 0;
586586
#else
587587
for(int i = 0; i < TAKENBUFFERS_MAX; i++) {
588-
if(takenbuffers[ i ] == InvalidBuffer) {
589-
continue;
588+
if(takenbuffers[ i ] == InvalidBuffer) {
589+
continue;
590590
}
591-
ReleaseBuffer(takenbuffers[ i ]);
592-
takenbuffers[ i ] = InvalidBuffer;
591+
ReleaseBuffer(takenbuffers[ i ]);
592+
takenbuffers[ i ] = InvalidBuffer;
593593
}
594594
pfree(takenbuffers);
595595
takenbuffers_next = 0;
596596
#endif
597597
}
598598

599-
static inline void *wal_index_node_retriever_sequential(int id)
600-
{
601-
BlockNumber blockno = 1;
602-
Buffer buf;
603-
Page page;
604-
HnswIndexPageSpecialBlock *special_block;
605-
HnswIndexTuple *nodepage;
606-
OffsetNumber offset, max_offset;
607-
// static cnt = 0;
608-
609-
elog(INFO, "Hnsw_disk: ldb_wal_index_node_retriever called with id %d", id);
610-
611-
while(BlockNumberIsValid(blockno)) {
612-
buf = ReadBufferExtended(INDEX_RELATION_FOR_RETRIEVER, MAIN_FORKNUM, blockno, RBM_NORMAL, NULL);
613-
LockBuffer(buf, BUFFER_LOCK_SHARE);
614-
page = BufferGetPage(buf);
615-
max_offset = PageGetMaxOffsetNumber(page);
616-
617-
for(offset = FirstOffsetNumber; offset <= max_offset; offset = OffsetNumberNext(offset)) {
618-
nodepage = (HnswIndexTuple *)PageGetItem(page, PageGetItemId(page, offset));
619-
if(nodepage->id == id) {
620-
#if LANTERNDB_COPYNODES
621-
if(wal_retriever_area == NULL || wal_retriever_area_offset + nodepage->size > wal_retriever_area_size) {
622-
elog(ERROR,
623-
"ERROR: wal_retriever_area "
624-
"is NULL or full");
625-
}
626-
memcpy(wal_retriever_area + wal_retriever_area_offset, nodepage->node, nodepage->size);
627-
wal_retriever_area_offset += nodepage->size;
628-
UnlockReleaseBuffer(buf);
629-
return wal_retriever_area + wal_retriever_area_offset - nodepage->size;
630-
#else
631-
UnlockReleaseBuffer(buf);
632-
elog(ERROR, "Nocopy is unimplemented for sequential index node retriever");
633-
#endif
634-
}
635-
}
636-
special_block = (HnswIndexPageSpecialBlock *)PageGetSpecialPointer(page);
637-
blockno = special_block->nextblockno;
638-
UnlockReleaseBuffer(buf);
639-
}
640-
elog(ERROR, "reached end of retriever without finding node %d", id);
641-
}
642-
643-
static void *wal_index_node_retriever_binary(int id)
644-
{
645-
BlockNumber lo = 1;
646-
BlockNumber hi = HEADER_FOR_EXTERNAL_RETRIEVER.num_blocks;
647-
Buffer buf;
648-
Page page;
649-
HnswIndexPageSpecialBlock *special_block;
650-
HnswIndexTuple *nodepage;
651-
OffsetNumber offset, max_offset;
652-
if(id > HEADER_FOR_EXTERNAL_RETRIEVER.num_vectors) {
653-
elog(ERROR, "id %d is out of range(0, %d)", id, HEADER_FOR_EXTERNAL_RETRIEVER.num_vectors);
654-
}
655-
// static cnt = 0;
656-
657-
// elog(INFO,
658-
// "Hnsw_disk: ldb_wal_index_node_retriever called with id %ld %d %d", id,
659-
// cnt++, HEADER_FOR_EXTERNAL_RETRIEVER.num_blocks);
660-
661-
while(lo <= hi) {
662-
// elog(INFO, "binary searching for %ld %d %d", id, lo , hi);
663-
BlockNumber mid = (hi + lo) / 2;
664-
buf = ReadBufferExtended(INDEX_RELATION_FOR_RETRIEVER, MAIN_FORKNUM, mid, RBM_NORMAL, NULL);
665-
LockBuffer(buf, BUFFER_LOCK_SHARE);
666-
page = BufferGetPage(buf);
667-
special_block = (HnswIndexPageSpecialBlock *)PageGetSpecialPointer(page);
668-
669-
//clang-format off...ahh does not work
670-
if(!(special_block->firstId <= id && id <= special_block->lastId)) {
671-
if(special_block->firstId > id) {
672-
/* ---lo--------************mid*************------hi--- */
673-
/* --------id---[firstId-------------lastId]----------- */
674-
675-
hi = mid - 1;
676-
} else if(special_block->lastId < id) {
677-
/* ---lo--------************mid*************------hi--- */
678-
/* -------------[firstId-------------lastId]--id------- */
679-
lo = mid + 1;
680-
} else {
681-
elog(ERROR, "ERROR: should be unreachable");
682-
}
683-
UnlockReleaseBuffer(buf);
684-
continue;
685-
}
686-
/* ---lo--------************mid*************------hi--- */
687-
/* -------------[firstId-------id----lastId]----------- */
688-
689-
max_offset = PageGetMaxOffsetNumber(page);
690-
691-
for(offset = FirstOffsetNumber; offset <= max_offset; offset = OffsetNumberNext(offset)) {
692-
nodepage = (HnswIndexTuple *)PageGetItem(page, PageGetItemId(page, offset));
693-
if(nodepage->id == id) {
694-
#if LANTERNDB_COPYNODES
695-
if(wal_retriever_area == NULL || wal_retriever_area_offset + nodepage->size > wal_retriever_area_size) {
696-
elog(ERROR,
697-
"ERROR: wal_retriever_area "
698-
"is NULL or full");
699-
}
700-
memcpy(wal_retriever_area + wal_retriever_area_offset, nodepage->node, nodepage->size);
701-
wal_retriever_area_offset += nodepage->size;
702-
UnlockReleaseBuffer(buf);
703-
return wal_retriever_area + wal_retriever_area_offset - nodepage->size;
704-
#else
705-
UnlockReleaseBuffer(buf);
706-
elog(ERROR, "Nocopy is unimplemented for sequential index node retriever");
707-
#endif
708-
}
709-
}
710-
UnlockReleaseBuffer(buf);
711-
elog(ERROR, "AAAA found a candidate block but id %d was not in there", id);
712-
}
713-
elog(ERROR, "reached end of retriever without finding node %d", id);
714-
}
715-
716599
static inline void *wal_index_node_retriever_exact(int id)
717600
{
718601
HnswBlockmapPage *blockmap_page;
@@ -826,15 +709,15 @@ static inline void *wal_index_node_retriever_exact(int id)
826709
return wal_retriever_area + wal_retriever_area_offset - nodepage->size;
827710
#else
828711
if(!idx_page_prelocked) {
829-
if(takenbuffers[ takenbuffers_next ] != InvalidBuffer) {
830-
ReleaseBuffer(takenbuffers[ takenbuffers_next ]);
831-
takenbuffers[ takenbuffers_next ] = InvalidBuffer;
712+
if(takenbuffers[ takenbuffers_next ] != InvalidBuffer) {
713+
ReleaseBuffer(takenbuffers[ takenbuffers_next ]);
714+
takenbuffers[ takenbuffers_next ] = InvalidBuffer;
832715
}
833-
takenbuffers[ takenbuffers_next ] = buf;
834-
takenbuffers_next++;
716+
takenbuffers[ takenbuffers_next ] = buf;
717+
takenbuffers_next++;
835718

836-
if(takenbuffers_next == TAKENBUFFERS_MAX) {
837-
// if(takenbuffers[ 0 ] != InvalidBuffer) {
719+
if(takenbuffers_next == TAKENBUFFERS_MAX) {
720+
// if(takenbuffers[ 0 ] != InvalidBuffer) {
838721
// ReleaseBuffer(takenbuffers[ 0 ]);
839722
// takenbuffers[ 0 ] = InvalidBuffer;
840723
// }
@@ -983,4 +866,4 @@ static inline void fill_blockno_mapping()
983866
blockno = special_block->nextblockno;
984867
UnlockReleaseBuffer(buf);
985868
}
986-
}
869+
}

‎src/hnsw/utils.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
#include "utils.h"
44

55
#include "hnsw.h"
6-
#include "usearch.h"
76
#include "options.h"
7+
#include "usearch.h"
88

99
void LogUsearchOptions(usearch_init_options_t *opts)
1010
{

0 commit comments

Comments
 (0)