From 9ebb977651f8781d23e281fd0027087b59d8d512 Mon Sep 17 00:00:00 2001 From: Luke Gruber Date: Thu, 13 Aug 2026 10:27:14 -0400 Subject: [PATCH 01/20] Page pool: allow OS to free page and arena memory when only 1 objspace This patch allows the OS to free page and arena memory. This only happens in single objspace mode at the end of a major GC. There is still an empty_pages list per objspace, and this stays resident as before. Only pages previously munmapped pre-rlgc are madvised(DONTNEED)'d. We call these discarded pages. Once an arena (2MB) has all of its 32 pages (each 64K) discarded, we munmap the arena. At least 1 arena is kept mapped at all times. The following fields are added to GC.stat for the default GC: page_pool_arenas // number of mapped arenas page_pool_arenas_freed // number of freed (unmapped) arenas page_pool_total_pages // total pages in all arenas (arena_count * 32 currently) page_pool_discarded_pages // number of madvise(DONTNEED) pages (they can still be mapped) Future work: * Reduce the amount of empty pages per objspace or get rid of objspace->empty_pages altogether and keep a certain amount of resident pages in the pool for all objspaces. * Allow reclaiming memory when there are multiple Ractors. --- cont.c | 49 +------- gc/default/default.c | 280 ++++++++++++++++++++++++++++++++++++++----- internal/vm_map.h | 95 +++++++++++++++ 3 files changed, 346 insertions(+), 78 deletions(-) create mode 100644 internal/vm_map.h diff --git a/cont.c b/cont.c index a340542f1d6132..7d3aba3093d145 100644 --- a/cont.c +++ b/cont.c @@ -16,13 +16,6 @@ #include #endif -// On Solaris, madvise() is NOT declared for SUS (XPG4v2) or later, -// but MADV_* macros are defined when __EXTENSIONS__ is defined. -#ifdef NEED_MADVICE_PROTOTYPE_USING_CADDR_T -#include -extern int madvise(caddr_t, size_t, int); -#endif - #include COROUTINE_H #include "eval_intern.h" @@ -34,6 +27,7 @@ extern int madvise(caddr_t, size_t, int); #include "internal/gc.h" #include "internal/proc.h" #include "internal/sanitizers.h" +#include "internal/vm_map.h" #include "internal/warnings.h" #include "ruby/fiber/scheduler.h" #include "yjit.h" @@ -501,12 +495,7 @@ fiber_pool_allocate_memory(size_t * count, size_t stride) } else { ruby_annotate_mmap(base, mmap_size, "Ruby:fiber_pool_allocate_memory"); -#if defined(MADV_FREE_REUSE) - // On Mac MADV_FREE_REUSE is necessary for the task_info api - // to keep the accounting accurate as possible when a page is marked as reusable - // it can possibly not occurring at first call thus re-iterating if necessary. - while (madvise(base, mmap_size, MADV_FREE_REUSE) == -1 && errno == EAGAIN); -#endif + rb_vm_map_reuse(base, mmap_size); return base; } #endif @@ -820,7 +809,7 @@ fiber_pool_stack_acquire(struct fiber_pool * fiber_pool) } // We advise the operating system that the stack memory pages are no longer being used. -// This introduce some performance overhead but allows system to relaim memory when there is pressure. +// This introduces some performance overhead but allows the system to reclaim memory when there is pressure. static inline void fiber_pool_stack_free(struct fiber_pool_stack * stack) { @@ -843,37 +832,7 @@ fiber_pool_stack_free(struct fiber_pool_stack * stack) // In addition, it's actually slightly desirable to not do anything here, // but that results in higher memory usage. -#ifdef __wasi__ - // WebAssembly doesn't support madvise, so we just don't do anything. -#elif VM_CHECK_MODE > 0 && defined(MADV_DONTNEED) - if (!advice) advice = MADV_DONTNEED; - // This immediately discards the pages and the memory is reset to zero. - madvise(base, size, advice); -#elif defined(MADV_FREE_REUSABLE) - if (!advice) advice = MADV_FREE_REUSABLE; - // Darwin / macOS / iOS. - // Acknowledge the kernel down to the task info api we make this - // page reusable for future use. - // As for MADV_FREE_REUSABLE below we ensure in the rare occasions the task was not - // completed at the time of the call to re-iterate. - while (madvise(base, size, advice) == -1 && errno == EAGAIN); -#elif defined(MADV_FREE) - if (!advice) advice = MADV_FREE; - // Recent Linux. - madvise(base, size, advice); -#elif defined(MADV_DONTNEED) - if (!advice) advice = MADV_DONTNEED; - // Old Linux. - madvise(base, size, advice); -#elif defined(POSIX_MADV_DONTNEED) - if (!advice) advice = POSIX_MADV_DONTNEED; - // Solaris? - posix_madvise(base, size, advice); -#elif defined(_WIN32) - VirtualAlloc(base, size, MEM_RESET, PAGE_READWRITE); - // Not available in all versions of Windows. - //DiscardVirtualMemory(base, size); -#endif + rb_vm_map_reusable_lazy(base, size, advice); #if defined(COROUTINE_SANITIZE_ADDRESS) __asan_poison_memory_region(fiber_pool_stack_poison_base(stack), fiber_pool_stack_poison_size(stack)); diff --git a/gc/default/default.c b/gc/default/default.c index 3784ae6e32d6b2..1306deca7f567e 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -38,6 +38,7 @@ #include "gc/gc_impl.h" #include "yjit.h" #include "zjit.h" +#include "internal/vm_map.h" #ifdef BUILDING_MODULAR_GC /* hrtime.h transitively includes internal/time.h -> internal/bits.h, which are @@ -775,15 +776,24 @@ typedef struct rb_objspace { typedef struct rb_global_objspace { struct { rb_nativethread_lock_t lock; - struct heap_page_body *freelist; /* bodies to reuse; the next pointer lives in the body */ + struct heap_page_body *hot_list; /* ≤ PAGE_POOL_HOT_MAX un-advised bodies; link at body offset 0 */ + int hot_count; + size_t os_page_size; /* sysconf(_SC_PAGE_SIZE), cached at init */ /* List of mmap'd memory regions (arenas) for page bodies. */ struct page_arena { struct page_arena *next; - char *start; /* usable area, HEAP_PAGE_ALIGN aligned */ - size_t size; /* usable bytes (a multiple of HEAP_PAGE_SIZE) */ - } *arenas; /* every arena, newest first */ - char *arena_cursor; /* first body not yet carved out of the newest arena */ - char *arena_end; + char *start; /* usable area, HEAP_PAGE_ALIGN aligned */ + size_t size; /* usable bytes (a multiple of HEAP_PAGE_SIZE) */ + struct heap_page_body *cold_freelist; /* free bodies of this arena; link at body offset 0 */ + int free_count; /* bodies of this arena currently free (hot list + cold_freelist) */ + int cold_count; /* bodies on cold_freelist (subset of free_count) */ + } *arenas; /* every arena, newest first */ + char *arena_cursor; /* first body not yet carved out of the newest arena */ + char *arena_end; /* end of current arena */ + struct page_arena *arena_current; /* arena that arena_cursor carves from */ + int arena_count; /* current mapped arenas (for GC.stat total_pages) */ + int advised_count; /* page bodies with MADV_* applied (for GC.stat discarded pages) */ + size_t arenas_unmapped; /* cumulative arenas munmapped (for GC.stat) */ } page_pool; /* Zombie pages left after the last global cycle (roughly the live data). Updated @@ -840,8 +850,9 @@ static rb_global_objspace_t *global_objspace = NULL; static void objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src); -static struct heap_page_body *page_pool_acquire(void); -static void page_pool_release(struct heap_page_body *body); +static struct heap_page_body *page_pool_acquire(struct page_arena **arena_out); +static void page_pool_release(struct heap_page_body *body, struct page_arena *arena); +static void page_pool_reclaim(rb_global_objspace_t *g); static void global_objspace_init(void) @@ -849,15 +860,24 @@ global_objspace_init(void) if (global_objspace == NULL) { rb_global_objspace_t *g = &rb_global_objspace_instance; rb_native_mutex_initialize(&g->page_pool.lock); - g->page_pool.freelist = NULL; + g->page_pool.hot_list = NULL; + g->page_pool.hot_count = 0; g->page_pool.arenas = NULL; g->page_pool.arena_cursor = NULL; g->page_pool.arena_end = NULL; + g->page_pool.arena_current = NULL; + g->page_pool.arena_count = 0; + g->page_pool.advised_count = 0; + g->page_pool.arenas_unmapped = 0; +#ifdef HAVE_MMAP + g->page_pool.os_page_size = sysconf(_SC_PAGE_SIZE); +#else + g->page_pool.os_page_size = 0; +#endif global_objspace = g; } } - #ifndef HEAP_PAGE_ALIGN_LOG /* default tiny heap size: 64KiB */ #define HEAP_PAGE_ALIGN_LOG 16 @@ -1015,6 +1035,7 @@ struct heap_page { struct heap_page *free_next; struct heap_page_body *body; + struct page_arena *arena; struct ccan_list_node page_node; bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT]; @@ -2196,11 +2217,11 @@ gc_aligned_free(void *ptr, size_t size) } static void -heap_page_body_free(struct heap_page_body *page_body) +heap_page_body_free(struct heap_page_body *page_body, struct page_arena *arena) { GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0); - page_pool_release(page_body); + page_pool_release(page_body, arena); } /* Insert into page_index. Writers serialize on page_pool.lock; lomem and himem are a @@ -2262,7 +2283,7 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page) { global_page_index_remove(page); objspace->heap_pages.freed_pages++; - heap_page_body_free(page->body); + heap_page_body_free(page->body, page->arena); free(page); } @@ -2353,9 +2374,24 @@ gc_aligned_malloc(size_t alignment, size_t size) } /* The page pool (global_objspace->page_pool): heap page bodies are carved out of large - * arenas and reused through the pool's freelist. */ + * arenas and reused through the pool. Free bodies are split into a small global hot + * list (≤ PAGE_POOL_HOT_MAX, never madvise'd) and per-arena cold freelists (eligible for + * OS release — see page_pool_reclaim). Both lists use an in-body link at offset 0. */ #define PAGE_POOL_ARENA_SIZE (HEAP_PAGE_SIZE * 32) /* 2MiB with 64KiB pages */ +#define PAGE_POOL_ARENA_BODIES (PAGE_POOL_ARENA_SIZE / HEAP_PAGE_SIZE) /* 32 */ +#define PAGE_POOL_HOT_MAX 0 /* disabled — empty_pages is the retention buffer */ +#define PAGE_POOL_ARENA_KEEP_HALF (PAGE_POOL_ARENA_BODIES / 2) /* 16 */ + +/* Steal bit 0 of the in-body link word: set iff the body has been madvise'd (cold). */ +#define PAGE_POOL_ADVISED_BIT ((uintptr_t)1) + +/* While a body is free, the arena back-pointer is stored at offset sizeof(header) — one + * word past the link, inside the spared first OS page. PAGE_POOL_SCRATCH_SIZE covers + * both the link (offset 0) and the tag for ASAN unpoison. */ +#define PAGE_POOL_BODY_ARENA(body) \ + (*(struct page_arena **)((char *)(body) + sizeof(struct heap_page_header))) +#define PAGE_POOL_SCRATCH_SIZE (sizeof(struct heap_page_header) + sizeof(void *)) #ifdef HAVE_MMAP /* mmap a new arena to carve from. Called with the pool lock held, at which point the @@ -2413,8 +2449,13 @@ page_pool_add_arena(rb_global_objspace_t *g) } arena->start = aligned; arena->size = PAGE_POOL_ARENA_SIZE; + arena->cold_freelist = NULL; + arena->free_count = 0; + arena->cold_count = 0; arena->next = g->page_pool.arenas; g->page_pool.arenas = arena; + g->page_pool.arena_count++; + g->page_pool.arena_current = arena; g->page_pool.arena_cursor = aligned; g->page_pool.arena_end = aligned + PAGE_POOL_ARENA_SIZE; @@ -2424,42 +2465,72 @@ page_pool_add_arena(rb_global_objspace_t *g) #endif static struct heap_page_body * -page_pool_acquire(void) +page_pool_acquire(struct page_arena **arena_out) { struct heap_page_body *body = NULL; + bool need_reuse = false; if (HEAP_PAGE_ALLOC_USE_MMAP) { #ifdef HAVE_MMAP rb_global_objspace_t *g = global_objspace; rb_native_mutex_lock(&g->page_pool.lock); - if (g->page_pool.freelist != NULL) { - body = g->page_pool.freelist; - asan_unpoison_memory_region(body, sizeof(struct heap_page_body *), false); - g->page_pool.freelist = *(struct heap_page_body **)body; + if (g->page_pool.hot_list != NULL) { + body = g->page_pool.hot_list; + asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false); + uintptr_t link = *(uintptr_t *)body; + g->page_pool.hot_list = (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT); + g->page_pool.hot_count--; + struct page_arena *arena = PAGE_POOL_BODY_ARENA(body); + arena->free_count--; + *arena_out = arena; } - else if (g->page_pool.arena_cursor != g->page_pool.arena_end || - page_pool_add_arena(g)) { - GC_ASSERT(g->page_pool.arena_cursor + HEAP_PAGE_SIZE <= g->page_pool.arena_end); - body = (struct heap_page_body *)g->page_pool.arena_cursor; - g->page_pool.arena_cursor += HEAP_PAGE_SIZE; + else { + // find cold page body (madvised reusable) + for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) { + if (a->cold_count > 0) { + body = a->cold_freelist; + asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false); + uintptr_t link = *(uintptr_t *)body; + a->cold_freelist = (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT); + a->cold_count--; + a->free_count--; + *arena_out = a; + need_reuse = (link & PAGE_POOL_ADVISED_BIT) != 0; + if (need_reuse) g->page_pool.advised_count--; + break; + } + } + if (body == NULL && + (g->page_pool.arena_cursor != g->page_pool.arena_end || + page_pool_add_arena(g))) { + GC_ASSERT(g->page_pool.arena_cursor + HEAP_PAGE_SIZE <= g->page_pool.arena_end); + body = (struct heap_page_body *)g->page_pool.arena_cursor; + g->page_pool.arena_cursor += HEAP_PAGE_SIZE; + *arena_out = g->page_pool.arena_current; + } } rb_native_mutex_unlock(&g->page_pool.lock); if (body != NULL) { + if (need_reuse) { + rb_vm_map_reuse((char *)body + g->page_pool.os_page_size, + HEAP_PAGE_SIZE - g->page_pool.os_page_size); + } asan_unpoison_memory_region(body, HEAP_PAGE_SIZE, false); } #endif } else { body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE); + *arena_out = NULL; } return body; } static void -page_pool_release(struct heap_page_body *body) +page_pool_release(struct heap_page_body *body, struct page_arena *arena) { if (HEAP_PAGE_ALLOC_USE_MMAP) { #ifdef HAVE_MMAP @@ -2467,10 +2538,20 @@ page_pool_release(struct heap_page_body *body) rb_native_mutex_lock(&g->page_pool.lock); /* A body in the empty-pages pool stays fully poisoned (see gc_sweep_page), so - * unpoison its head before linking it into the pool freelist. */ - asan_unpoison_memory_region(body, sizeof(struct heap_page_body *), false); - *(struct heap_page_body **)body = g->page_pool.freelist; - g->page_pool.freelist = body; + * unpoison the scratch area (link + arena tag) before writing. */ + asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false); + arena->free_count++; + PAGE_POOL_BODY_ARENA(body) = arena; + if (g->page_pool.hot_count < PAGE_POOL_HOT_MAX) { + *(uintptr_t *)body = (uintptr_t)g->page_pool.hot_list; + g->page_pool.hot_list = body; + g->page_pool.hot_count++; + } + else { + *(uintptr_t *)body = (uintptr_t)arena->cold_freelist; + arena->cold_freelist = body; + arena->cold_count++; + } asan_poison_memory_region(body, HEAP_PAGE_SIZE); rb_native_mutex_unlock(&g->page_pool.lock); #endif @@ -2480,10 +2561,123 @@ page_pool_release(struct heap_page_body *body) } } +/* Allow the OS to reclaim pool memory. Runs only at major GC in single-objspace mode + * (see gc_sweep_finish). + * + * Step A: madvise cold bodies, sparing the first OS page (which holds the in-body + * freelist link and arena tag). + * + * Step B: munmap arenas whose 32 bodies are all free, keeping one extra empty + * arena as a retention buffer when the remaining free pool is < half an arena. */ +static void +page_pool_reclaim(rb_global_objspace_t *g) +{ + if (!HEAP_PAGE_ALLOC_USE_MMAP) return; +#ifdef HAVE_MMAP + size_t os_page_size = g->page_pool.os_page_size; + + rb_native_mutex_lock(&g->page_pool.lock); + + /* Step A — advise cold bodies (immediate release: drop RSS now if the platform allows). */ + if (os_page_size < HEAP_PAGE_SIZE) { + for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) { + for (struct heap_page_body *body = a->cold_freelist; body; ) { + asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false); + uintptr_t link = *(uintptr_t *)body; + struct heap_page_body *next = + (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT); + if (!(link & PAGE_POOL_ADVISED_BIT)) { + rb_vm_map_reusable_immediate((char *)body + os_page_size, + HEAP_PAGE_SIZE - os_page_size, 0); + *(uintptr_t *)body = link | PAGE_POOL_ADVISED_BIT; + g->page_pool.advised_count++; + } + asan_poison_memory_region(body, PAGE_POOL_SCRATCH_SIZE); + body = next; + } + } + } + + /* Step B — munmap fully-free arenas (with retention buffer). + * + * total_free = Σ free_count; free_count already includes hot-list bodies + * (page_pool_release increments it unconditionally), so no separate hot_count. + * An arena is eligible when all 32 of its bodies are free AND none sit on + * the hot list (≤5 entries, pre-scanned). Keep one extra empty arena when + * the rest of the free pool is < half an arena, to avoid thrash. */ + int total_free = 0; + for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) { + total_free += a->free_count; + } + + struct page_arena *hot_arenas[PAGE_POOL_HOT_MAX ? PAGE_POOL_HOT_MAX : 1]; + int n_hot_arenas = 0; + /* Collect arenas that have a hot body (≤ PAGE_POOL_HOT_MAX entries). */ + for (struct heap_page_body *body = g->page_pool.hot_list; body; ) { + asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false); + uintptr_t link = *(uintptr_t *)body; + struct heap_page_body *next = + (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT); + struct page_arena *arena = PAGE_POOL_BODY_ARENA(body); + bool found = false; + for (int i = 0; i < n_hot_arenas; i++) { + if (hot_arenas[i] == arena) { found = true; break; } + } + if (!found && n_hot_arenas < PAGE_POOL_HOT_MAX) { + hot_arenas[n_hot_arenas++] = arena; + } + asan_poison_memory_region(body, PAGE_POOL_SCRATCH_SIZE); + body = next; + } + + bool retained_one = false; + struct page_arena **pp = &g->page_pool.arenas; + // munmap fully free arenas + while (*pp) { + struct page_arena *a = *pp; + bool has_hot = false; + for (int i = 0; i < n_hot_arenas; i++) { + if (hot_arenas[i] == a) { has_hot = true; break; } + } + if (a->free_count != PAGE_POOL_ARENA_BODIES || has_hot) { + pp = &a->next; + continue; + } + GC_ASSERT(a->cold_count == PAGE_POOL_ARENA_BODIES); + + int free_elsewhere = total_free - PAGE_POOL_ARENA_BODIES; + if (free_elsewhere < PAGE_POOL_ARENA_KEEP_HALF && !retained_one) { + retained_one = true; + pp = &a->next; + continue; + } + + *pp = a->next; + if (munmap(a->start, a->size)) { + rb_bug("page_pool_reclaim: munmap failed"); + } + total_free -= PAGE_POOL_ARENA_BODIES; + g->page_pool.advised_count -= PAGE_POOL_ARENA_BODIES; + g->page_pool.arena_count--; + g->page_pool.arenas_unmapped++; + if (a == g->page_pool.arena_current) { + // During next acquire, any remaining arenas that have cold bodies are used. This is guaranteed + // because of the retention buffer. + g->page_pool.arena_current = NULL; + g->page_pool.arena_cursor = NULL; + g->page_pool.arena_end = NULL; + } + free(a); + } + + rb_native_mutex_unlock(&g->page_pool.lock); +#endif +} + static struct heap_page_body * -heap_page_body_allocate(void) +heap_page_body_allocate(struct page_arena **arena_out) { - struct heap_page_body *page_body = page_pool_acquire(); + struct heap_page_body *page_body = page_pool_acquire(arena_out); GC_ASSERT(page_body == NULL || (uintptr_t)page_body % HEAP_PAGE_ALIGN == 0); @@ -2514,14 +2708,15 @@ heap_page_resurrect(rb_objspace_t *objspace) static struct heap_page * heap_page_allocate(rb_objspace_t *objspace) { - struct heap_page_body *page_body = heap_page_body_allocate(); + struct page_arena *arena; + struct heap_page_body *page_body = heap_page_body_allocate(&arena); if (page_body == 0) { rb_memerror(); } struct heap_page *page = calloc1(sizeof(struct heap_page)); if (page == 0) { - heap_page_body_free(page_body); + heap_page_body_free(page_body, arena); rb_memerror(); } @@ -2552,6 +2747,7 @@ heap_page_allocate(rb_objspace_t *objspace) if (heap_pages_himem < end) heap_pages_himem = end; page->body = page_body; + page->arena = arena; page_body->header.page = page; page->objspace = objspace; @@ -4852,6 +5048,11 @@ gc_sweep_finish(rb_objspace_t *objspace) gc_prof_set_heap_info(objspace); heap_pages_free_unused_pages(objspace); + if (rb_gc_single_objspace_p() && is_full_marking(objspace)) { + /* gc_marks_finish retains ~2/3 of empty pages in objspace->empty_pages for reuse, + * only the excess reaches the pool. */ + page_pool_reclaim(global_objspace); + } for (int i = 0; i < HEAP_COUNT; i++) { rb_heap_t *heap = &heaps[i]; @@ -10020,6 +10221,10 @@ enum gc_stat_sym { gc_stat_sym_total_remembered_normal_object_count, gc_stat_sym_total_remembered_shady_object_count, #endif + gc_stat_sym_page_pool_arenas, + gc_stat_sym_page_pool_arenas_freed, + gc_stat_sym_page_pool_total_pages, + gc_stat_sym_page_pool_discarded_pages, gc_stat_sym_last }; @@ -10072,6 +10277,10 @@ setup_gc_stat_symbols(void) S(total_remembered_normal_object_count); S(total_remembered_shady_object_count); #endif /* RGENGC_PROFILE */ + S(page_pool_arenas); + S(page_pool_arenas_freed); + S(page_pool_total_pages); + S(page_pool_discarded_pages); #undef S } } @@ -10158,6 +10367,11 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) SET(heap_final_slots, total_final_slots_count(objspace)); SET(heap_marked_slots, objspace->marked_slots); + SET(page_pool_arenas, global_objspace->page_pool.arena_count); + SET(page_pool_arenas_freed, global_objspace->page_pool.arenas_unmapped); + SET(page_pool_total_pages, (size_t)global_objspace->page_pool.arena_count * PAGE_POOL_ARENA_BODIES); + SET(page_pool_discarded_pages, global_objspace->page_pool.advised_count); + #if RGENGC_PROFILE SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count); SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count); diff --git a/internal/vm_map.h b/internal/vm_map.h new file mode 100644 index 00000000000000..b1be510d0d3e52 --- /dev/null +++ b/internal/vm_map.h @@ -0,0 +1,95 @@ +#ifndef INTERNAL_VM_MAP_H +#define INTERNAL_VM_MAP_H +/** + * @author Ruby developers + * @copyright This file is a part of the programming language Ruby. + * Permission is hereby granted, to either redistribute and/or + * modify this file, provided that the conditions mentioned in the + * file COPYING are met. Consult the file for details. + * @brief Shared per-platform virtual memory mapping helpers. + */ +#include "ruby/internal/config.h" +#include /* for size_t */ + +#ifdef HAVE_SYS_MMAN_H +# include +#endif +#ifdef HAVE_ERRNO_H +# include /* for errno, EAGAIN */ +#endif + +// On Solaris, madvise() is NOT declared for SUS (XPG4v2) or later, +// but MADV_* macros are defined when __EXTENSIONS__ is defined. +#ifdef NEED_MADVICE_PROTOTYPE_USING_CADDR_T +#include +extern int madvise(caddr_t, size_t, int); +#endif + +// Reduces RSS lazily if possible, otherwise does it immediately. Keeps the mapping. +static inline void +rb_vm_map_reusable_lazy(void *addr, size_t len, int advice) +{ +#ifdef __wasi__ + /* WebAssembly doesn't support madvise. */ +#elif defined(VM_CHECK_MODE) && VM_CHECK_MODE > 0 && defined(MADV_DONTNEED) + if (!advice) advice = MADV_DONTNEED; + madvise(addr, len, advice); +#elif defined(MADV_FREE_REUSABLE) + /* Darwin / macOS / iOS. Drops phys_footprint; pages stay in resident_size + * until the kernel reclaims them. Retry on EAGAIN. */ + if (!advice) advice = MADV_FREE_REUSABLE; + while (madvise(addr, len, advice) == -1 && errno == EAGAIN); +#elif defined(MADV_FREE) + /* Recent Linux. */ + if (!advice) advice = MADV_FREE; + madvise(addr, len, advice); +#elif defined(MADV_DONTNEED) + /* Old Linux. */ + if (!advice) advice = MADV_DONTNEED; + madvise(addr, len, advice); +#elif defined(POSIX_MADV_DONTNEED) + /* Solaris. */ + if (!advice) advice = POSIX_MADV_DONTNEED; + posix_madvise(addr, len, advice); +#elif defined(_WIN32) + VirtualAlloc(addr, len, MEM_RESET, PAGE_READWRITE); +#endif +} + +// Reduces RSS immediately if possible, otherwise does it lazily. Keeps the mapping. +static inline void +rb_vm_map_reusable_immediate(void *addr, size_t len, int advice) +{ +#ifdef __wasi__ + /* WebAssembly doesn't support madvise. */ +#elif defined(MADV_FREE_REUSABLE) + /* The most aggressive option is MADV_FREE_REUSABLE — same as _lazy. It drops + * phys_footprint immediately. Retry on EAGAIN. */ + if (!advice) advice = MADV_FREE_REUSABLE; + while (madvise(addr, len, advice) == -1 && errno == EAGAIN); +#elif defined(MADV_DONTNEED) + /* Linux: kernel discards physical pages immediately; next touch + * zero-faults and RSS drops at once. */ + if (!advice) advice = MADV_DONTNEED; + madvise(addr, len, advice); +#elif defined(POSIX_MADV_DONTNEED) + if (!advice) advice = POSIX_MADV_DONTNEED; + posix_madvise(addr, len, advice); +#elif defined(_WIN32) + // Does not reduce working set immediately. We would need 2 system calls to do this. + VirtualAlloc(addr, len, MEM_RESET, PAGE_READWRITE); +#endif +} + +/* Re-acquire pages previously given back. Only needed for certain platforms. */ +static inline void +rb_vm_map_reuse(void *addr, size_t len) +{ +#if defined(MADV_FREE_REUSE) + /* Darwin: mandatory handshake with MADV_FREE_REUSABLE. Retry on EAGAIN. */ + while (madvise(addr, len, MADV_FREE_REUSE) == -1 && errno == EAGAIN); +#endif + /* Linux/Windows/wasi: no-op (fault re-materializes zero-filled pages). */ +} + +#endif /* INTERNAL_VM_MAP_H */ From 3c6636956100cac4bbbb86e35613cdd32be4951b Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 12:58:17 +0000 Subject: [PATCH 02/20] thread: wake the timer thread when an enqueued ractor has no server ractor_sched_enq only signalled ractor.sched.cond. The signal reaches a parked shared native thread, and a running one revisits the queue in ractor_sched_deq (under the same lock) before it can wait, so as long as any shared native thread exists the entry is served. With none -- each one dedicated to a blocking region or retired -- the signal is lost, and if the timer thread is also in its untimed sleep (timeslice_wait_inf), nothing serves the enqueued ractor: its runnable thread never runs. native_thread_check_and_create_shared, which would repair the shortage, runs on the timer thread's timeout branch, which is never reached again. Wake the timer thread under the same lock its timeout decision is made under, from the two transitions that can strand an entry: an enqueue that finds snt_cnt == 0, and the blocking-region entry that dedicates the last shared native thread away while the queue is non-empty. Both are off the ordinary path, and timer_thread_wakeup_locked writes to the communication pipe only when the timer thread sleeps untimed, so the common case costs nothing. [Bug #21504] Co-Authored-By: Claude Fable 5 --- bootstraptest/test_thread.rb | 41 ++++++++++++++++++++++++++++++++++++ thread_pthread.c | 15 +++++++++++++ 2 files changed, 56 insertions(+) diff --git a/bootstraptest/test_thread.rb b/bootstraptest/test_thread.rb index 2d846a0408f541..23bbfcadfdaf4f 100644 --- a/bootstraptest/test_thread.rb +++ b/bootstraptest/test_thread.rb @@ -571,3 +571,44 @@ def inspect got == 'x' ? 'ok' : got.inspect end.value } + +# A ractor made runnable while every shared native thread is dedicated to a +# blocking region must still be served: ractor_sched_enq has to wake the timer +# thread, whose untimed sleep otherwise never ends. [Bug #21504] +assert_equal 'ok', %q{ + lockpath = "mn_enq_wake_#{$$}.lock" + flagpath = "mn_enq_wake_#{$$}.flag" + begin + File.write(lockpath, "") + lock = File.open(lockpath, "r+") + lock.flock(File::LOCK_EX) + r = Ractor.new(lockpath, flagpath) do |lockpath, flagpath| + f = File.open(lockpath, "r+") + t = Thread.new do + sleep 0.2 # let the Ractor.receive below park first + # Stay off the scheduler until the timer thread is in its untimed sleep; + # blocking right away would be repaired by the pending 10ms timeout. + t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + nil while Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0 < 0.05 + f.flock(File::LOCK_EX) # the last shared native thread goes dedicated + end + msg = Ractor.receive + File.write(flagpath, "") + t.join + msg + end + sleep 1 # r is parked, its flock thread is dedicated, the timer sleeps untimed + r.send(:ok) + served = false + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5 + until served || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline + served = File.exist?(flagpath) + sleep 0.05 + end + lock.flock(File::LOCK_UN) + served ? r.value.to_s : 'the enqueued ractor was never served' + ensure + File.unlink(lockpath) rescue nil + File.unlink(flagpath) rescue nil + end +} diff --git a/thread_pthread.c b/thread_pthread.c index b473b7a46450d2..32bcd504b93ebd 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -1402,6 +1402,15 @@ ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r) rb_native_cond_signal(&vm->ractor.sched.cond); + // The signal reaches a parked snt, and a running one revisits the + // queue in ractor_sched_deq before it can wait (same lock as here). + // With every snt dedicated or retired, only the timer thread's + // timeout branch can serve the entry or widen the pool: wake it + // (a no-op unless it sleeps untimed). + if (vm->ractor.sched.snt_cnt == 0) { + timer_thread_wakeup_locked(vm); + } + // ractor_sched_dump(vm); } ractor_sched_unlock(vm, cr); @@ -1940,6 +1949,12 @@ native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_threa { vm->ractor.sched.snt_cnt--; vm->ractor.sched.dnt_cnt++; + + // This may have dedicated the last snt away from a pending + // entry whose enqueue saw snt_cnt > 0 (see ractor_sched_enq). + if (vm->ractor.sched.snt_cnt == 0 && vm->ractor.sched.grq_cnt > 0) { + timer_thread_wakeup_locked(vm); + } } ractor_sched_unlock(vm, cr); } From 6f176c892540131ce7d829483c9162ab31a4df08 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 13:07:17 +0000 Subject: [PATCH 03/20] thread: do not publish an M:N thread before its native thread exists native_thread_create_shared called thread_sched_to_ready before trying to widen the shared pool, so when pthread_create failed the creator's failure path (marking the thread killed, removing it from the living set, rb_ractor_cancel_creation) ran against a thread that an already parked shared native thread could pick up and run to its end concurrently -- including rb_ractor_postmortem_free of the very rb_thread_t the creator is still writing to. Observed under RLIMIT_NPROC exhaustion as ractor.c: rb_ractor_cancel_creation: r->status_ == ractor_blocking ractor.c: ractor_check_blocking: cr->threads.cnt >= cr->threads.blocking_cnt + 1 i.e. the living-set removal ran twice. Widen the pool first and make the thread ready only after that succeeded; the failure path then handles a thread that never became runnable, which is what it assumes. A never-started coroutine's stack and context are already freed by rb_threadptr_sched_free. Also roll snt_cnt back when native_thread_create0 fails, and free the allocated rb_native_thread. The count stayed raised before, so native_thread_check_and_create_shared concluded forever that the pool was wide enough and never tried again: permanent starvation instead of a retry on the next timer tick. Co-Authored-By: Claude Fable 5 --- bootstraptest/test_thread.rb | 48 ++++++++++++++++++++++++++++++++++++ thread_pthread_mn.c | 23 +++++++++++++---- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/bootstraptest/test_thread.rb b/bootstraptest/test_thread.rb index 23bbfcadfdaf4f..f57c46fcf34703 100644 --- a/bootstraptest/test_thread.rb +++ b/bootstraptest/test_thread.rb @@ -612,3 +612,51 @@ def inspect File.unlink(flagpath) rescue nil end } + +# Creating a thread when no native thread can be spawned must fail cleanly: +# the thread must not be published to the scheduler before its native thread +# exists, or an existing shared thread runs it to death concurrently with the +# creator's failure path (living-set removal races its own). +assert_equal 'ok', %q{ + can_limit = begin + Process.setrlimit(:NPROC, Process.getrlimit(:NPROC)[0]) + true + rescue StandardError, NotImplementedError + false + end + if !can_limit + 'ok' # cannot make thread creation fail on this platform; nothing to test + else + warm = 2.times.map { Ractor.new { nil until Ractor.receive == :quit } } + sleep 0.3 # the pool now has shared native threads parked for the warm ractors + Process.setrlimit(:NPROC, 1) + # RLIMIT_NPROC binds neither root (CI containers) nor macOS threads; + # probe that thread creation actually fails before asserting on it. + limited = begin + Thread.new {}.join + false + rescue ThreadError + true + end + result = + if !limited + 'ok' + else + errs = 0 + 20.times do + begin + Ractor.new { :born } + rescue ThreadError + errs += 1 + end + end + sleep 0.5 # a wrongly-published thread would be served and die about now + errs == 20 ? 'ok' : "#{errs} of 20 raised" + end + warm.each { |r| r.send(:quit) } + warm.each(&:value) + GC.start + result + end +} + diff --git a/thread_pthread_mn.c b/thread_pthread_mn.c index 3ac938d8229116..9d0f2728edbaac 100644 --- a/thread_pthread_mn.c +++ b/thread_pthread_mn.c @@ -458,7 +458,16 @@ native_thread_check_and_create_shared(rb_vm_t *vm) if (need_to_make) { struct rb_native_thread *nt = native_thread_alloc(); nt->vm = vm; - return native_thread_create0(nt); + int err = native_thread_create0(nt); + if (err) { + // Roll back, or this function would conclude forever that the + // pool is wide enough and never try again. + rb_native_mutex_lock(&vm->ractor.sched.lock); + vm->ractor.sched.snt_cnt--; + rb_native_mutex_unlock(&vm->ractor.sched.lock); + native_thread_destroy(nt); + } + return err; } else { return 0; @@ -619,11 +628,15 @@ native_thread_create_shared(rb_thread_t *th) tctx->co.argument = th; RUBY_DEBUG_LOG("th:%u vm_stack:%p machine_stack:%p", rb_th_serial(th), vm_stack, machine_stack); - thread_sched_to_ready(TH_SCHED(th), th); - // setup nt. th is runnable now and a Ractor's thread that runs to its end frees - // its own rb_thread_t (rb_ractor_postmortem_free), so th must not be read again. - return native_thread_check_and_create_shared(vm); + // Widen the pool before publishing th. Once ready, a Ractor's thread that + // runs to its end frees its own rb_thread_t (rb_ractor_postmortem_free), + // and the caller's create-failure path assumes th never became runnable. + int create_err = native_thread_check_and_create_shared(vm); + if (create_err) return create_err; + + thread_sched_to_ready(TH_SCHED(th), th); + return 0; } #else // USE_MN_THREADS From 4e4e3ded8b2014398d40d3b891ef492140f0023f Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 13:14:25 +0000 Subject: [PATCH 04/20] thread: leave RB_WAITFD_PRI waits to the blocking path waitfd_to_waiting_flag maps RB_WAITFD_PRI to no thread_sched_waiting_* event, so the M:N scheduler registered nothing for it yet parked the thread: a timeout-less IO#wait(IO::PRIORITY) on an M:N thread slept forever (a VM_ASSERT in thread_io_wait_events on debug builds). Exclude it from thread_io_mn_schedulable; poll(2) on the blocking path handles POLLPRI. The test lives in test/ruby/test_ractor.rb rather than bootstraptest because it needs the socket extension, which btest runs without. Co-Authored-By: Claude Fable 5 --- test/ruby/test_ractor.rb | 23 +++++++++++++++++++++++ thread.c | 6 +++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/test/ruby/test_ractor.rb b/test/ruby/test_ractor.rb index a25a24ce8ea069..1950d4cb966a23 100644 --- a/test/ruby/test_ractor.rb +++ b/test/ruby/test_ractor.rb @@ -771,4 +771,27 @@ def test_move_array_sharing_its_embedded_elements end RUBY end + + def test_io_priority_wait_on_mn_thread + omit 'POLLPRI/MSG_OOB semantics differ on windows' if RUBY_PLATFORM =~ /mswin|mingw/ + # A timeout-less IO#wait(IO::PRIORITY) on an M:N thread must take the + # blocking path: the M:N scheduler has no event for POLLPRI and used to + # register nothing yet park the thread forever. + assert_separately([], __FILE__, __LINE__, <<-'RUBY') + Warning[:experimental] = false + require 'socket' + r = Ractor.new do + serv = TCPServer.new("127.0.0.1", 0) + c = TCPSocket.new("127.0.0.1", serv.addr[1]) + s = serv.accept + t = Thread.new { c.wait(IO::PRIORITY, nil) } + sleep 0.5 + s.send("!", Socket::MSG_OOB) + woken = t.join(5) + [serv, c, s].each(&:close) + woken ? :ok : :timeout + end + assert_equal :ok, r.value + RUBY + end end diff --git a/thread.c b/thread.c index e90e344a0de395..f84836f315fb08 100644 --- a/thread.c +++ b/thread.c @@ -2055,7 +2055,11 @@ static bool thread_io_mn_schedulable(rb_thread_t *th, int events, const struct timeval *timeout) { #if defined(USE_MN_THREADS) && USE_MN_THREADS - return !th_has_dedicated_nt(th) && (events || timeout) && th->blocking; + // RB_WAITFD_PRI has no thread_sched_waiting_* event: the scheduler would + // register nothing and park the thread forever. POLLPRI works on the + // blocking path. + return !th_has_dedicated_nt(th) && (events || timeout) && th->blocking && + !(events & ~(RB_WAITFD_IN | RB_WAITFD_OUT)); #else return false; #endif From 78f2945bde25be566f93bbe369eeb39ed12d40ad Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 13:14:44 +0000 Subject: [PATCH 05/20] thread: do not clobber a sleeper's status in thread_sched_wait_events The M:N wait set th->status to THREAD_STOPPED_FOREVER around the park and to THREAD_RUNNABLE after it, unconditionally. A dedicated thread's native_cond_sleep leaves status alone: sleep_hrtime sets THREAD_STOPPED, re-sleeps while it stays so (SLEEP_SPURIOUS_CHECK), and only a waker sets THREAD_RUNNABLE. Rewriting it to THREAD_RUNNABLE made every spurious wakeup end the sleep: an interrupt deferred by Thread.handle_interrupt cut sleep(n) short on an M:N thread. Touch status only when the thread entered as THREAD_RUNNABLE (an io wait), which shows "sleep" while parked as a dedicated thread's blocking region does; a sleeper's status now belongs to its caller on both kinds of native thread. Co-Authored-By: Claude Fable 5 --- bootstraptest/test_thread.rb | 26 ++++++++++++++++++++++++++ thread_pthread_mn.c | 11 ++++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/bootstraptest/test_thread.rb b/bootstraptest/test_thread.rb index f57c46fcf34703..321247256d3980 100644 --- a/bootstraptest/test_thread.rb +++ b/bootstraptest/test_thread.rb @@ -660,3 +660,29 @@ def inspect end } +# An M:N thread's sleep must survive a spurious wakeup: an interrupt that +# handle_interrupt defers wakes the sleeper, whose status must stay +# THREAD_STOPPED so that sleep_hrtime sleeps the remaining time, as it does +# on a dedicated native thread. +assert_equal 'ok', %q{ + Ractor.new do + elapsed = nil + th = Thread.new do + Thread.handle_interrupt(RuntimeError => :never) do + t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + sleep 1.0 + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0 + end + end + sleep 0.3 + begin th.raise(RuntimeError, "deferred"); rescue RuntimeError; end + begin th.join; rescue RuntimeError; end + if elapsed.nil? + 'the sleeper died inside handle_interrupt :never' + elsif elapsed >= 0.9 + 'ok' + else + "slept only %.2fs of 1.0s" % elapsed + end + end.value +} diff --git a/thread_pthread_mn.c b/thread_pthread_mn.c index 9d0f2728edbaac..68e4fc237c8d65 100644 --- a/thread_pthread_mn.c +++ b/thread_pthread_mn.c @@ -108,9 +108,16 @@ thread_sched_wait_events(struct rb_thread_sched *sched, rb_thread_t *th, int fd, else { RUBY_DEBUG_LOG("sleep"); - th->status = THREAD_STOPPED_FOREVER; + // A sleeper's status belongs to the caller: sleep_hrtime + // re-sleeps while it stays THREAD_STOPPED and only a waker may + // change it, as with native_cond_sleep on a dedicated nt. An + // io wait enters as THREAD_RUNNABLE and shows "sleep" while + // parked, as a dedicated nt's blocking region does. + enum rb_thread_status prev_status = th->status; + if (prev_status == THREAD_RUNNABLE) th->status = THREAD_STOPPED_FOREVER; thread_sched_wakeup_next_thread(sched, th, true); thread_sched_wait_running_turn(sched, th, true); + if (prev_status == THREAD_RUNNABLE) th->status = THREAD_RUNNABLE; RUBY_DEBUG_LOG("wakeup"); } @@ -120,8 +127,6 @@ thread_sched_wait_events(struct rb_thread_sched *sched, rb_thread_t *th, int fd, if (need_cancel) { timer_thread_cancel_waiting(th); } - - th->status = THREAD_RUNNABLE; } else { // Ready right now, or not registerable at all -- only the former may From bd2b9be906e1433d2d882471001f3d7eda9c8458 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 13:14:59 +0000 Subject: [PATCH 06/20] thread: RB_NOGVL_UBF_ASYNC_SAFE does not apply to the sentinel ubfs rb_nogvl resolved RUBY_UBF_IO / RUBY_UBF_PROCESS to ubf_select before looking at RB_NOGVL_UBF_ASYNC_SAFE, so a caller combining the two had ubf_select invoked from the signal handler path (rb_thread_wakeup_timer_thread), where its ubf_list_lock and ractor scheduler lock can self-deadlock. No in-tree caller does this, but the API accepted it. Honour the flag only for a caller-provided ubf; the sentinels fall back to the ordinary deferred invocation. Co-Authored-By: Claude Fable 5 --- thread.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/thread.c b/thread.c index f84836f315fb08..43ea3dd2892eb6 100644 --- a/thread.c +++ b/thread.c @@ -1780,10 +1780,13 @@ rb_nogvl(void *(*func)(void *), void *data1, bool is_main_thread = vm->ractor.main_thread == th; int saved_errno = 0; - rb_thread_resolve_unblock_function(&ubf, &data2, th); + bool sentinel_ubf = rb_thread_resolve_unblock_function(&ubf, &data2, th); if (ubf && rb_ractor_living_thread_num(th->ractor) == 1 && is_main_thread) { - if (flags & RB_NOGVL_UBF_ASYNC_SAFE) { + // ubf_select, which the sentinel ubfs resolve to, takes ubf_list_lock + // and the ractor scheduler lock: not async-signal-safe, whatever the + // caller claims. + if ((flags & RB_NOGVL_UBF_ASYNC_SAFE) && !sentinel_ubf) { vm->ubf_async_safe = 1; } } From 33124c7c2e1167fafdd9b0c804a359e846edcfb3 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 13:21:55 +0000 Subject: [PATCH 07/20] thread: clean up the M:N scheduler's remaining fork residue In the forked child: - Do not pthread_mutex_destroy timer_th.waiting_lock. glibc returns EBUSY (which rb_native_mutex_destroy turns into rb_bug_errno) for a mutex that another ractor's M:N thread held at the fork moment. Reinitializing it, as the code already does, is the whole story. - Reinitialize nt_machine_stack_lock, whose holder at the fork moment does not exist in the child; the next machine-stack alloc or free would deadlock on it. - Forget timer_th.event_fd on the kqueue backend instead of leaving the stale number around. A kqueue is not inherited across fork, so it must not be closed either: the number may name a reused fd by then. - Reset dnt_cnt in thread_sched_atfork next to snt_cnt: only the forking thread's native thread survives. Its consumer is only a debug log, but the log lied after fork. Co-Authored-By: Claude Fable 5 --- thread_pthread.c | 18 ++++++++++++++++-- thread_pthread_mn.c | 7 +++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/thread_pthread.c b/thread_pthread.c index 32bcd504b93ebd..3500c2524ea2b7 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -333,6 +333,8 @@ static void timer_thread_wakeup_force(void); static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th); static void ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched); #if USE_MN_THREADS +static void nt_machine_stack_atfork(void); + // A coroutine thread's execution context: the coroutine_context (first, so // th->sched.context points at the whole block) plus what is needed to free // it without the rb_thread_t. Owned by the execution: the dying thread marks @@ -1738,9 +1740,11 @@ thread_sched_atfork(struct rb_thread_sched *sched) if (th_has_dedicated_nt(th)) { vm->ractor.sched.snt_cnt = 0; + vm->ractor.sched.dnt_cnt = 1; } else { vm->ractor.sched.snt_cnt = 1; + vm->ractor.sched.dnt_cnt = 0; } vm->ractor.sched.running_cnt = 0; @@ -1770,6 +1774,9 @@ thread_sched_atfork(struct rb_thread_sched *sched) ccan_list_head_init(&vm->ractor.sched.timeslice_threads); ccan_list_head_init(&vm->ractor.sched.running_threads); +#if USE_MN_THREADS + nt_machine_stack_atfork(); +#endif rb_internal_thread_event_hooks_rw_lock_atfork(); VM_ASSERT(sched->is_running); @@ -3351,8 +3358,15 @@ rb_thread_create_timer_thread(void) CLOSE_INVALIDATE_PAIR(timer_th.comm_fds); #if HAVE_SYS_EPOLL_H && USE_MN_THREADS close_invalidate(&timer_th.event_fd, "close event_fd"); -#endif - rb_native_mutex_destroy(&timer_th.waiting_lock); +#elif HAVE_SYS_EVENT_H && USE_MN_THREADS + // A kqueue is not inherited across fork: the number names a closed + // fd in the child, and closing it could hit a reused one. + timer_th.event_fd = -1; +#endif + // No mutex_destroy for waiting_lock: glibc returns EBUSY (and + // rb_native_mutex_destroy rb_bugs) for a mutex another ractor's + // M:N thread held at the fork moment. The initialize below + // starts over. } ccan_list_head_init(&timer_th.waiting); diff --git a/thread_pthread_mn.c b/thread_pthread_mn.c index 68e4fc237c8d65..debf7f2aa8ee67 100644 --- a/thread_pthread_mn.c +++ b/thread_pthread_mn.c @@ -196,6 +196,13 @@ struct nt_machine_stack_footer { static rb_nativethread_lock_t nt_machine_stack_lock = RB_NATIVETHREAD_LOCK_INIT; +// The holder at the fork moment does not exist in the child; start over. +static void +nt_machine_stack_atfork(void) +{ + rb_native_mutex_initialize(&nt_machine_stack_lock); +} + #include // vm_stack_size + machine_stack_size + 1 * (guard page size) From dce74ebf0454bdc935b1a66e816cb907995b7c85 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 13:28:55 +0000 Subject: [PATCH 08/20] thread: free a shared native thread's resources when it retires A retiring shared thread -- ractor_sched_deq giving itself back, or a thread rejoining a full pool (nt->retiring) -- just returned from nt_start, leaking its rb_native_thread: the struct, the coroutine context, the condvars and the sigaltstack. This was the old "how to free nt and nt->altstack?" TODO, and retiring made it recurring: every pool shrink leaked one. Both exits drop the nt from the counts before leaving the loop, so nothing can reference it; the exiting thread frees it itself, disarming its sigaltstack registration first so a late signal cannot run on the freed block. Verified with SNT_IDLE_RETIRE=0 (retire on the first idle wakeup): 168 retire/free pairs in a bursty ractor workload and a clean btest run. Co-Authored-By: Claude Fable 5 --- thread_pthread.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/thread_pthread.c b/thread_pthread.c index 3500c2524ea2b7..32778e9fc0da48 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -2055,6 +2055,20 @@ native_thread_destroy(struct rb_native_thread *nt) } } +// A retiring snt frees its own rb_native_thread as it exits. Disarm the +// altstack registration first: a signal between the free and the thread's +// end must not run on the freed block. +static void +native_thread_destroy_self(struct rb_native_thread *nt) +{ +#ifdef USE_SIGALTSTACK + stack_t disable = {0}; + disable.ss_flags = SS_DISABLE; + sigaltstack(&disable, NULL); +#endif + native_thread_destroy(nt); +} + #if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP #define STACKADDR_AVAILABLE 1 #elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP @@ -2440,6 +2454,8 @@ nt_start(void *ptr) coroutine_initialize_main(nt->nt_context); } + bool retired = false; + while (1) { if (nt->dedicated) { // wait running turn @@ -2464,7 +2480,10 @@ nt_start(void *ptr) } else { RUBY_DEBUG_LOG("check next"); - if (nt->retiring) break; // came back with no room in the shared pool + if (nt->retiring) { // came back with no room in the shared pool + retired = true; + break; + } rb_ractor_t *r = ractor_sched_deq(vm, NULL); @@ -2505,7 +2524,8 @@ nt_start(void *ptr) } } else { - // retired: this nt is done. + // ractor_sched_deq retired this nt. + retired = true; break; } @@ -2516,6 +2536,12 @@ nt_start(void *ptr) } } + if (retired) { + // The counts dropped this nt already; nothing can reference it now. + RUBY_DEBUG_LOG("retired nt:%u", nt->serial); + native_thread_destroy_self(nt); + } + return NULL; } From 582b3ef3507b7d4828e4bec26fc96ba6823c46b3 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 13:33:17 +0000 Subject: [PATCH 09/20] thread: harden the M:N scheduler's latent overflow and error paths None of these is reachable with default settings; each is a mine for a configuration or platform that steps on it. - timer_thread_set_timeout: a sleeper's deadline further away than INT_MAX ms truncated to a negative timeout, i.e. an untimed epoll_wait, and the sleeper was never woken. Clamp to INT_MAX. - nt_thread_stack_size: the size sum went through int, so stack sizes totalling over 2GB went negative (stack_count 0, and nt_alloc_stack retries mapping 512MB chunks forever). Compute in size_t. Also page-align the VM stack and machine stack areas: the configured sizes align only to 4KB, and on 16KB/64KB-page systems the guard page and the MAP_FIXED machine stack must land on page boundaries. - nt_alloc_stack: an mprotect/mmap failure had already consumed the slot (uninitialized_stack_count--); restore it. - nt_free_stack: prev_free_chunk == NULL cannot double as the "not on the free list" test -- the list's tail also has it NULL, and re-pushing the tail self-cycled the list. Track membership in its own flag. - ruby_stack_overflowed_p: the __APPLE__ branch runs when th == NULL yet read th->nt->thread_id; ask pthread_self() instead. - rb_thread_create_timer_thread: rb_bug on pthread_create failure instead of silently running without a timer thread. Co-Authored-By: Claude Fable 5 --- thread_pthread.c | 16 ++++++++++++---- thread_pthread_mn.c | 41 +++++++++++++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/thread_pthread.c b/thread_pthread.c index 32778e9fc0da48..5bf4c04040d864 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -3150,8 +3150,10 @@ timer_thread_set_timeout(rb_vm_t *vm) RUBY_DEBUG_LOG("th:%u now:%lu rel:%lu", rb_th_serial(th), (unsigned long)now, (unsigned long)hrrel); - // TODO: overflow? - int thread_timeout = (int)((hrrel + RB_HRTIME_PER_MSEC - 1) / RB_HRTIME_PER_MSEC); // ms + rb_hrtime_t msec = (hrrel + RB_HRTIME_PER_MSEC - 1) / RB_HRTIME_PER_MSEC; + // A deadline further away than INT_MAX ms must clamp, not truncate: + // a negative timeout would be an untimed epoll_wait. + int thread_timeout = msec > INT_MAX ? INT_MAX : (int)msec; // ms // Use minimum of scheduler timeout and thread sleep timeout if (timeout < 0 || thread_timeout < timeout) { @@ -3405,7 +3407,12 @@ rb_thread_create_timer_thread(void) timer_thread_setup_mn(); } - pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM()); + int err = pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM()); + if (err != 0) { + // The timer thread delivers signals and drives the M:N scheduler; + // running without one only defers the failure to stranger places. + rb_bug_errno("pthread_create (timer thread)", err); + } } static int @@ -3445,7 +3452,8 @@ ruby_stack_overflowed_p(const rb_thread_t *th, const void *addr) #ifdef STACKADDR_AVAILABLE else if (get_stack(&base, &size) == 0) { # ifdef __APPLE__ - if (pthread_equal(th->nt->thread_id, native_main_thread.id)) { + // th is NULL in this branch; ask about the calling thread itself. + if (pthread_equal(pthread_self(), native_main_thread.id)) { struct rlimit rlim; if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) { size = (size_t)rlim.rlim_cur; diff --git a/thread_pthread_mn.c b/thread_pthread_mn.c index debf7f2aa8ee67..a1ded57e08b8ff 100644 --- a/thread_pthread_mn.c +++ b/thread_pthread_mn.c @@ -179,6 +179,9 @@ get_sysconf_page_size(void) static struct nt_stack_chunk_header { struct nt_stack_chunk_header *prev_chunk; struct nt_stack_chunk_header *prev_free_chunk; + // prev_free_chunk == NULL cannot double as the membership test: the free + // list's tail also has it NULL, and re-pushing the tail self-cycles it. + bool on_free_list; uint16_t start_page; uint16_t stack_count; @@ -205,7 +208,23 @@ nt_machine_stack_atfork(void) #include -// vm_stack_size + machine_stack_size + 1 * (guard page size) +// Page-align the configured sizes: the layout puts a guard page and a +// MAP_FIXED machine stack behind the VM stack, and both must land on page +// boundaries whatever RUBY_THREAD_VM_STACK_SIZE and +// RUBY_THREAD_MACHINE_STACK_SIZE hold (those align only to 4KB). +static inline size_t +nt_vm_stack_area(const rb_vm_t *vm) +{ + return (size_t)roomof(vm->default_params.thread_vm_stack_size, MSTACK_PAGE_SIZE) * MSTACK_PAGE_SIZE; +} + +static inline size_t +nt_machine_stack_area(const rb_vm_t *vm) +{ + return (size_t)roomof(vm->default_params.thread_machine_stack_size, MSTACK_PAGE_SIZE) * MSTACK_PAGE_SIZE; +} + +// vm stack area + guard page + machine stack area static inline size_t nt_thread_stack_size(void) { @@ -213,9 +232,7 @@ nt_thread_stack_size(void) if (LIKELY(msz > 0)) return msz; rb_vm_t *vm = GET_VM(); - int sz = (int)(vm->default_params.thread_vm_stack_size + vm->default_params.thread_machine_stack_size + MSTACK_PAGE_SIZE); - int page_num = roomof(sz, MSTACK_PAGE_SIZE); - msz = (size_t)page_num * MSTACK_PAGE_SIZE; + msz = nt_vm_stack_area(vm) + MSTACK_PAGE_SIZE + nt_machine_stack_area(vm); return msz; } @@ -252,6 +269,7 @@ nt_alloc_thread_stack_chunk(void) ch->start_page = header_page_cnt; ch->prev_chunk = nt_stack_chunks; ch->prev_free_chunk = nt_free_stack_chunks; + ch->on_free_list = true; // the caller makes it the free-list head ch->uninitialized_stack_count = ch->stack_count = (uint16_t)stack_count; ch->free_stack_pos = 0; @@ -284,7 +302,7 @@ nt_stack_chunk_get_stack(const rb_vm_t *vm, struct nt_stack_chunk_header *ch, si const char *vstack, *mstack; const char *guard_page; vstack = nt_stack_chunk_get_stack_start(ch, idx); - guard_page = vstack + vm->default_params.thread_vm_stack_size; + guard_page = vstack + nt_vm_stack_area(vm); mstack = guard_page + MSTACK_PAGE_SIZE; struct nt_machine_stack_footer *msf = nt_stack_chunk_get_msf(vm, mstack); @@ -344,18 +362,19 @@ nt_alloc_stack(rb_vm_t *vm, void **vm_stack, void **machine_stack) // The chunk was mapped PROT_NONE; enable the VM stack and // machine stack pages, leaving the guard page as PROT_NONE. char *stack_start = nt_stack_chunk_get_stack_start(ch, idx); - size_t vm_stack_size = vm->default_params.thread_vm_stack_size; - size_t mstack_size = nt_thread_stack_size() - vm_stack_size - MSTACK_PAGE_SIZE; - char *mstack_start = stack_start + vm_stack_size + MSTACK_PAGE_SIZE; + size_t vm_stack_area = nt_vm_stack_area(vm); + size_t mstack_size = nt_thread_stack_size() - vm_stack_area - MSTACK_PAGE_SIZE; + char *mstack_start = stack_start + vm_stack_area + MSTACK_PAGE_SIZE; int mstack_flags = MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE; #if defined(MAP_STACK) && !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__) mstack_flags |= MAP_STACK; #endif - if (mprotect(stack_start, vm_stack_size, PROT_READ | PROT_WRITE) != 0 || + if (mprotect(stack_start, vm_stack_area, PROT_READ | PROT_WRITE) != 0 || mmap(mstack_start, mstack_size, PROT_READ | PROT_WRITE, mstack_flags, -1, 0) == MAP_FAILED) { err = errno; + ch->uninitialized_stack_count++; // the slot was not consumed } else { nt_stack_chunk_get_stack(vm, ch, idx, vm_stack, machine_stack); @@ -364,6 +383,7 @@ nt_alloc_stack(rb_vm_t *vm, void **vm_stack, void **machine_stack) else { nt_free_stack_chunks = ch->prev_free_chunk; ch->prev_free_chunk = NULL; + ch->on_free_list = false; goto retry; } } @@ -421,7 +441,8 @@ nt_free_stack(void *mstack) RUBY_DEBUG_LOG("stack:%p mstack:%p ch:%p index:%d", stack, mstack, ch, idx); - if (ch->prev_free_chunk == NULL) { + if (!ch->on_free_list) { + ch->on_free_list = true; ch->prev_free_chunk = nt_free_stack_chunks; nt_free_stack_chunks = ch; } From f3456dbfdc7a9615bb7b67c06346ed5ac58c9442 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 13 Aug 2026 13:35:47 +0000 Subject: [PATCH 10/20] thread: M:N scheduler cleanups - verify_waiting_list compared w->data.timeout with itself, so the sorted-order assert was vacuously true and the waiting list's O(n) insert has been unverified even on debug builds. - timer_thread_register_waiting repeated the rel && *rel > 0 check that the branch right above it had just done. - Remove THREAD_BLOCKING_YIELD: no user, and it calls thread_sched_to_waiting with a signature that predates the yield_immediately parameter, so any new user fails to compile. - native_thread_check_and_create_shared took ractor.sched.lock through the raw mutex API, bypassing the lock-owner bookkeeping every other taker goes through (ractor_sched_lock). - The stack chunk layout comment claimed "0th page is Redzone"; the header has occupied the head pages since 407dd02c1b. Co-Authored-By: Claude Fable 5 --- thread_pthread.c | 22 ---------------------- thread_pthread_mn.c | 16 ++++++---------- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/thread_pthread.c b/thread_pthread.c index 5bf4c04040d864..79c696b4060bf6 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -3543,28 +3543,6 @@ ruby_ppoll(struct pollfd *fds, nfds_t nfds, # define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask)) #endif -/* - * Single CPU setups benefit from explicit sched_yield() before ppoll(), - * since threads may be too starved to enter the GVL waitqueue for - * us to detect contention. Instead, we want to kick other threads - * so they can run and possibly prevent us from entering slow paths - * in ppoll() or similar syscalls. - * - * Confirmed on FreeBSD 11.2 and Linux 4.19. - * [ruby-core:90417] [Bug #15398] - */ -#define THREAD_BLOCKING_YIELD(th) do { \ - const rb_thread_t *next_th; \ - struct rb_thread_sched *sched = TH_SCHED(th); \ - RB_VM_SAVE_MACHINE_CONTEXT(th); \ - thread_sched_to_waiting(sched, (th)); \ - next_th = sched->running; \ - rb_native_mutex_unlock(&sched->lock_); \ - native_thread_yield(); /* TODO: needed? */ \ - if (!next_th && rb_ractor_living_thread_num(th->ractor) > 1) { \ - native_thread_yield(); \ - } - static void native_sleep(rb_thread_t *th, rb_hrtime_t *rel) { diff --git a/thread_pthread_mn.c b/thread_pthread_mn.c index a1ded57e08b8ff..d8e8fb37caf5b0 100644 --- a/thread_pthread_mn.c +++ b/thread_pthread_mn.c @@ -167,7 +167,7 @@ get_sysconf_page_size(void) // 512MB chunk // 131,072 pages (> 65,536) -// 0th page is Redzone. Start from 1st page. +// Head pages hold the chunk header (see start_page); stacks follow. /* * <--> machine stack + vm stack @@ -460,7 +460,7 @@ native_thread_check_and_create_shared(rb_vm_t *vm) { bool need_to_make = false; - rb_native_mutex_lock(&vm->ractor.sched.lock); + ractor_sched_lock(vm, NULL); // NULL: the timer thread also calls this { unsigned int schedulable_ractor_cnt = vm->ractor.cnt; RUBY_ASSERT(schedulable_ractor_cnt >= 1); @@ -486,7 +486,7 @@ native_thread_check_and_create_shared(rb_vm_t *vm) RUBY_DEBUG_LOG("snt:%d ractor_cnt:%d", (int)vm->ractor.sched.snt_cnt, (int)vm->ractor.cnt); } } - rb_native_mutex_unlock(&vm->ractor.sched.lock); + ractor_sched_unlock(vm, NULL); if (need_to_make) { struct rb_native_thread *nt = native_thread_alloc(); @@ -495,9 +495,9 @@ native_thread_check_and_create_shared(rb_vm_t *vm) if (err) { // Roll back, or this function would conclude forever that the // pool is wide enough and never try again. - rb_native_mutex_lock(&vm->ractor.sched.lock); + ractor_sched_lock(vm, NULL); vm->ractor.sched.snt_cnt--; - rb_native_mutex_unlock(&vm->ractor.sched.lock); + ractor_sched_unlock(vm, NULL); native_thread_destroy(nt); } return err; @@ -899,7 +899,7 @@ verify_waiting_list(void) // fprintf(stderr, "verify_waiting_list th:%u abs:%lu\n", rb_th_serial(wth), (unsigned long)wth->sched.waiting_reason.data.timeout); if (prev_w) { rb_hrtime_t timeout = w->data.timeout; - rb_hrtime_t prev_timeout = w->data.timeout; + rb_hrtime_t prev_timeout = prev_w->data.timeout; VM_ASSERT(timeout == 0 || prev_timeout <= timeout); } prev_w = w; @@ -983,10 +983,6 @@ timer_thread_register_waiting(rb_thread_t *th, int fd, enum thread_sched_waiting } } - if (rel && *rel > 0) { - flags |= thread_sched_waiting_timeout; - } - if (flags & thread_sched_waiting_timeout) { VM_ASSERT(rel != NULL); abs = rb_hrtime_add(rb_hrtime_now(), *rel); From 477fd8a3b58e0eb73877f05988622341b234318c Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Fri, 14 Aug 2026 04:56:20 +0000 Subject: [PATCH 11/20] thread: let timed fd waits ride the M:N scheduler thread_io_wait() sent every fd wait with a timeout to the blocking region (ppoll on a dedicated native thread), even under M:N. A pattern like IO#wait_readable(0.5) -- WEBrick's keepalive loop, or any flag-polling wait -- therefore cost a native thread handoff per call: ~0.7ms to reacquire the running slot after each ppoll under load, serializing servers to a fraction of the 1:1 throughput. Register such waits with the timer thread instead, as fd+timeout combined waiting. That registration path had no caller until now, and its expiry path timer_thread_deq_wakeup() left the waiter on its fd's waiter list with the fd still armed; unregister it there the same way the cancel path does. A zero timeout keeps using ppoll: it is a plain probe and needs no parking. WEBrick c=256: MN 2.3k -> 6.0k rps (hello, now above 1:1's 5.7k), 0.37k -> 1.5k rps (50 memcached accesses per request). Co-Authored-By: Claude Fable 5 --- thread.c | 28 ++++++++++++++++++++-------- thread_pthread.c | 8 +++++++- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/thread.c b/thread.c index 43ea3dd2892eb6..fc1b8b00e377c9 100644 --- a/thread.c +++ b/thread.c @@ -2070,6 +2070,7 @@ thread_io_mn_schedulable(rb_thread_t *th, int events, const struct timeval *time enum io_wait_result { io_wait_ready, // the MN scheduler waited and the fd is ready + io_wait_timed_out, // the MN scheduler waited until the timeout expired io_wait_unhandled, // the MN scheduler did not wait; use the blocking path }; @@ -2101,8 +2102,7 @@ thread_io_wait_events(rb_thread_t *th, int fd, int events, const struct timeval case thread_sched_wait_event: return io_wait_ready; case thread_sched_wait_timeout: - // Let the caller re-examine the fd through the blocking path. - return io_wait_unhandled; + return io_wait_timed_out; case thread_sched_wait_unavailable: // Never waited: reporting "ready" here would fabricate readiness and // spin, so hand the wait back to the caller's blocking path. @@ -4852,7 +4852,7 @@ thread_io_wait(rb_thread_t *th, struct rb_io *io, int fd, int events, struct tim volatile int result = 0; nfds_t nfds; struct rb_io_blocking_operation blocking_operation; - enum ruby_tag_type state; + enum ruby_tag_type state = TAG_NONE; volatile int lerrno; RUBY_ASSERT(th); @@ -4860,16 +4860,28 @@ thread_io_wait(rb_thread_t *th, struct rb_io *io, int fd, int events, struct tim if (io) { blocking_operation.ec = ec; +COMPILER_WARNING_PUSH +#if RBIMPL_COMPILER_IS(GCC) +COMPILER_WARNING_IGNORED(-Wdangling-pointer) +#endif + // rb_io_blocking_operation_exit() below unlinks it on every path. rb_io_blocking_operation_enter(io, &blocking_operation); +COMPILER_WARNING_POP } - if (timeout == NULL && thread_io_wait_events(th, fd, events, NULL, false) == io_wait_ready) { - // fd is readable - state = 0; + // A zero timeout is a plain probe; ppoll answers it without parking. + bool mn_wait = timeout == NULL || timeout->tv_sec != 0 || timeout->tv_usec != 0; + + switch (mn_wait ? thread_io_wait_events(th, fd, events, timeout, false) : io_wait_unhandled) { + case io_wait_ready: fds[0].revents = events; errno = 0; - } - else { + break; + case io_wait_timed_out: + // revents stays 0, so the result below becomes 0 as with ppoll's timeout. + errno = 0; + break; + case io_wait_unhandled: EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { rb_hrtime_t *to, rel, end = 0; diff --git a/thread_pthread.c b/thread_pthread.c index 79c696b4060bf6..38135ce30b4a4b 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -3202,11 +3202,17 @@ timer_thread_deq_wakeup(rb_vm_t *vm, rb_hrtime_t now, uint32_t *event_serial) // delete from waiting list ccan_list_del_init(&w->node); + rb_thread_t *th = thread_sched_waiting_thread(w); + +#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS + // An fd+timeout waiter is also on its fd's waiter list; leave it there too. + timer_thread_unregister_waiting(th, w->data.fd, w->flags); +#endif + // setup result w->flags = thread_sched_waiting_none; w->data.result = 0; - rb_thread_t *th = thread_sched_waiting_thread(w); *event_serial = w->data.event_serial; return th; } From 75d53029986f6c95f3fd66d8b05c3222d63f21b2 Mon Sep 17 00:00:00 2001 From: Luke Gruber Date: Fri, 14 Aug 2026 12:04:37 -0400 Subject: [PATCH 12/20] Get main Ractor to not take VM lock during local GC marking when JIT is enabled It still needs to take the VM lock occasionally to protect certain global data structures during marking but it doesn't have to take the lock the entire time. The problem with a long-held VM lock is that it increases the chance that other Ractors will block on it (and increases the amount of time they will be blocked). Note: It still takes the VM lock around the sweep of every page but I'll work on that in a separate PR. --- gc.c | 11 +++++------ gc/default/default.c | 46 +++++++++++++++++++------------------------- iseq.c | 5 ++--- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/gc.c b/gc.c index a8aaff88c5f503..241951082293cd 100644 --- a/gc.c +++ b/gc.c @@ -3333,20 +3333,18 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) * live there. A non-main Ractor's local GC skips them; a global GC walks all. */ if (global_gc || objspace == vm->ractor.main_ractor->objspace) { /* Only the main Ractor can register at_exit/END procs (a non-main one gets an - * IsolationError), and end_procs is a lock-free linked list, so only main -- - * the thread that registers, or a stop-the-world global GC, walks it. */ + * IsolationError) so end_procs is a lock-free linked list */ MARK_CHECKPOINT("end_proc"); rb_mark_end_proc(); MARK_CHECKPOINT("vm"); - /* rb_vm_mark walks VM-global weak tables that other Ractors rewrite under the - * VM lock, so main's otherwise lock-free local GC takes a no-barrier VM lock - * for this stretch; under a global GC the barrier already protects it. */ + /* rb_vm_mark and the JIT root marks walk VM-global weak tables and shared singleton + * JIT state that other Ractors rewrite under the VM lock, so main's otherwise + * lock-free local GC takes the VM lock for this stretch */ const bool vm_mark_needs_lock = rb_multi_ractor_p() && !global_gc; unsigned int vm_mark_lock_lev = 0; if (vm_mark_needs_lock) vm_mark_lock_lev = RB_GC_VM_LOCK_NO_BARRIER(); rb_vm_mark(vm); - if (vm_mark_needs_lock) RB_GC_VM_UNLOCK_NO_BARRIER(vm_mark_lock_lev); if (global_gc) { /* Mark and pin the shareable REFs of in-flight (off-heap) move couriers, @@ -3376,6 +3374,7 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) rb_zjit_root_mark(); } #endif + if (vm_mark_needs_lock) RB_GC_VM_UNLOCK_NO_BARRIER(vm_mark_lock_lev); if (global_gc || rb_gc_single_objspace_p()) { MARK_CHECKPOINT("global_symbols"); diff --git a/gc/default/default.c b/gc/default/default.c index 1306deca7f567e..89468669853880 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -4693,12 +4693,11 @@ gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context } } - /* main's local GC is lock-free, but freeing a shareable object referenced from a - * VM-global weak table (rb_gc_obj_free_vm_weak_references: ci_table, fstring, symbol, - * cme) mutates that table, so wrap the page's free loop in a no-barrier VM lock. - * Under a global GC the barrier already protects those tables, and a compacting - * local GC holds the barrier VM lock from gc_enter, so this nests harmlessly. A - * non-main Ractor's local GC never frees such objects and does not take it. */ + /* main's local GC is lock-free, but freeing a dead object can mutate VM-global state + * that other Ractors rewrite under the VM lock: weak tables (rb_gc_obj_free_vm_weak_ + * references: ci_table, fstring, symbol, cme). (JIT iseq frees are not reached here: + * iseqs are born shareable and a local GC never frees shareable objects.) Wrap the + * page's free loop in a no-barrier VM lock (FIXME). */ const bool sweep_needs_vm_lock = objspace == global_objspace->main_objspace && rb_gc_multi_ractor_p() && !objspace->flags.during_global_gc; unsigned int sweep_lock_lev = 0; @@ -8543,18 +8542,18 @@ gc_clock_end(struct timespec *ts) } /* Whether a non-global local GC holds the no-barrier VM lock for its whole run. Main's - * ordinary local GC is lock-free; only its compaction or an enabled JIT holds it (see the - * comment in the function body). */ + * ordinary local GC is lock-free; only compaction holds it (see the comment in the + * function body). */ static inline bool gc_local_gc_holds_vm_lock(const rb_objspace_t *objspace) { - /* Main's local GC is lock-free at the gc_enter level (bounded no-barrier windows - * cover the VM-global weak tables; compaction takes its barrier lock separately). - * What DOES hold the lock for the whole GC is an enabled JIT: marking reaches - * rb_yjit_iseq_mark / rb_zjit_iseq_mark through shareable iseq payloads, which must - * exclude another Ractor's concurrent compile (rb_iseq_mark_and_move asserts it). */ + /* Main's local GC is lock-free at the gc_enter level. The VM-global roots and JIT + * root marks that need the VM lock take a bounded no-barrier window in rb_gc_mark_roots. + * (JIT iseq payload marks and frees are not reached during a local GC: iseqs are born + * shareable and a local GC never traverses or frees them.) Compaction takes its + * barrier lock separately (gc_enter handles it before this function runs). */ return objspace == global_objspace->main_objspace && - (objspace->flags.during_compacting || rb_yjit_enabled_p || rb_zjit_enabled_p); + objspace->flags.during_compacting; } static inline bool @@ -8562,20 +8561,15 @@ gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_ { /* A local GC runs on its owner thread and takes neither the VM lock nor a barrier: * containment makes the heap single-writer (only a stop-the-world global GC writes pages - * across objspaces). There are two exceptions. + * across objspaces). * - * - A global GC stops the world (VM lock + barrier). A GC has no safepoints and a - * thread only joins after gc_exit, so the barrier implicitly waits for every in-flight - * local GC. - * - Main objspace's local GC also walks VM-global roots (rb_vm_mark) that change under - * the VM lock, so it takes the lock without raising a barrier. Non-main objspaces run - * as they are. A thread waiting for the VM lock here joins a pending global barrier - * *before* starting its own GC, never in the middle of one. + * Main's local GC walks VM-global roots (rb_vm_mark) and JIT root marks that change under + * the VM lock but takes the lock in rb_gc_mark_roots rather than holding it for the full GC. * - * Hence a GC must never take the VM lock from inside itself: the waiter would join a - * pending barrier mid-collection and expose its half-collected heap to the global GC. - * Shared structures the GC paths touch use their own native mutexes (registered - * globals, generic fields) or the page-pool lock. */ + * NOTE: The GC must never take the barrier VM lock from inside itself: the waiter could + * join a pending barrier mid-collection and expose its half-collected heap to the global + * GC. A no-barrier lock is safe. Other shared structures the GC paths touch use their own + * native mutexes or the page-pool lock. */ *lock_lev = 0; RUBY_DTRACE_GC_HOOK(ENTER, event); diff --git a/iseq.c b/iseq.c index f7c0602bcd233f..fe5f06851cce47 100644 --- a/iseq.c +++ b/iseq.c @@ -423,9 +423,8 @@ rb_iseq_mark_and_move(rb_iseq_t *iseq, bool reference_updating) #if USE_YJIT || USE_ZJIT /* The JIT payload's critical section is the VM lock (racing other Ractors' - * compile/invalidate; yjit/zjit assert it). A lock-free local GC also reaches - * here, so take it without joining a barrier. mmtk marks on a GC worker with no - * EC, where the lock cannot be taken, nor needed: stop-the-world. */ + * compile/invalidate). Iseqs are born shareable, so a multi-Ractor local GC + * never traverses them. */ const bool jit_payload_lock_p = rb_gc_multi_objspace_p(); bool jit_payload_p = false; # if USE_YJIT From 70a0bfa820422b6cf83162d18276d0c17072ce46 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sat, 15 Aug 2026 01:35:58 +0900 Subject: [PATCH 13/20] [ruby/digest] Reapply "Add BLAKE3 support in Digest" This reverts commit https://github.com/ruby/digest/commit/0296b240dc17. https://github.com/ruby/digest/commit/49e16371f7 --- ext/digest/blake3/LICENSE_A2 | 202 ++++ ext/digest/blake3/LICENSE_A2LLVM | 219 +++++ ext/digest/blake3/LICENSE_CC0 | 121 +++ ext/digest/blake3/blake3.c | 651 +++++++++++++ ext/digest/blake3/blake3.h | 86 ++ ext/digest/blake3/blake3_avx2.c | 326 +++++++ ext/digest/blake3/blake3_avx512.c | 1388 +++++++++++++++++++++++++++ ext/digest/blake3/blake3_dispatch.c | 332 +++++++ ext/digest/blake3/blake3_impl.h | 333 +++++++ ext/digest/blake3/blake3_neon.c | 367 +++++++ ext/digest/blake3/blake3_portable.c | 160 +++ ext/digest/blake3/blake3_sse2.c | 566 +++++++++++ ext/digest/blake3/blake3_sse41.c | 560 +++++++++++ ext/digest/blake3/blake3init.c | 82 ++ ext/digest/blake3/extconf.rb | 102 ++ ext/digest/digest.gemspec | 3 + test/digest/test_digest.rb | 102 +- 17 files changed, 5599 insertions(+), 1 deletion(-) create mode 100644 ext/digest/blake3/LICENSE_A2 create mode 100644 ext/digest/blake3/LICENSE_A2LLVM create mode 100644 ext/digest/blake3/LICENSE_CC0 create mode 100644 ext/digest/blake3/blake3.c create mode 100644 ext/digest/blake3/blake3.h create mode 100644 ext/digest/blake3/blake3_avx2.c create mode 100644 ext/digest/blake3/blake3_avx512.c create mode 100644 ext/digest/blake3/blake3_dispatch.c create mode 100644 ext/digest/blake3/blake3_impl.h create mode 100644 ext/digest/blake3/blake3_neon.c create mode 100644 ext/digest/blake3/blake3_portable.c create mode 100644 ext/digest/blake3/blake3_sse2.c create mode 100644 ext/digest/blake3/blake3_sse41.c create mode 100644 ext/digest/blake3/blake3init.c create mode 100644 ext/digest/blake3/extconf.rb diff --git a/ext/digest/blake3/LICENSE_A2 b/ext/digest/blake3/LICENSE_A2 new file mode 100644 index 00000000000000..d512ca94d0ca6b --- /dev/null +++ b/ext/digest/blake3/LICENSE_A2 @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Jack O'Connor and Samuel Neves + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ext/digest/blake3/LICENSE_A2LLVM b/ext/digest/blake3/LICENSE_A2LLVM new file mode 100644 index 00000000000000..8d11bec0e86f88 --- /dev/null +++ b/ext/digest/blake3/LICENSE_A2LLVM @@ -0,0 +1,219 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Jack O'Connor and Samuel Neves + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/ext/digest/blake3/LICENSE_CC0 b/ext/digest/blake3/LICENSE_CC0 new file mode 100644 index 00000000000000..0e259d42c99674 --- /dev/null +++ b/ext/digest/blake3/LICENSE_CC0 @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/ext/digest/blake3/blake3.c b/ext/digest/blake3/blake3.c new file mode 100644 index 00000000000000..00f91f444922df --- /dev/null +++ b/ext/digest/blake3/blake3.c @@ -0,0 +1,651 @@ +#include +#include +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" + +const char *blake3_version(void) { return BLAKE3_VERSION_STRING; } + +INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], + uint8_t flags) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + self->buf_len = 0; + self->blocks_compressed = 0; + self->flags = flags; +} + +INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], + uint64_t chunk_counter) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = chunk_counter; + self->blocks_compressed = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + self->buf_len = 0; +} + +INLINE size_t chunk_state_len(const blake3_chunk_state *self) { + return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + + ((size_t)self->buf_len); +} + +INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, + const uint8_t *input, size_t input_len) { + size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); + if (take > input_len) { + take = input_len; + } + uint8_t *dest = self->buf + ((size_t)self->buf_len); + memcpy(dest, input, take); + self->buf_len += (uint8_t)take; + return take; +} + +INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { + if (self->blocks_compressed == 0) { + return CHUNK_START; + } else { + return 0; + } +} + +typedef struct { + uint32_t input_cv[8]; + uint64_t counter; + uint8_t block[BLAKE3_BLOCK_LEN]; + uint8_t block_len; + uint8_t flags; +} output_t; + +INLINE output_t make_output(const uint32_t input_cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + output_t ret; + memcpy(ret.input_cv, input_cv, 32); + memcpy(ret.block, block, BLAKE3_BLOCK_LEN); + ret.block_len = block_len; + ret.counter = counter; + ret.flags = flags; + return ret; +} + +// Chaining values within a given chunk (specifically the compress_in_place +// interface) are represented as words. This avoids unnecessary bytes<->words +// conversion overhead in the portable implementation. However, the hash_many +// interface handles both user input and parent node blocks, so it accepts +// bytes. For that reason, chaining values in the CV stack are represented as +// bytes. +INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { + uint32_t cv_words[8]; + memcpy(cv_words, self->input_cv, 32); + blake3_compress_in_place(cv_words, self->block, self->block_len, + self->counter, self->flags); + store_cv_words(cv, cv_words); +} + +INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, + size_t out_len) { + if (out_len == 0) { + return; + } + uint64_t output_block_counter = seek / 64; + size_t offset_within_block = seek % 64; + uint8_t wide_buf[64]; + if(offset_within_block) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + const size_t available_bytes = 64 - offset_within_block; + const size_t bytes = out_len > available_bytes ? available_bytes : out_len; + memcpy(out, wide_buf + offset_within_block, bytes); + out += bytes; + out_len -= bytes; + output_block_counter += 1; + } + if(out_len / 64) { + blake3_xof_many(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, out, out_len / 64); + } + output_block_counter += out_len / 64; + out += out_len & -64; + out_len -= out_len & -64; + if(out_len) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + memcpy(out, wide_buf, out_len); + } +} + +INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, + size_t input_len) { + if (self->buf_len > 0) { + size_t take = chunk_state_fill_buf(self, input, input_len); + input += take; + input_len -= take; + if (input_len > 0) { + blake3_compress_in_place( + self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + self->buf_len = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + } + } + + while (input_len > BLAKE3_BLOCK_LEN) { + blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, + self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + input += BLAKE3_BLOCK_LEN; + input_len -= BLAKE3_BLOCK_LEN; + } + + chunk_state_fill_buf(self, input, input_len); +} + +INLINE output_t chunk_state_output(const blake3_chunk_state *self) { + uint8_t block_flags = + self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; + return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, + block_flags); +} + +INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], + const uint32_t key[8], uint8_t flags) { + return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); +} + +// Given some input larger than one chunk, return the number of bytes that +// should go in the left subtree. This is the largest power-of-2 number of +// chunks that leaves at least 1 byte for the right subtree. +INLINE size_t left_subtree_len(size_t input_len) { + // Subtract 1 to reserve at least one byte for the right side. input_len + // should always be greater than BLAKE3_CHUNK_LEN. + size_t full_chunks = (input_len - 1) / BLAKE3_CHUNK_LEN; + return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time +// on a single thread. Write out the chunk chaining values and return the +// number of chunks hashed. These chunks are never the root and never empty; +// those cases use a different codepath. +INLINE size_t compress_chunks_parallel(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(0 < input_len); + assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); +#endif + + const uint8_t *chunks_array[MAX_SIMD_DEGREE]; + size_t input_position = 0; + size_t chunks_array_len = 0; + while (input_len - input_position >= BLAKE3_CHUNK_LEN) { + chunks_array[chunks_array_len] = &input[input_position]; + input_position += BLAKE3_CHUNK_LEN; + chunks_array_len += 1; + } + + blake3_hash_many(chunks_array, chunks_array_len, + BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, + true, flags, CHUNK_START, CHUNK_END, out); + + // Hash the remaining partial chunk, if there is one. Note that the empty + // chunk (meaning the empty message) is a different codepath. + if (input_len > input_position) { + uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, key, flags); + chunk_state.chunk_counter = counter; + chunk_state_update(&chunk_state, &input[input_position], + input_len - input_position); + output_t output = chunk_state_output(&chunk_state); + output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]); + return chunks_array_len + 1; + } else { + return chunks_array_len; + } +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time +// on a single thread. Write out the parent chaining values and return the +// number of parents hashed. (If there's an odd input chaining value left over, +// return it as an additional output.) These parents are never the root and +// never empty; those cases use a different codepath. +INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, + size_t num_chaining_values, + const uint32_t key[8], uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(2 <= num_chaining_values); + assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); +#endif + + const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; + size_t parents_array_len = 0; + while (num_chaining_values - (2 * parents_array_len) >= 2) { + parents_array[parents_array_len] = + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; + parents_array_len += 1; + } + + blake3_hash_many(parents_array, parents_array_len, 1, key, + 0, // Parents always use counter 0. + false, flags | PARENT, + 0, // Parents have no start flags. + 0, // Parents have no end flags. + out); + + // If there's an odd child left over, it becomes an output. + if (num_chaining_values > 2 * parents_array_len) { + memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], + BLAKE3_OUT_LEN); + return parents_array_len + 1; + } else { + return parents_array_len; + } +} + +// The wide helper function returns (writes out) an array of chaining values +// and returns the length of that array. The number of chaining values returned +// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, +// if the input is shorter than that many chunks. The reason for maintaining a +// wide array of chaining values going back up the tree, is to allow the +// implementation to hash as many parents in parallel as possible. +// +// As a special case when the SIMD degree is 1, this function will still return +// at least 2 outputs. This guarantees that this function doesn't perform the +// root compression. (If it did, it would use the wrong flags, and also we +// wouldn't be able to implement extendable output.) Note that this function is +// not used when the whole input is only 1 chunk long; that's a different +// codepath. +// +// Why not just have the caller split the input on the first update(), instead +// of implementing this special rule? Because we don't want to limit SIMD or +// multi-threading parallelism for that update(). +size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb) { + // Note that the single chunk case does *not* bump the SIMD degree up to 2 + // when it is 1. If this implementation adds multi-threading in the future, + // this gives us the option of multi-threading even the 2-chunk case, which + // can help performance on smaller platforms. + if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { + return compress_chunks_parallel(input, input_len, key, chunk_counter, flags, + out); + } + + // With more than simd_degree chunks, we need to recurse. Start by dividing + // the input into left and right subtrees. (Note that this is only optimal + // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree + // of 3 or something, we'll need a more complicated strategy.) + size_t left_input_len = left_subtree_len(input_len); + size_t right_input_len = input_len - left_input_len; + const uint8_t *right_input = &input[left_input_len]; + uint64_t right_chunk_counter = + chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); + + // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to + // account for the special case of returning 2 outputs when the SIMD degree + // is 1. + uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t degree = blake3_simd_degree(); + if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { + // The special case: We always use a degree of at least two, to make + // sure there are two outputs. Except, as noted above, at the chunk + // level, where we allow degree=1. (Note that the 1-chunk-input case is + // a different codepath.) + degree = 2; + } + uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; + + // Recurse! + size_t left_n = SIZE_MAX; + size_t right_n = SIZE_MAX; + +#if defined(BLAKE3_USE_TBB) + blake3_compress_subtree_wide_join_tbb( + key, flags, use_tbb, + // left-hand side + input, left_input_len, chunk_counter, cv_array, &left_n, + // right-hand side + right_input, right_input_len, right_chunk_counter, right_cvs, &right_n); +#else + left_n = blake3_compress_subtree_wide( + input, left_input_len, key, chunk_counter, flags, cv_array, use_tbb); + right_n = blake3_compress_subtree_wide(right_input, right_input_len, key, + right_chunk_counter, flags, right_cvs, + use_tbb); +#endif // BLAKE3_USE_TBB + + // The special case again. If simd_degree=1, then we'll have left_n=1 and + // right_n=1. Rather than compressing them into a single output, return + // them directly, to make sure we always have at least two outputs. + if (left_n == 1) { + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); + return 2; + } + + // Otherwise, do one layer of parent node compression. + size_t num_chaining_values = left_n + right_n; + return compress_parents_parallel(cv_array, num_chaining_values, key, flags, + out); +} + +// Hash a subtree with compress_subtree_wide(), and then condense the resulting +// list of chaining values down to a single parent node. Don't compress that +// last parent node, however. Instead, return its message bytes (the +// concatenated chaining values of its children). This is necessary when the +// first call to update() supplies a complete subtree, because the topmost +// parent node of that subtree could end up being the root. It's also necessary +// for extended output in the general case. +// +// As with compress_subtree_wide(), this function is not used on inputs of 1 +// chunk or less. That's a different codepath. +INLINE void +compress_subtree_to_parent_node(const uint8_t *input, size_t input_len, + const uint32_t key[8], uint64_t chunk_counter, + uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN], + bool use_tbb) { +#if defined(BLAKE3_TESTING) + assert(input_len > BLAKE3_CHUNK_LEN); +#endif + + uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, + chunk_counter, flags, cv_array, use_tbb); + assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); + // The following loop never executes when MAX_SIMD_DEGREE_OR_2 is 2, because + // as we just asserted, num_cvs will always be <=2 in that case. But GCC + // (particularly GCC 8.5) can't tell that it never executes, and if NDEBUG is + // set then it emits incorrect warnings here. We tried a few different + // hacks to silence these, but in the end our hacks just produced different + // warnings (see https://github.com/BLAKE3-team/BLAKE3/pull/380). Out of + // desperation, we ifdef out this entire loop when we know it's not needed. +#if MAX_SIMD_DEGREE_OR_2 > 2 + // If MAX_SIMD_DEGREE_OR_2 is greater than 2 and there's enough input, + // compress_subtree_wide() returns more than 2 chaining values. Condense + // them into 2 by forming parent nodes repeatedly. + uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; + while (num_cvs > 2) { + num_cvs = + compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); + memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); + } +#endif + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); +} + +INLINE void hasher_init_base(blake3_hasher *self, const uint32_t key[8], + uint8_t flags) { + memcpy(self->key, key, BLAKE3_KEY_LEN); + chunk_state_init(&self->chunk, key, flags); + self->cv_stack_len = 0; +} + +void blake3_hasher_init(blake3_hasher *self) { hasher_init_base(self, IV, 0); } + +void blake3_hasher_init_keyed(blake3_hasher *self, + const uint8_t key[BLAKE3_KEY_LEN]) { + uint32_t key_words[8]; + load_key_words(key, key_words); + hasher_init_base(self, key_words, KEYED_HASH); +} + +void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, + size_t context_len) { + blake3_hasher context_hasher; + hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT); + blake3_hasher_update(&context_hasher, context, context_len); + uint8_t context_key[BLAKE3_KEY_LEN]; + blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); + uint32_t context_key_words[8]; + load_key_words(context_key, context_key_words); + hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL); +} + +void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context) { + blake3_hasher_init_derive_key_raw(self, context, strlen(context)); +} + +// As described in hasher_push_cv() below, we do "lazy merging", delaying +// merges until right before the next CV is about to be added. This is +// different from the reference implementation. Another difference is that we +// aren't always merging 1 chunk at a time. Instead, each CV might represent +// any power-of-two number of chunks, as long as the smaller-above-larger stack +// order is maintained. Instead of the "count the trailing 0-bits" algorithm +// described in the spec, we use a "count the total number of 1-bits" variant +// that doesn't require us to retain the subtree size of the CV on top of the +// stack. The principle is the same: each CV that should remain in the stack is +// represented by a 1-bit in the total number of chunks (or bytes) so far. +INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { + size_t post_merge_stack_len = (size_t)popcnt(total_len); + while (self->cv_stack_len > post_merge_stack_len) { + uint8_t *parent_node = + &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; + output_t output = parent_output(parent_node, self->key, self->chunk.flags); + output_chaining_value(&output, parent_node); + self->cv_stack_len -= 1; + } +} + +// In reference_impl.rs, we merge the new CV with existing CVs from the stack +// before pushing it. We can do that because we know more input is coming, so +// we know none of the merges are root. +// +// This setting is different. We want to feed as much input as possible to +// compress_subtree_wide(), without setting aside anything for the chunk_state. +// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once +// as a single subtree, if at all possible. +// +// This leads to two problems: +// 1) This 64 KiB input might be the only call that ever gets made to update. +// In this case, the root node of the 64 KiB subtree would be the root node +// of the whole tree, and it would need to be ROOT finalized. We can't +// compress it until we know. +// 2) This 64 KiB input might complete a larger tree, whose root node is +// similarly going to be the root of the whole tree. For example, maybe +// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the +// node at the root of the 256 KiB subtree until we know how to finalize it. +// +// The second problem is solved with "lazy merging". That is, when we're about +// to add a CV to the stack, we don't merge it with anything first, as the +// reference impl does. Instead we do merges using the *previous* CV that was +// added, which is sitting on top of the stack, and we put the new CV +// (unmerged) on top of the stack afterwards. This guarantees that we never +// merge the root node until finalize(). +// +// Solving the first problem requires an additional tool, +// compress_subtree_to_parent_node(). That function always returns the top +// *two* chaining values of the subtree it's compressing. We then do lazy +// merging with each of them separately, so that the second CV will always +// remain unmerged. (That also helps us support extendable output when we're +// hashing an input all-at-once.) +INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], + uint64_t chunk_counter) { + hasher_merge_cv_stack(self, chunk_counter); + memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, + BLAKE3_OUT_LEN); + self->cv_stack_len += 1; +} + +INLINE void blake3_hasher_update_base(blake3_hasher *self, const void *input, + size_t input_len, bool use_tbb) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_update(&hasher, v.data(), v.size()); + if (input_len == 0) { + return; + } + + const uint8_t *input_bytes = (const uint8_t *)input; + + // If we have some partial chunk bytes in the internal chunk_state, we need + // to finish that chunk first. + if (chunk_state_len(&self->chunk) > 0) { + size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); + if (take > input_len) { + take = input_len; + } + chunk_state_update(&self->chunk, input_bytes, take); + input_bytes += take; + input_len -= take; + // If we've filled the current chunk and there's more coming, finalize this + // chunk and proceed. In this case we know it's not the root. + if (input_len > 0) { + output_t output = chunk_state_output(&self->chunk); + uint8_t chunk_cv[32]; + output_chaining_value(&output, chunk_cv); + hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); + chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); + } else { + return; + } + } + + // Now the chunk_state is clear, and we have more input. If there's more than + // a single chunk (so, definitely not the root chunk), hash the largest whole + // subtree we can, with the full benefits of SIMD (and maybe in the future, + // multi-threading) parallelism. Two restrictions: + // - The subtree has to be a power-of-2 number of chunks. Only subtrees along + // the right edge can be incomplete, and we don't know where the right edge + // is going to be until we get to finalize(). + // - The subtree must evenly divide the total number of chunks up until this + // point (if total is not 0). If the current incomplete subtree is only + // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have + // to complete the current subtree first. + // Because we might need to break up the input to form powers of 2, or to + // evenly divide what we already have, this part runs in a loop. + while (input_len > BLAKE3_CHUNK_LEN) { + size_t subtree_len = round_down_to_power_of_2(input_len); + uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; + // Shrink the subtree_len until it evenly divides the count so far. We know + // that subtree_len itself is a power of 2, so we can use a bitmasking + // trick instead of an actual remainder operation. (Note that if the caller + // consistently passes power-of-2 inputs of the same size, as is hopefully + // typical, this loop condition will always fail, and subtree_len will + // always be the full length of the input.) + // + // An aside: We don't have to shrink subtree_len quite this much. For + // example, if count_so_far is 1, we could pass 2 chunks to + // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still + // get the right answer in the end, and we might get to use 2-way SIMD + // parallelism. The problem with this optimization, is that it gets us + // stuck always hashing 2 chunks. The total number of chunks will remain + // odd, and we'll never graduate to higher degrees of parallelism. See + // https://github.com/BLAKE3-team/BLAKE3/issues/69. + while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { + subtree_len /= 2; + } + // The shrunken subtree_len might now be 1 chunk long. If so, hash that one + // chunk by itself. Otherwise, compress the subtree into a pair of CVs. + uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; + if (subtree_len <= BLAKE3_CHUNK_LEN) { + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, self->key, self->chunk.flags); + chunk_state.chunk_counter = self->chunk.chunk_counter; + chunk_state_update(&chunk_state, input_bytes, subtree_len); + output_t output = chunk_state_output(&chunk_state); + uint8_t cv[BLAKE3_OUT_LEN]; + output_chaining_value(&output, cv); + hasher_push_cv(self, cv, chunk_state.chunk_counter); + } else { + // This is the high-performance happy path, though getting here depends + // on the caller giving us a long enough input. + uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; + compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, + self->chunk.chunk_counter, + self->chunk.flags, cv_pair, use_tbb); + hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); + hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], + self->chunk.chunk_counter + (subtree_chunks / 2)); + } + self->chunk.chunk_counter += subtree_chunks; + input_bytes += subtree_len; + input_len -= subtree_len; + } + + // If there's any remaining input less than a full chunk, add it to the chunk + // state. In that case, also do a final merge loop to make sure the subtree + // stack doesn't contain any unmerged pairs. The remaining input means we + // know these merges are non-root. This merge loop isn't strictly necessary + // here, because hasher_push_chunk_cv already does its own merge loop, but it + // simplifies blake3_hasher_finalize below. + if (input_len > 0) { + chunk_state_update(&self->chunk, input_bytes, input_len); + hasher_merge_cv_stack(self, self->chunk.chunk_counter); + } +} + +void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = false; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} + +#if defined(BLAKE3_USE_TBB) +void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = true; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} +#endif // BLAKE3_USE_TBB + +void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, + size_t out_len) { + blake3_hasher_finalize_seek(self, 0, out, out_len); +} + +void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, + uint8_t *out, size_t out_len) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_finalize(&hasher, v.data(), v.size()); + if (out_len == 0) { + return; + } + + // If the subtree stack is empty, then the current chunk is the root. + if (self->cv_stack_len == 0) { + output_t output = chunk_state_output(&self->chunk); + output_root_bytes(&output, seek, out, out_len); + return; + } + // If there are any bytes in the chunk state, finalize that chunk and do a + // roll-up merge between that chunk hash and every subtree in the stack. In + // this case, the extra merge loop at the end of blake3_hasher_update + // guarantees that none of the subtrees in the stack need to be merged with + // each other first. Otherwise, if there are no bytes in the chunk state, + // then the top of the stack is a chunk hash, and we start the merge from + // that. + output_t output; + size_t cvs_remaining; + if (chunk_state_len(&self->chunk) > 0) { + cvs_remaining = self->cv_stack_len; + output = chunk_state_output(&self->chunk); + } else { + // There are always at least 2 CVs in the stack in this case. + cvs_remaining = self->cv_stack_len - 2; + output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, + self->chunk.flags); + } + while (cvs_remaining > 0) { + cvs_remaining -= 1; + uint8_t parent_block[BLAKE3_BLOCK_LEN]; + memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); + output_chaining_value(&output, &parent_block[32]); + output = parent_output(parent_block, self->key, self->chunk.flags); + } + output_root_bytes(&output, seek, out, out_len); +} + +void blake3_hasher_reset(blake3_hasher *self) { + chunk_state_reset(&self->chunk, self->key, 0); + self->cv_stack_len = 0; +} diff --git a/ext/digest/blake3/blake3.h b/ext/digest/blake3/blake3.h new file mode 100644 index 00000000000000..423154ff75858d --- /dev/null +++ b/ext/digest/blake3/blake3.h @@ -0,0 +1,86 @@ +#ifndef BLAKE3_H +#define BLAKE3_H + +#include +#include + +#if !defined(BLAKE3_API) +# if defined(_WIN32) || defined(__CYGWIN__) +# if defined(BLAKE3_DLL) +# if defined(BLAKE3_DLL_EXPORTS) +# define BLAKE3_API __declspec(dllexport) +# else +# define BLAKE3_API __declspec(dllimport) +# endif +# define BLAKE3_PRIVATE +# else +# define BLAKE3_API +# define BLAKE3_PRIVATE +# endif +# elif __GNUC__ >= 4 +# define BLAKE3_API __attribute__((visibility("default"))) +# define BLAKE3_PRIVATE __attribute__((visibility("hidden"))) +# else +# define BLAKE3_API +# define BLAKE3_PRIVATE +# endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define BLAKE3_VERSION_STRING "1.8.5" +#define BLAKE3_KEY_LEN 32 +#define BLAKE3_OUT_LEN 32 +#define BLAKE3_BLOCK_LEN 64 +#define BLAKE3_CHUNK_LEN 1024 +#define BLAKE3_MAX_DEPTH 54 + +// This struct is a private implementation detail. It has to be here because +// it's part of the blake3_hasher structure defined below. +typedef struct { + uint32_t cv[8]; + uint64_t chunk_counter; + uint8_t buf[BLAKE3_BLOCK_LEN]; + uint8_t buf_len; + uint8_t blocks_compressed; + uint8_t flags; +} blake3_chunk_state; + +typedef struct { + uint32_t key[8]; + blake3_chunk_state chunk; + uint8_t cv_stack_len; + // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, + // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk + // requires a 4th entry, rather than merging everything down to 1, because we + // don't know whether more input is coming. This is different from how the + // reference implementation does things. + uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; +} blake3_hasher; + +BLAKE3_API const char *blake3_version(void); +BLAKE3_API void blake3_hasher_init(blake3_hasher *self); +BLAKE3_API void blake3_hasher_init_keyed(blake3_hasher *self, + const uint8_t key[BLAKE3_KEY_LEN]); +BLAKE3_API void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); +BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, + size_t context_len); +BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len); +#if defined(BLAKE3_USE_TBB) +BLAKE3_API void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len); +#endif // BLAKE3_USE_TBB +BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, + size_t out_len); +BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, + uint8_t *out, size_t out_len); +BLAKE3_API void blake3_hasher_reset(blake3_hasher *self); + +#ifdef __cplusplus +} +#endif + +#endif /* BLAKE3_H */ diff --git a/ext/digest/blake3/blake3_avx2.c b/ext/digest/blake3/blake3_avx2.c new file mode 100644 index 00000000000000..381e7c422f33c7 --- /dev/null +++ b/ext/digest/blake3/blake3_avx2.c @@ -0,0 +1,326 @@ +#include "blake3_impl.h" + +#include + +#define DEGREE 8 + +INLINE __m256i loadu(const uint8_t src[32]) { + return _mm256_loadu_si256((const __m256i *)src); +} + +INLINE void storeu(__m256i src, uint8_t dest[16]) { + _mm256_storeu_si256((__m256i *)dest, src); +} + +INLINE __m256i addv(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m256i xorv(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } + +INLINE __m256i set1(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } + +INLINE __m256i rot16(__m256i x) { + return _mm256_shuffle_epi8( + x, _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, + 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); +} + +INLINE __m256i rot12(__m256i x) { + return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12)); +} + +INLINE __m256i rot8(__m256i x) { + return _mm256_shuffle_epi8( + x, _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, + 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); +} + +INLINE __m256i rot7(__m256i x) { + return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7)); +} + +INLINE void round_fn(__m256i v[16], __m256i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m256i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high + // is 22/33/66/77. + __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); + __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); + __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); + __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); + __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); + __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); + __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); + __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); + + // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is + // 11/33. + __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); + __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); + __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); + __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); + __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); + __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); + __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); + __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); + + // Interleave 128-bit lanes. + vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); + vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); + vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); + vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); + vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); + vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); + vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); + vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m256i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m256i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m256i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m256i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m256i)]); + out[4] = loadu(&inputs[4][block_offset + 0 * sizeof(__m256i)]); + out[5] = loadu(&inputs[5][block_offset + 0 * sizeof(__m256i)]); + out[6] = loadu(&inputs[6][block_offset + 0 * sizeof(__m256i)]); + out[7] = loadu(&inputs[7][block_offset + 0 * sizeof(__m256i)]); + out[8] = loadu(&inputs[0][block_offset + 1 * sizeof(__m256i)]); + out[9] = loadu(&inputs[1][block_offset + 1 * sizeof(__m256i)]); + out[10] = loadu(&inputs[2][block_offset + 1 * sizeof(__m256i)]); + out[11] = loadu(&inputs[3][block_offset + 1 * sizeof(__m256i)]); + out[12] = loadu(&inputs[4][block_offset + 1 * sizeof(__m256i)]); + out[13] = loadu(&inputs[5][block_offset + 1 * sizeof(__m256i)]); + out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]); + out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]); + for (size_t i = 0; i < 8; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[8]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m256i *out_lo, __m256i *out_hi) { + const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter); + const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + const __m256i add1 = _mm256_and_si256(mask, add0); + __m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1); + __m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)), + _mm256_xor_si256( l, _mm256_set1_epi32(0x80000000))); + __m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m256i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m256i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m256i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m256i block_flags_vec = set1(block_flags); + __m256i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m256i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(h_vecs); + storeu(h_vecs[0], &out[0 * sizeof(__m256i)]); + storeu(h_vecs[1], &out[1 * sizeof(__m256i)]); + storeu(h_vecs[2], &out[2 * sizeof(__m256i)]); + storeu(h_vecs[3], &out[3 * sizeof(__m256i)]); + storeu(h_vecs[4], &out[4 * sizeof(__m256i)]); + storeu(h_vecs[5], &out[5 * sizeof(__m256i)]); + storeu(h_vecs[6], &out[6 * sizeof(__m256i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m256i)]); +} + +#if !defined(BLAKE3_NO_SSE41) +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#else +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif + +void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash8_avx2(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } +#if !defined(BLAKE3_NO_SSE41) + blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, out); +#else + blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); +#endif +} diff --git a/ext/digest/blake3/blake3_avx512.c b/ext/digest/blake3/blake3_avx512.c new file mode 100644 index 00000000000000..f88a32d312b106 --- /dev/null +++ b/ext/digest/blake3/blake3_avx512.c @@ -0,0 +1,1388 @@ +#include "blake3_impl.h" + +#include + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu_128(const uint8_t src[16]) { + return _mm_loadu_si128((void*)src); +} + +INLINE __m256i loadu_256(const uint8_t src[32]) { + return _mm256_loadu_si256((void*)src); +} + +INLINE __m512i loadu_512(const uint8_t src[64]) { + return _mm512_loadu_si512((void*)src); +} + +INLINE void storeu_128(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((void*)dest, src); +} + +INLINE void storeu_256(__m256i src, uint8_t dest[16]) { + _mm256_storeu_si256((void*)dest, src); +} + +INLINE void storeu_512(__m512i src, uint8_t dest[16]) { + _mm512_storeu_si512((void*)dest, src); +} + +INLINE __m128i add_128(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +INLINE __m256i add_256(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } + +INLINE __m512i add_512(__m512i a, __m512i b) { return _mm512_add_epi32(a, b); } + +INLINE __m128i xor_128(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m256i xor_256(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } + +INLINE __m512i xor_512(__m512i a, __m512i b) { return _mm512_xor_si512(a, b); } + +INLINE __m128i set1_128(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m256i set1_256(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } + +INLINE __m512i set1_512(uint32_t x) { return _mm512_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16_128(__m128i x) { return _mm_ror_epi32(x, 16); } + +INLINE __m256i rot16_256(__m256i x) { return _mm256_ror_epi32(x, 16); } + +INLINE __m512i rot16_512(__m512i x) { return _mm512_ror_epi32(x, 16); } + +INLINE __m128i rot12_128(__m128i x) { return _mm_ror_epi32(x, 12); } + +INLINE __m256i rot12_256(__m256i x) { return _mm256_ror_epi32(x, 12); } + +INLINE __m512i rot12_512(__m512i x) { return _mm512_ror_epi32(x, 12); } + +INLINE __m128i rot8_128(__m128i x) { return _mm_ror_epi32(x, 8); } + +INLINE __m256i rot8_256(__m256i x) { return _mm256_ror_epi32(x, 8); } + +INLINE __m512i rot8_512(__m512i x) { return _mm512_ror_epi32(x, 8); } + +INLINE __m128i rot7_128(__m128i x) { return _mm_ror_epi32(x, 7); } + +INLINE __m256i rot7_256(__m256i x) { return _mm256_ror_epi32(x, 7); } + +INLINE __m512i rot7_512(__m512i x) { return _mm512_ror_epi32(x, 7); } + +/* + * ---------------------------------------------------------------------------- + * compress_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = add_128(add_128(*row0, m), *row1); + *row3 = xor_128(*row3, *row0); + *row3 = rot16_128(*row3); + *row2 = add_128(*row2, *row3); + *row1 = xor_128(*row1, *row2); + *row1 = rot12_128(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = add_128(add_128(*row0, m), *row1); + *row3 = xor_128(*row3, *row0); + *row3 = rot8_128(*row3); + *row2 = add_128(*row2, *row3); + *row1 = xor_128(*row1, *row2); + *row1 = rot7_128(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu_128((uint8_t *)&cv[0]); + rows[1] = loadu_128((uint8_t *)&cv[4]); + rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu_128(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu_128(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu_128(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu_128(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_xof_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu_128(xor_128(rows[0], rows[2]), &out[0]); + storeu_128(xor_128(rows[1], rows[3]), &out[16]); + storeu_128(xor_128(rows[2], loadu_128((uint8_t *)&cv[0])), &out[32]); + storeu_128(xor_128(rows[3], loadu_128((uint8_t *)&cv[4])), &out[48]); +} + +void blake3_compress_in_place_avx512(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu_128(xor_128(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu_128(xor_128(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +/* + * ---------------------------------------------------------------------------- + * hash4_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn4(__m128i v[16], __m128i m[16], size_t r) { + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[15] = rot16_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot12_128(v[4]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[15] = rot8_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot7_128(v[4]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot16_128(v[15]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[4] = rot12_128(v[4]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot8_128(v[15]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + v[4] = rot7_128(v[4]); +} + +INLINE void transpose_vecs_128(__m128i vecs[4]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_128(&out[0]); + transpose_vecs_128(&out[4]); + transpose_vecs_128(&out[8]); + transpose_vecs_128(&out[12]); +} + +INLINE void load_counters4(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + uint64_t mask = (increment_counter ? ~0 : 0); + __m256i mask_vec = _mm256_set1_epi64x(mask); + __m256i deltas = _mm256_setr_epi64x(0, 1, 2, 3); + deltas = _mm256_and_si256(mask_vec, deltas); + __m256i counters = + _mm256_add_epi64(_mm256_set1_epi64x((int64_t)counter), deltas); + *out_lo = _mm256_cvtepi64_epi32(counters); + *out_hi = _mm256_cvtepi64_epi32(_mm256_srli_epi64(counters, 32)); +} + +static +void blake3_hash4_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), + set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters4(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1_128(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1_128(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn4(v, msg_vecs, 0); + round_fn4(v, msg_vecs, 1); + round_fn4(v, msg_vecs, 2); + round_fn4(v, msg_vecs, 3); + round_fn4(v, msg_vecs, 4); + round_fn4(v, msg_vecs, 5); + round_fn4(v, msg_vecs, 6); + h_vecs[0] = xor_128(v[0], v[8]); + h_vecs[1] = xor_128(v[1], v[9]); + h_vecs[2] = xor_128(v[2], v[10]); + h_vecs[3] = xor_128(v[3], v[11]); + h_vecs[4] = xor_128(v[4], v[12]); + h_vecs[5] = xor_128(v[5], v[13]); + h_vecs[6] = xor_128(v[6], v[14]); + h_vecs[7] = xor_128(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_128(&h_vecs[0]); + transpose_vecs_128(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu_128(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu_128(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu_128(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu_128(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu_128(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu_128(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu_128(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu_128(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +static +void blake3_xof4_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[4 * 64]) { + __m128i h_vecs[8] = { + set1_128(cv[0]), set1_128(cv[1]), set1_128(cv[2]), set1_128(cv[3]), + set1_128(cv[4]), set1_128(cv[5]), set1_128(cv[6]), set1_128(cv[7]), + }; + uint32_t block_words[16]; + load_block_words(block, block_words); + __m128i msg_vecs[16]; + for (size_t i = 0; i < 16; i++) { + msg_vecs[i] = set1_128(block_words[i]); + } + __m128i counter_low_vec, counter_high_vec; + load_counters4(counter, true, &counter_low_vec, &counter_high_vec); + __m128i block_len_vec = set1_128(block_len); + __m128i block_flags_vec = set1_128(flags); + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn4(v, msg_vecs, 0); + round_fn4(v, msg_vecs, 1); + round_fn4(v, msg_vecs, 2); + round_fn4(v, msg_vecs, 3); + round_fn4(v, msg_vecs, 4); + round_fn4(v, msg_vecs, 5); + round_fn4(v, msg_vecs, 6); + for (size_t i = 0; i < 8; i++) { + v[i] = xor_128(v[i], v[i+8]); + v[i+8] = xor_128(v[i+8], h_vecs[i]); + } + transpose_vecs_128(&v[0]); + transpose_vecs_128(&v[4]); + transpose_vecs_128(&v[8]); + transpose_vecs_128(&v[12]); + for (size_t i = 0; i < 4; i++) { + storeu_128(v[i+ 0], &out[(4*i+0) * sizeof(__m128i)]); + storeu_128(v[i+ 4], &out[(4*i+1) * sizeof(__m128i)]); + storeu_128(v[i+ 8], &out[(4*i+2) * sizeof(__m128i)]); + storeu_128(v[i+12], &out[(4*i+3) * sizeof(__m128i)]); + } +} + +/* + * ---------------------------------------------------------------------------- + * hash8_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn8(__m256i v[16], __m256i m[16], size_t r) { + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_256(v[0], v[4]); + v[1] = add_256(v[1], v[5]); + v[2] = add_256(v[2], v[6]); + v[3] = add_256(v[3], v[7]); + v[12] = xor_256(v[12], v[0]); + v[13] = xor_256(v[13], v[1]); + v[14] = xor_256(v[14], v[2]); + v[15] = xor_256(v[15], v[3]); + v[12] = rot16_256(v[12]); + v[13] = rot16_256(v[13]); + v[14] = rot16_256(v[14]); + v[15] = rot16_256(v[15]); + v[8] = add_256(v[8], v[12]); + v[9] = add_256(v[9], v[13]); + v[10] = add_256(v[10], v[14]); + v[11] = add_256(v[11], v[15]); + v[4] = xor_256(v[4], v[8]); + v[5] = xor_256(v[5], v[9]); + v[6] = xor_256(v[6], v[10]); + v[7] = xor_256(v[7], v[11]); + v[4] = rot12_256(v[4]); + v[5] = rot12_256(v[5]); + v[6] = rot12_256(v[6]); + v[7] = rot12_256(v[7]); + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_256(v[0], v[4]); + v[1] = add_256(v[1], v[5]); + v[2] = add_256(v[2], v[6]); + v[3] = add_256(v[3], v[7]); + v[12] = xor_256(v[12], v[0]); + v[13] = xor_256(v[13], v[1]); + v[14] = xor_256(v[14], v[2]); + v[15] = xor_256(v[15], v[3]); + v[12] = rot8_256(v[12]); + v[13] = rot8_256(v[13]); + v[14] = rot8_256(v[14]); + v[15] = rot8_256(v[15]); + v[8] = add_256(v[8], v[12]); + v[9] = add_256(v[9], v[13]); + v[10] = add_256(v[10], v[14]); + v[11] = add_256(v[11], v[15]); + v[4] = xor_256(v[4], v[8]); + v[5] = xor_256(v[5], v[9]); + v[6] = xor_256(v[6], v[10]); + v[7] = xor_256(v[7], v[11]); + v[4] = rot7_256(v[4]); + v[5] = rot7_256(v[5]); + v[6] = rot7_256(v[6]); + v[7] = rot7_256(v[7]); + + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_256(v[0], v[5]); + v[1] = add_256(v[1], v[6]); + v[2] = add_256(v[2], v[7]); + v[3] = add_256(v[3], v[4]); + v[15] = xor_256(v[15], v[0]); + v[12] = xor_256(v[12], v[1]); + v[13] = xor_256(v[13], v[2]); + v[14] = xor_256(v[14], v[3]); + v[15] = rot16_256(v[15]); + v[12] = rot16_256(v[12]); + v[13] = rot16_256(v[13]); + v[14] = rot16_256(v[14]); + v[10] = add_256(v[10], v[15]); + v[11] = add_256(v[11], v[12]); + v[8] = add_256(v[8], v[13]); + v[9] = add_256(v[9], v[14]); + v[5] = xor_256(v[5], v[10]); + v[6] = xor_256(v[6], v[11]); + v[7] = xor_256(v[7], v[8]); + v[4] = xor_256(v[4], v[9]); + v[5] = rot12_256(v[5]); + v[6] = rot12_256(v[6]); + v[7] = rot12_256(v[7]); + v[4] = rot12_256(v[4]); + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_256(v[0], v[5]); + v[1] = add_256(v[1], v[6]); + v[2] = add_256(v[2], v[7]); + v[3] = add_256(v[3], v[4]); + v[15] = xor_256(v[15], v[0]); + v[12] = xor_256(v[12], v[1]); + v[13] = xor_256(v[13], v[2]); + v[14] = xor_256(v[14], v[3]); + v[15] = rot8_256(v[15]); + v[12] = rot8_256(v[12]); + v[13] = rot8_256(v[13]); + v[14] = rot8_256(v[14]); + v[10] = add_256(v[10], v[15]); + v[11] = add_256(v[11], v[12]); + v[8] = add_256(v[8], v[13]); + v[9] = add_256(v[9], v[14]); + v[5] = xor_256(v[5], v[10]); + v[6] = xor_256(v[6], v[11]); + v[7] = xor_256(v[7], v[8]); + v[4] = xor_256(v[4], v[9]); + v[5] = rot7_256(v[5]); + v[6] = rot7_256(v[6]); + v[7] = rot7_256(v[7]); + v[4] = rot7_256(v[4]); +} + +INLINE void transpose_vecs_256(__m256i vecs[8]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high + // is 22/33/66/77. + __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); + __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); + __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); + __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); + __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); + __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); + __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); + __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); + + // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is + // 11/33. + __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); + __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); + __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); + __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); + __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); + __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); + __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); + __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); + + // Interleave 128-bit lanes. + vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); + vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); + vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); + vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); + vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); + vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); + vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); + vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); +} + +INLINE void transpose_msg_vecs8(const uint8_t *const *inputs, + size_t block_offset, __m256i out[16]) { + out[0] = loadu_256(&inputs[0][block_offset + 0 * sizeof(__m256i)]); + out[1] = loadu_256(&inputs[1][block_offset + 0 * sizeof(__m256i)]); + out[2] = loadu_256(&inputs[2][block_offset + 0 * sizeof(__m256i)]); + out[3] = loadu_256(&inputs[3][block_offset + 0 * sizeof(__m256i)]); + out[4] = loadu_256(&inputs[4][block_offset + 0 * sizeof(__m256i)]); + out[5] = loadu_256(&inputs[5][block_offset + 0 * sizeof(__m256i)]); + out[6] = loadu_256(&inputs[6][block_offset + 0 * sizeof(__m256i)]); + out[7] = loadu_256(&inputs[7][block_offset + 0 * sizeof(__m256i)]); + out[8] = loadu_256(&inputs[0][block_offset + 1 * sizeof(__m256i)]); + out[9] = loadu_256(&inputs[1][block_offset + 1 * sizeof(__m256i)]); + out[10] = loadu_256(&inputs[2][block_offset + 1 * sizeof(__m256i)]); + out[11] = loadu_256(&inputs[3][block_offset + 1 * sizeof(__m256i)]); + out[12] = loadu_256(&inputs[4][block_offset + 1 * sizeof(__m256i)]); + out[13] = loadu_256(&inputs[5][block_offset + 1 * sizeof(__m256i)]); + out[14] = loadu_256(&inputs[6][block_offset + 1 * sizeof(__m256i)]); + out[15] = loadu_256(&inputs[7][block_offset + 1 * sizeof(__m256i)]); + for (size_t i = 0; i < 8; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_256(&out[0]); + transpose_vecs_256(&out[8]); +} + +INLINE void load_counters8(uint64_t counter, bool increment_counter, + __m256i *out_lo, __m256i *out_hi) { + uint64_t mask = (increment_counter ? ~0 : 0); + __m512i mask_vec = _mm512_set1_epi64(mask); + __m512i deltas = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + deltas = _mm512_and_si512(mask_vec, deltas); + __m512i counters = + _mm512_add_epi64(_mm512_set1_epi64((int64_t)counter), deltas); + *out_lo = _mm512_cvtepi64_epi32(counters); + *out_hi = _mm512_cvtepi64_epi32(_mm512_srli_epi64(counters, 32)); +} + +static +void blake3_hash8_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m256i h_vecs[8] = { + set1_256(key[0]), set1_256(key[1]), set1_256(key[2]), set1_256(key[3]), + set1_256(key[4]), set1_256(key[5]), set1_256(key[6]), set1_256(key[7]), + }; + __m256i counter_low_vec, counter_high_vec; + load_counters8(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m256i block_len_vec = set1_256(BLAKE3_BLOCK_LEN); + __m256i block_flags_vec = set1_256(block_flags); + __m256i msg_vecs[16]; + transpose_msg_vecs8(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m256i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_256(IV[0]), set1_256(IV[1]), set1_256(IV[2]), set1_256(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn8(v, msg_vecs, 0); + round_fn8(v, msg_vecs, 1); + round_fn8(v, msg_vecs, 2); + round_fn8(v, msg_vecs, 3); + round_fn8(v, msg_vecs, 4); + round_fn8(v, msg_vecs, 5); + round_fn8(v, msg_vecs, 6); + h_vecs[0] = xor_256(v[0], v[8]); + h_vecs[1] = xor_256(v[1], v[9]); + h_vecs[2] = xor_256(v[2], v[10]); + h_vecs[3] = xor_256(v[3], v[11]); + h_vecs[4] = xor_256(v[4], v[12]); + h_vecs[5] = xor_256(v[5], v[13]); + h_vecs[6] = xor_256(v[6], v[14]); + h_vecs[7] = xor_256(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_256(h_vecs); + storeu_256(h_vecs[0], &out[0 * sizeof(__m256i)]); + storeu_256(h_vecs[1], &out[1 * sizeof(__m256i)]); + storeu_256(h_vecs[2], &out[2 * sizeof(__m256i)]); + storeu_256(h_vecs[3], &out[3 * sizeof(__m256i)]); + storeu_256(h_vecs[4], &out[4 * sizeof(__m256i)]); + storeu_256(h_vecs[5], &out[5 * sizeof(__m256i)]); + storeu_256(h_vecs[6], &out[6 * sizeof(__m256i)]); + storeu_256(h_vecs[7], &out[7 * sizeof(__m256i)]); +} + +static +void blake3_xof8_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[8 * 64]) { + __m256i h_vecs[8] = { + set1_256(cv[0]), set1_256(cv[1]), set1_256(cv[2]), set1_256(cv[3]), + set1_256(cv[4]), set1_256(cv[5]), set1_256(cv[6]), set1_256(cv[7]), + }; + uint32_t block_words[16]; + load_block_words(block, block_words); + __m256i msg_vecs[16]; + for (size_t i = 0; i < 16; i++) { + msg_vecs[i] = set1_256(block_words[i]); + } + __m256i counter_low_vec, counter_high_vec; + load_counters8(counter, true, &counter_low_vec, &counter_high_vec); + __m256i block_len_vec = set1_256(block_len); + __m256i block_flags_vec = set1_256(flags); + __m256i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_256(IV[0]), set1_256(IV[1]), set1_256(IV[2]), set1_256(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn8(v, msg_vecs, 0); + round_fn8(v, msg_vecs, 1); + round_fn8(v, msg_vecs, 2); + round_fn8(v, msg_vecs, 3); + round_fn8(v, msg_vecs, 4); + round_fn8(v, msg_vecs, 5); + round_fn8(v, msg_vecs, 6); + for (size_t i = 0; i < 8; i++) { + v[i] = xor_256(v[i], v[i+8]); + v[i+8] = xor_256(v[i+8], h_vecs[i]); + } + transpose_vecs_256(&v[0]); + transpose_vecs_256(&v[8]); + for (size_t i = 0; i < 8; i++) { + storeu_256(v[i+0], &out[(2*i+0) * sizeof(__m256i)]); + storeu_256(v[i+8], &out[(2*i+1) * sizeof(__m256i)]); + } +} + +/* + * ---------------------------------------------------------------------------- + * hash16_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn16(__m512i v[16], __m512i m[16], size_t r) { + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_512(v[0], v[4]); + v[1] = add_512(v[1], v[5]); + v[2] = add_512(v[2], v[6]); + v[3] = add_512(v[3], v[7]); + v[12] = xor_512(v[12], v[0]); + v[13] = xor_512(v[13], v[1]); + v[14] = xor_512(v[14], v[2]); + v[15] = xor_512(v[15], v[3]); + v[12] = rot16_512(v[12]); + v[13] = rot16_512(v[13]); + v[14] = rot16_512(v[14]); + v[15] = rot16_512(v[15]); + v[8] = add_512(v[8], v[12]); + v[9] = add_512(v[9], v[13]); + v[10] = add_512(v[10], v[14]); + v[11] = add_512(v[11], v[15]); + v[4] = xor_512(v[4], v[8]); + v[5] = xor_512(v[5], v[9]); + v[6] = xor_512(v[6], v[10]); + v[7] = xor_512(v[7], v[11]); + v[4] = rot12_512(v[4]); + v[5] = rot12_512(v[5]); + v[6] = rot12_512(v[6]); + v[7] = rot12_512(v[7]); + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_512(v[0], v[4]); + v[1] = add_512(v[1], v[5]); + v[2] = add_512(v[2], v[6]); + v[3] = add_512(v[3], v[7]); + v[12] = xor_512(v[12], v[0]); + v[13] = xor_512(v[13], v[1]); + v[14] = xor_512(v[14], v[2]); + v[15] = xor_512(v[15], v[3]); + v[12] = rot8_512(v[12]); + v[13] = rot8_512(v[13]); + v[14] = rot8_512(v[14]); + v[15] = rot8_512(v[15]); + v[8] = add_512(v[8], v[12]); + v[9] = add_512(v[9], v[13]); + v[10] = add_512(v[10], v[14]); + v[11] = add_512(v[11], v[15]); + v[4] = xor_512(v[4], v[8]); + v[5] = xor_512(v[5], v[9]); + v[6] = xor_512(v[6], v[10]); + v[7] = xor_512(v[7], v[11]); + v[4] = rot7_512(v[4]); + v[5] = rot7_512(v[5]); + v[6] = rot7_512(v[6]); + v[7] = rot7_512(v[7]); + + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_512(v[0], v[5]); + v[1] = add_512(v[1], v[6]); + v[2] = add_512(v[2], v[7]); + v[3] = add_512(v[3], v[4]); + v[15] = xor_512(v[15], v[0]); + v[12] = xor_512(v[12], v[1]); + v[13] = xor_512(v[13], v[2]); + v[14] = xor_512(v[14], v[3]); + v[15] = rot16_512(v[15]); + v[12] = rot16_512(v[12]); + v[13] = rot16_512(v[13]); + v[14] = rot16_512(v[14]); + v[10] = add_512(v[10], v[15]); + v[11] = add_512(v[11], v[12]); + v[8] = add_512(v[8], v[13]); + v[9] = add_512(v[9], v[14]); + v[5] = xor_512(v[5], v[10]); + v[6] = xor_512(v[6], v[11]); + v[7] = xor_512(v[7], v[8]); + v[4] = xor_512(v[4], v[9]); + v[5] = rot12_512(v[5]); + v[6] = rot12_512(v[6]); + v[7] = rot12_512(v[7]); + v[4] = rot12_512(v[4]); + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_512(v[0], v[5]); + v[1] = add_512(v[1], v[6]); + v[2] = add_512(v[2], v[7]); + v[3] = add_512(v[3], v[4]); + v[15] = xor_512(v[15], v[0]); + v[12] = xor_512(v[12], v[1]); + v[13] = xor_512(v[13], v[2]); + v[14] = xor_512(v[14], v[3]); + v[15] = rot8_512(v[15]); + v[12] = rot8_512(v[12]); + v[13] = rot8_512(v[13]); + v[14] = rot8_512(v[14]); + v[10] = add_512(v[10], v[15]); + v[11] = add_512(v[11], v[12]); + v[8] = add_512(v[8], v[13]); + v[9] = add_512(v[9], v[14]); + v[5] = xor_512(v[5], v[10]); + v[6] = xor_512(v[6], v[11]); + v[7] = xor_512(v[7], v[8]); + v[4] = xor_512(v[4], v[9]); + v[5] = rot7_512(v[5]); + v[6] = rot7_512(v[6]); + v[7] = rot7_512(v[7]); + v[4] = rot7_512(v[4]); +} + +// 0b10001000, or lanes a0/a2/b0/b2 in little-endian order +#define LO_IMM8 0x88 + +INLINE __m512i unpack_lo_128(__m512i a, __m512i b) { + return _mm512_shuffle_i32x4(a, b, LO_IMM8); +} + +// 0b11011101, or lanes a1/a3/b1/b3 in little-endian order +#define HI_IMM8 0xdd + +INLINE __m512i unpack_hi_128(__m512i a, __m512i b) { + return _mm512_shuffle_i32x4(a, b, HI_IMM8); +} + +INLINE void transpose_vecs_512(__m512i vecs[16]) { + // Interleave 32-bit lanes. The _0 unpack is lanes + // 0/0/1/1/4/4/5/5/8/8/9/9/12/12/13/13, and the _2 unpack is lanes + // 2/2/3/3/6/6/7/7/10/10/11/11/14/14/15/15. + __m512i ab_0 = _mm512_unpacklo_epi32(vecs[0], vecs[1]); + __m512i ab_2 = _mm512_unpackhi_epi32(vecs[0], vecs[1]); + __m512i cd_0 = _mm512_unpacklo_epi32(vecs[2], vecs[3]); + __m512i cd_2 = _mm512_unpackhi_epi32(vecs[2], vecs[3]); + __m512i ef_0 = _mm512_unpacklo_epi32(vecs[4], vecs[5]); + __m512i ef_2 = _mm512_unpackhi_epi32(vecs[4], vecs[5]); + __m512i gh_0 = _mm512_unpacklo_epi32(vecs[6], vecs[7]); + __m512i gh_2 = _mm512_unpackhi_epi32(vecs[6], vecs[7]); + __m512i ij_0 = _mm512_unpacklo_epi32(vecs[8], vecs[9]); + __m512i ij_2 = _mm512_unpackhi_epi32(vecs[8], vecs[9]); + __m512i kl_0 = _mm512_unpacklo_epi32(vecs[10], vecs[11]); + __m512i kl_2 = _mm512_unpackhi_epi32(vecs[10], vecs[11]); + __m512i mn_0 = _mm512_unpacklo_epi32(vecs[12], vecs[13]); + __m512i mn_2 = _mm512_unpackhi_epi32(vecs[12], vecs[13]); + __m512i op_0 = _mm512_unpacklo_epi32(vecs[14], vecs[15]); + __m512i op_2 = _mm512_unpackhi_epi32(vecs[14], vecs[15]); + + // Interleave 64-bit lanes. The _0 unpack is lanes + // 0/0/0/0/4/4/4/4/8/8/8/8/12/12/12/12, the _1 unpack is lanes + // 1/1/1/1/5/5/5/5/9/9/9/9/13/13/13/13, the _2 unpack is lanes + // 2/2/2/2/6/6/6/6/10/10/10/10/14/14/14/14, and the _3 unpack is lanes + // 3/3/3/3/7/7/7/7/11/11/11/11/15/15/15/15. + __m512i abcd_0 = _mm512_unpacklo_epi64(ab_0, cd_0); + __m512i abcd_1 = _mm512_unpackhi_epi64(ab_0, cd_0); + __m512i abcd_2 = _mm512_unpacklo_epi64(ab_2, cd_2); + __m512i abcd_3 = _mm512_unpackhi_epi64(ab_2, cd_2); + __m512i efgh_0 = _mm512_unpacklo_epi64(ef_0, gh_0); + __m512i efgh_1 = _mm512_unpackhi_epi64(ef_0, gh_0); + __m512i efgh_2 = _mm512_unpacklo_epi64(ef_2, gh_2); + __m512i efgh_3 = _mm512_unpackhi_epi64(ef_2, gh_2); + __m512i ijkl_0 = _mm512_unpacklo_epi64(ij_0, kl_0); + __m512i ijkl_1 = _mm512_unpackhi_epi64(ij_0, kl_0); + __m512i ijkl_2 = _mm512_unpacklo_epi64(ij_2, kl_2); + __m512i ijkl_3 = _mm512_unpackhi_epi64(ij_2, kl_2); + __m512i mnop_0 = _mm512_unpacklo_epi64(mn_0, op_0); + __m512i mnop_1 = _mm512_unpackhi_epi64(mn_0, op_0); + __m512i mnop_2 = _mm512_unpacklo_epi64(mn_2, op_2); + __m512i mnop_3 = _mm512_unpackhi_epi64(mn_2, op_2); + + // Interleave 128-bit lanes. The _0 unpack is + // 0/0/0/0/8/8/8/8/0/0/0/0/8/8/8/8, the _1 unpack is + // 1/1/1/1/9/9/9/9/1/1/1/1/9/9/9/9, and so on. + __m512i abcdefgh_0 = unpack_lo_128(abcd_0, efgh_0); + __m512i abcdefgh_1 = unpack_lo_128(abcd_1, efgh_1); + __m512i abcdefgh_2 = unpack_lo_128(abcd_2, efgh_2); + __m512i abcdefgh_3 = unpack_lo_128(abcd_3, efgh_3); + __m512i abcdefgh_4 = unpack_hi_128(abcd_0, efgh_0); + __m512i abcdefgh_5 = unpack_hi_128(abcd_1, efgh_1); + __m512i abcdefgh_6 = unpack_hi_128(abcd_2, efgh_2); + __m512i abcdefgh_7 = unpack_hi_128(abcd_3, efgh_3); + __m512i ijklmnop_0 = unpack_lo_128(ijkl_0, mnop_0); + __m512i ijklmnop_1 = unpack_lo_128(ijkl_1, mnop_1); + __m512i ijklmnop_2 = unpack_lo_128(ijkl_2, mnop_2); + __m512i ijklmnop_3 = unpack_lo_128(ijkl_3, mnop_3); + __m512i ijklmnop_4 = unpack_hi_128(ijkl_0, mnop_0); + __m512i ijklmnop_5 = unpack_hi_128(ijkl_1, mnop_1); + __m512i ijklmnop_6 = unpack_hi_128(ijkl_2, mnop_2); + __m512i ijklmnop_7 = unpack_hi_128(ijkl_3, mnop_3); + + // Interleave 128-bit lanes again for the final outputs. + vecs[0] = unpack_lo_128(abcdefgh_0, ijklmnop_0); + vecs[1] = unpack_lo_128(abcdefgh_1, ijklmnop_1); + vecs[2] = unpack_lo_128(abcdefgh_2, ijklmnop_2); + vecs[3] = unpack_lo_128(abcdefgh_3, ijklmnop_3); + vecs[4] = unpack_lo_128(abcdefgh_4, ijklmnop_4); + vecs[5] = unpack_lo_128(abcdefgh_5, ijklmnop_5); + vecs[6] = unpack_lo_128(abcdefgh_6, ijklmnop_6); + vecs[7] = unpack_lo_128(abcdefgh_7, ijklmnop_7); + vecs[8] = unpack_hi_128(abcdefgh_0, ijklmnop_0); + vecs[9] = unpack_hi_128(abcdefgh_1, ijklmnop_1); + vecs[10] = unpack_hi_128(abcdefgh_2, ijklmnop_2); + vecs[11] = unpack_hi_128(abcdefgh_3, ijklmnop_3); + vecs[12] = unpack_hi_128(abcdefgh_4, ijklmnop_4); + vecs[13] = unpack_hi_128(abcdefgh_5, ijklmnop_5); + vecs[14] = unpack_hi_128(abcdefgh_6, ijklmnop_6); + vecs[15] = unpack_hi_128(abcdefgh_7, ijklmnop_7); +} + +INLINE void transpose_msg_vecs16(const uint8_t *const *inputs, + size_t block_offset, __m512i out[16]) { + out[0] = loadu_512(&inputs[0][block_offset]); + out[1] = loadu_512(&inputs[1][block_offset]); + out[2] = loadu_512(&inputs[2][block_offset]); + out[3] = loadu_512(&inputs[3][block_offset]); + out[4] = loadu_512(&inputs[4][block_offset]); + out[5] = loadu_512(&inputs[5][block_offset]); + out[6] = loadu_512(&inputs[6][block_offset]); + out[7] = loadu_512(&inputs[7][block_offset]); + out[8] = loadu_512(&inputs[8][block_offset]); + out[9] = loadu_512(&inputs[9][block_offset]); + out[10] = loadu_512(&inputs[10][block_offset]); + out[11] = loadu_512(&inputs[11][block_offset]); + out[12] = loadu_512(&inputs[12][block_offset]); + out[13] = loadu_512(&inputs[13][block_offset]); + out[14] = loadu_512(&inputs[14][block_offset]); + out[15] = loadu_512(&inputs[15][block_offset]); + for (size_t i = 0; i < 16; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_512(out); +} + +INLINE void load_counters16(uint64_t counter, bool increment_counter, + __m512i *out_lo, __m512i *out_hi) { + const __m512i mask = _mm512_set1_epi32(-(int32_t)increment_counter); + const __m512i deltas = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const __m512i masked_deltas = _mm512_and_si512(deltas, mask); + const __m512i low_words = _mm512_add_epi32( + _mm512_set1_epi32((int32_t)counter), + masked_deltas); + // The carry bit is 1 if the high bit of the word was 1 before addition and is + // 0 after. + // NOTE: It would be a bit more natural to use _mm512_cmp_epu32_mask to + // compute the carry bits here, and originally we did, but that intrinsic is + // broken under GCC 5.4. See https://github.com/BLAKE3-team/BLAKE3/issues/271. + const __m512i carries = _mm512_srli_epi32( + _mm512_andnot_si512( + low_words, // 0 after (gets inverted by andnot) + _mm512_set1_epi32((int32_t)counter)), // and 1 before + 31); + const __m512i high_words = _mm512_add_epi32( + _mm512_set1_epi32((int32_t)(counter >> 32)), + carries); + *out_lo = low_words; + *out_hi = high_words; +} + +static +void blake3_hash16_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, + uint8_t *out) { + __m512i h_vecs[8] = { + set1_512(key[0]), set1_512(key[1]), set1_512(key[2]), set1_512(key[3]), + set1_512(key[4]), set1_512(key[5]), set1_512(key[6]), set1_512(key[7]), + }; + __m512i counter_low_vec, counter_high_vec; + load_counters16(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m512i block_len_vec = set1_512(BLAKE3_BLOCK_LEN); + __m512i block_flags_vec = set1_512(block_flags); + __m512i msg_vecs[16]; + transpose_msg_vecs16(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m512i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_512(IV[0]), set1_512(IV[1]), set1_512(IV[2]), set1_512(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn16(v, msg_vecs, 0); + round_fn16(v, msg_vecs, 1); + round_fn16(v, msg_vecs, 2); + round_fn16(v, msg_vecs, 3); + round_fn16(v, msg_vecs, 4); + round_fn16(v, msg_vecs, 5); + round_fn16(v, msg_vecs, 6); + h_vecs[0] = xor_512(v[0], v[8]); + h_vecs[1] = xor_512(v[1], v[9]); + h_vecs[2] = xor_512(v[2], v[10]); + h_vecs[3] = xor_512(v[3], v[11]); + h_vecs[4] = xor_512(v[4], v[12]); + h_vecs[5] = xor_512(v[5], v[13]); + h_vecs[6] = xor_512(v[6], v[14]); + h_vecs[7] = xor_512(v[7], v[15]); + + block_flags = flags; + } + + // transpose_vecs_512 operates on a 16x16 matrix of words, but we only have 8 + // state vectors. Pad the matrix with zeros. After transposition, store the + // lower half of each vector. + __m512i padded[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_512(0), set1_512(0), set1_512(0), set1_512(0), + set1_512(0), set1_512(0), set1_512(0), set1_512(0), + }; + transpose_vecs_512(padded); + _mm256_mask_storeu_epi32(&out[0 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[0])); + _mm256_mask_storeu_epi32(&out[1 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[1])); + _mm256_mask_storeu_epi32(&out[2 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[2])); + _mm256_mask_storeu_epi32(&out[3 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[3])); + _mm256_mask_storeu_epi32(&out[4 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[4])); + _mm256_mask_storeu_epi32(&out[5 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[5])); + _mm256_mask_storeu_epi32(&out[6 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[6])); + _mm256_mask_storeu_epi32(&out[7 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[7])); + _mm256_mask_storeu_epi32(&out[8 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[8])); + _mm256_mask_storeu_epi32(&out[9 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[9])); + _mm256_mask_storeu_epi32(&out[10 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[10])); + _mm256_mask_storeu_epi32(&out[11 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[11])); + _mm256_mask_storeu_epi32(&out[12 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[12])); + _mm256_mask_storeu_epi32(&out[13 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[13])); + _mm256_mask_storeu_epi32(&out[14 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[14])); + _mm256_mask_storeu_epi32(&out[15 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[15])); +} + +static +void blake3_xof16_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[16 * 64]) { + __m512i h_vecs[8] = { + set1_512(cv[0]), set1_512(cv[1]), set1_512(cv[2]), set1_512(cv[3]), + set1_512(cv[4]), set1_512(cv[5]), set1_512(cv[6]), set1_512(cv[7]), + }; + uint32_t block_words[16]; + load_block_words(block, block_words); + __m512i msg_vecs[16]; + for (size_t i = 0; i < 16; i++) { + msg_vecs[i] = set1_512(block_words[i]); + } + __m512i counter_low_vec, counter_high_vec; + load_counters16(counter, true, &counter_low_vec, &counter_high_vec); + __m512i block_len_vec = set1_512(block_len); + __m512i block_flags_vec = set1_512(flags); + __m512i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_512(IV[0]), set1_512(IV[1]), set1_512(IV[2]), set1_512(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn16(v, msg_vecs, 0); + round_fn16(v, msg_vecs, 1); + round_fn16(v, msg_vecs, 2); + round_fn16(v, msg_vecs, 3); + round_fn16(v, msg_vecs, 4); + round_fn16(v, msg_vecs, 5); + round_fn16(v, msg_vecs, 6); + for (size_t i = 0; i < 8; i++) { + v[i] = xor_512(v[i], v[i+8]); + v[i+8] = xor_512(v[i+8], h_vecs[i]); + } + transpose_vecs_512(&v[0]); + for (size_t i = 0; i < 16; i++) { + storeu_512(v[i], &out[i * sizeof(__m512i)]); + } +} + +/* + * ---------------------------------------------------------------------------- + * hash_many_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void hash_one_avx512(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_avx512(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= 16) { + blake3_hash16_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 16; + } + inputs += 16; + num_inputs -= 16; + out = &out[16 * BLAKE3_OUT_LEN]; + } + while (num_inputs >= 8) { + blake3_hash8_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 8; + } + inputs += 8; + num_inputs -= 8; + out = &out[8 * BLAKE3_OUT_LEN]; + } + while (num_inputs >= 4) { + blake3_hash4_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 4; + } + inputs += 4; + num_inputs -= 4; + out = &out[4 * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_avx512(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} + +void blake3_xof_many_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t* out, size_t outblocks) { + while (outblocks >= 16) { + blake3_xof16_avx512(cv, block, block_len, counter, flags, out); + counter += 16; + outblocks -= 16; + out += 16 * BLAKE3_BLOCK_LEN; + } + while (outblocks >= 8) { + blake3_xof8_avx512(cv, block, block_len, counter, flags, out); + counter += 8; + outblocks -= 8; + out += 8 * BLAKE3_BLOCK_LEN; + } + while (outblocks >= 4) { + blake3_xof4_avx512(cv, block, block_len, counter, flags, out); + counter += 4; + outblocks -= 4; + out += 4 * BLAKE3_BLOCK_LEN; + } + while (outblocks > 0) { + blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); + counter += 1; + outblocks -= 1; + out += BLAKE3_BLOCK_LEN; + } +} diff --git a/ext/digest/blake3/blake3_dispatch.c b/ext/digest/blake3/blake3_dispatch.c new file mode 100644 index 00000000000000..14dfbbe0c8f385 --- /dev/null +++ b/ext/digest/blake3/blake3_dispatch.c @@ -0,0 +1,332 @@ +#include +#include +#include + +#include "blake3_impl.h" + +#if defined(_MSC_VER) +#include +#endif + +#if defined(IS_X86) +#if defined(_MSC_VER) +#include +#elif defined(__GNUC__) +#include +#else +#undef IS_X86 /* Unimplemented! */ +#endif +#endif + +#if !defined(BLAKE3_ATOMICS) +#if defined(__has_include) +#if __has_include() && !defined(_MSC_VER) +#define BLAKE3_ATOMICS 1 +#else +#define BLAKE3_ATOMICS 0 +#endif /* __has_include() && !defined(_MSC_VER) */ +#else +#define BLAKE3_ATOMICS 0 +#endif /* defined(__has_include) */ +#endif /* BLAKE3_ATOMICS */ + +#if BLAKE3_ATOMICS +#define ATOMIC_INT _Atomic int +#define ATOMIC_LOAD(x) x +#define ATOMIC_STORE(x, y) x = y +#elif defined(_MSC_VER) +#define ATOMIC_INT LONG +#define ATOMIC_LOAD(x) InterlockedOr(&x, 0) +#define ATOMIC_STORE(x, y) InterlockedExchange(&x, y) +#else +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) x +#define ATOMIC_STORE(x, y) x = y +#endif + +#define MAYBE_UNUSED(x) (void)((x)) + +#if defined(IS_X86) +static uint64_t xgetbv(void) { +#if defined(_MSC_VER) + return _xgetbv(0); +#else + uint32_t eax = 0, edx = 0; + __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); + return ((uint64_t)edx << 32) | eax; +#endif +} + +static void cpuid(uint32_t out[4], uint32_t id) { +#if defined(_MSC_VER) + __cpuid((int *)out, id); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__("movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#endif +} + +static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { +#if defined(_MSC_VER) + __cpuidex((int *)out, id, sid); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__("movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#endif +} + + +enum cpu_feature { + SSE2 = 1 << 0, + SSSE3 = 1 << 1, + SSE41 = 1 << 2, + AVX = 1 << 3, + AVX2 = 1 << 4, + AVX512F = 1 << 5, + AVX512VL = 1 << 6, + /* ... */ + UNDEFINED = 1 << 30 +}; + +#if !defined(BLAKE3_TESTING) +static /* Allow the variable to be controlled manually for testing */ +#endif + ATOMIC_INT g_cpu_features = UNDEFINED; + +#if !defined(BLAKE3_TESTING) +static +#endif + enum cpu_feature + get_cpu_features(void) { + + /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ + enum cpu_feature features = ATOMIC_LOAD(g_cpu_features); + if (features != UNDEFINED) { + return features; + } else { +#if defined(IS_X86) + uint32_t regs[4] = {0}; + uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; + (void)edx; + features = 0; + cpuid(regs, 0); + const int max_id = *eax; + cpuid(regs, 1); +#if defined(__amd64__) || defined(_M_X64) + features |= SSE2; +#else + if (*edx & (1UL << 26)) + features |= SSE2; +#endif + if (*ecx & (1UL << 9)) + features |= SSSE3; + if (*ecx & (1UL << 19)) + features |= SSE41; + + if (*ecx & (1UL << 27)) { // OSXSAVE + const uint64_t mask = xgetbv(); + if ((mask & 6) == 6) { // SSE and AVX states + if (*ecx & (1UL << 28)) + features |= AVX; + if (max_id >= 7) { + cpuidex(regs, 7, 0); + if (*ebx & (1UL << 5)) + features |= AVX2; + if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm + if (*ebx & (1UL << 31)) + features |= AVX512VL; + if (*ebx & (1UL << 16)) + features |= AVX512F; + } + } + } + } + ATOMIC_STORE(g_cpu_features, features); + return features; +#else + /* How to detect NEON? */ + return 0; +#endif + } +} +#endif + +void blake3_compress_in_place(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_compress_in_place_avx512(cv, block, block_len, counter, flags); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_compress_in_place_sse41(cv, block, block_len, counter, flags); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_compress_in_place_sse2(cv, block, block_len, counter, flags); + return; + } +#endif +#endif + blake3_compress_in_place_portable(cv, block, block_len, counter, flags); +} + +void blake3_compress_xof(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64]) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out); + return; + } +#endif +#endif + blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); +} + + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks) { + if (outblocks == 0) { + // The current assembly implementation always outputs at least 1 block. + return; + } +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks); + return; + } +#endif +#endif + for(size_t i = 0; i < outblocks; ++i) { + blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i); + } +} + +void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { + blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_AVX2) + if (features & AVX2) { + blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#endif + +#if BLAKE3_USE_NEON == 1 + blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, out); + return; +#endif + + blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); +} + +// The dynamically detected SIMD degree of the current platform. +size_t blake3_simd_degree(void) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { + return 16; + } +#endif +#if !defined(BLAKE3_NO_AVX2) + if (features & AVX2) { + return 8; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + return 4; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + return 4; + } +#endif +#endif +#if BLAKE3_USE_NEON == 1 + return 4; +#endif + return 1; +} diff --git a/ext/digest/blake3/blake3_impl.h b/ext/digest/blake3/blake3_impl.h new file mode 100644 index 00000000000000..88e71e41e90f3f --- /dev/null +++ b/ext/digest/blake3/blake3_impl.h @@ -0,0 +1,333 @@ +#ifndef BLAKE3_IMPL_H +#define BLAKE3_IMPL_H + +#include +#include +#include +#include +#include + +#include "blake3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// internal flags +enum blake3_flags { + CHUNK_START = 1 << 0, + CHUNK_END = 1 << 1, + PARENT = 1 << 2, + ROOT = 1 << 3, + KEYED_HASH = 1 << 4, + DERIVE_KEY_CONTEXT = 1 << 5, + DERIVE_KEY_MATERIAL = 1 << 6, +}; + +// This C implementation tries to support recent versions of GCC, Clang, and +// MSVC. +#if defined(_MSC_VER) +#define INLINE static __forceinline +#else +#define INLINE static inline __attribute__((always_inline)) +#endif + +#ifdef __cplusplus +#define NOEXCEPT noexcept +#else +#define NOEXCEPT +#endif + +#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC) +#define IS_X86 +#define IS_X86_64 +#endif + +#if defined(__i386__) || defined(_M_IX86) +#define IS_X86 +#define IS_X86_32 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +#define IS_AARCH64 +#endif + +#if defined(IS_X86) +#if defined(_MSC_VER) +#include +#endif +#endif + +#if !defined(BLAKE3_USE_NEON) + // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness + #if defined(IS_AARCH64) + #if defined(__ARM_BIG_ENDIAN) + #define BLAKE3_USE_NEON 0 + #else + #define BLAKE3_USE_NEON 1 + #endif + #else + #define BLAKE3_USE_NEON 0 + #endif +#endif + +#if defined(IS_X86) +#define MAX_SIMD_DEGREE 16 +#elif BLAKE3_USE_NEON == 1 +#define MAX_SIMD_DEGREE 4 +#else +#define MAX_SIMD_DEGREE 1 +#endif + +// There are some places where we want a static size that's equal to the +// MAX_SIMD_DEGREE, but also at least 2. +#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) + +static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, + 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, + 0x1F83D9ABUL, 0x5BE0CD19UL}; + +static const uint8_t MSG_SCHEDULE[7][16] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, + {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, + {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, + {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, + {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, + {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, +}; + +/* Find index of the highest set bit */ +/* x is assumed to be nonzero. */ +static unsigned int highest_one(uint64_t x) { +#if defined(__GNUC__) || defined(__clang__) + return 63 ^ (unsigned int)__builtin_clzll(x); +#elif defined(_MSC_VER) && defined(IS_X86_64) + unsigned long index; + _BitScanReverse64(&index, x); + return index; +#elif defined(_MSC_VER) && defined(IS_X86_32) + if(x >> 32) { + unsigned long index; + _BitScanReverse(&index, (unsigned long)(x >> 32)); + return 32 + index; + } else { + unsigned long index; + _BitScanReverse(&index, (unsigned long)x); + return index; + } +#else + unsigned int c = 0; + if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } + if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } + if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } + if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } + if(x & 0x000000000000000cULL) { x >>= 2; c += 2; } + if(x & 0x0000000000000002ULL) { c += 1; } + return c; +#endif +} + +// Count the number of 1 bits. +INLINE unsigned int popcnt(uint64_t x) { +#if defined(__GNUC__) || defined(__clang__) + return (unsigned int)__builtin_popcountll(x); +#else + unsigned int count = 0; + while (x != 0) { + count += 1; + x &= x - 1; + } + return count; +#endif +} + +// Largest power of two less than or equal to x. As a special case, returns 1 +// when x is 0. +INLINE uint64_t round_down_to_power_of_2(uint64_t x) { + return 1ULL << highest_one(x | 1); +} + +INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } + +INLINE uint32_t counter_high(uint64_t counter) { + return (uint32_t)(counter >> 32); +} + +INLINE uint32_t load32(const void *src) { + const uint8_t *p = (const uint8_t *)src; + return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | + ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); +} + +INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], + uint32_t key_words[8]) { + key_words[0] = load32(&key[0 * 4]); + key_words[1] = load32(&key[1 * 4]); + key_words[2] = load32(&key[2 * 4]); + key_words[3] = load32(&key[3 * 4]); + key_words[4] = load32(&key[4 * 4]); + key_words[5] = load32(&key[5 * 4]); + key_words[6] = load32(&key[6 * 4]); + key_words[7] = load32(&key[7 * 4]); +} + +INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN], + uint32_t block_words[16]) { + for (size_t i = 0; i < 16; i++) { + block_words[i] = load32(&block[i * 4]); + } +} + +INLINE void store32(void *dst, uint32_t w) { + uint8_t *p = (uint8_t *)dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +} + +INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { + store32(&bytes_out[0 * 4], cv_words[0]); + store32(&bytes_out[1 * 4], cv_words[1]); + store32(&bytes_out[2 * 4], cv_words[2]); + store32(&bytes_out[3 * 4], cv_words[3]); + store32(&bytes_out[4 * 4], cv_words[4]); + store32(&bytes_out[5 * 4], cv_words[5]); + store32(&bytes_out[6 * 4], cv_words[6]); + store32(&bytes_out[7 * 4], cv_words[7]); +} + +void blake3_compress_in_place(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64]); + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks); + +void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out); + +size_t blake3_simd_degree(void); + +BLAKE3_PRIVATE size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb); + +#if defined(BLAKE3_USE_TBB) +BLAKE3_PRIVATE void blake3_compress_subtree_wide_join_tbb( + // shared params + const uint32_t key[8], uint8_t flags, bool use_tbb, + // left-hand side params + const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter, + uint8_t *l_cvs, size_t *l_n, + // right-hand side params + const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter, + uint8_t *r_cvs, size_t *r_n) NOEXCEPT; +#endif + +// Declarations for implementation-specific functions. +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +#if defined(IS_X86) +#if !defined(BLAKE3_NO_SSE2) +void blake3_compress_in_place_sse2(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse2(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_SSE41) +void blake3_compress_in_place_sse41(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse41(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_AVX2) +void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_AVX512) +void blake3_compress_in_place_avx512(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); + +void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +#if !defined(_WIN32) && !defined(__CYGWIN__) +void blake3_xof_many_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t* out, size_t outblocks); +#endif +#endif +#endif + +#if BLAKE3_USE_NEON == 1 +void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* BLAKE3_IMPL_H */ diff --git a/ext/digest/blake3/blake3_neon.c b/ext/digest/blake3/blake3_neon.c new file mode 100644 index 00000000000000..794ea80f4715c6 --- /dev/null +++ b/ext/digest/blake3/blake3_neon.c @@ -0,0 +1,367 @@ +#include "blake3_impl.h" + +#include + +#ifdef __ARM_BIG_ENDIAN +#error "This implementation only supports little-endian ARM." +// It might be that all we need for big-endian support here is to get the loads +// and stores right, but step zero would be finding a way to test it in CI. +#endif + +INLINE uint32x4_t loadu_128(const uint8_t src[16]) { + // vld1q_u32 has alignment requirements. Don't use it. + return vreinterpretq_u32_u8(vld1q_u8(src)); +} + +INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { + // vst1q_u32 has alignment requirements. Don't use it. + vst1q_u8(dest, vreinterpretq_u8_u32(src)); +} + +INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { + return vaddq_u32(a, b); +} + +INLINE uint32x4_t xor_128(uint32x4_t a, uint32x4_t b) { + return veorq_u32(a, b); +} + +INLINE uint32x4_t set1_128(uint32_t x) { return vld1q_dup_u32(&x); } + +INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + uint32_t array[4] = {a, b, c, d}; + return vld1q_u32(array); +} + +INLINE uint32x4_t rot16_128(uint32x4_t x) { + // The straightforward implementation would be two shifts and an or, but that's + // slower on microarchitectures we've tested. See + // https://github.com/BLAKE3-team/BLAKE3/pull/319. + // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); + return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x))); +} + +INLINE uint32x4_t rot12_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 12), vshlq_n_u32(x, 32 - 12)); + return vsriq_n_u32(vshlq_n_u32(x, 32-12), x, 12); +} + +INLINE uint32x4_t rot8_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8)); +#if defined(__clang__) + return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12)); +#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700 + static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12}; + return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8)); +#else + return vsriq_n_u32(vshlq_n_u32(x, 32-8), x, 8); +#endif +} + +INLINE uint32x4_t rot7_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 7), vshlq_n_u32(x, 32 - 7)); + return vsriq_n_u32(vshlq_n_u32(x, 32-7), x, 7); +} + +// TODO: compress_neon + +// TODO: hash2_neon + +/* + * ---------------------------------------------------------------------------- + * hash4_neon + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn4(uint32x4_t v[16], uint32x4_t m[16], size_t r) { + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[15] = rot16_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot12_128(v[4]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[15] = rot8_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot7_128(v[4]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot16_128(v[15]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[4] = rot12_128(v[4]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot8_128(v[15]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + v[4] = rot7_128(v[4]); +} + +INLINE void transpose_vecs_128(uint32x4_t vecs[4]) { + // Individually transpose the four 2x2 sub-matrices in each corner. + uint32x4x2_t rows01 = vtrnq_u32(vecs[0], vecs[1]); + uint32x4x2_t rows23 = vtrnq_u32(vecs[2], vecs[3]); + + // Swap the top-right and bottom-left 2x2s (which just got transposed). + vecs[0] = + vcombine_u32(vget_low_u32(rows01.val[0]), vget_low_u32(rows23.val[0])); + vecs[1] = + vcombine_u32(vget_low_u32(rows01.val[1]), vget_low_u32(rows23.val[1])); + vecs[2] = + vcombine_u32(vget_high_u32(rows01.val[0]), vget_high_u32(rows23.val[0])); + vecs[3] = + vcombine_u32(vget_high_u32(rows01.val[1]), vget_high_u32(rows23.val[1])); +} + +INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, + size_t block_offset, uint32x4_t out[16]) { + out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(uint32x4_t)]); + out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(uint32x4_t)]); + out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(uint32x4_t)]); + out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(uint32x4_t)]); + out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(uint32x4_t)]); + out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(uint32x4_t)]); + out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(uint32x4_t)]); + out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(uint32x4_t)]); + out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(uint32x4_t)]); + out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(uint32x4_t)]); + out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(uint32x4_t)]); + out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(uint32x4_t)]); + out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(uint32x4_t)]); + out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(uint32x4_t)]); + out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(uint32x4_t)]); + out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(uint32x4_t)]); + transpose_vecs_128(&out[0]); + transpose_vecs_128(&out[4]); + transpose_vecs_128(&out[8]); + transpose_vecs_128(&out[12]); +} + +INLINE void load_counters4(uint64_t counter, bool increment_counter, + uint32x4_t *out_low, uint32x4_t *out_high) { + uint64_t mask = (increment_counter ? ~0 : 0); + *out_low = set4( + counter_low(counter + (mask & 0)), counter_low(counter + (mask & 1)), + counter_low(counter + (mask & 2)), counter_low(counter + (mask & 3))); + *out_high = set4( + counter_high(counter + (mask & 0)), counter_high(counter + (mask & 1)), + counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); +} + +static void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, + uint8_t *out) { + uint32x4_t h_vecs[8] = { + set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), + set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), + }; + uint32x4_t counter_low_vec, counter_high_vec; + load_counters4(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + uint32x4_t block_len_vec = set1_128(BLAKE3_BLOCK_LEN); + uint32x4_t block_flags_vec = set1_128(block_flags); + uint32x4_t msg_vecs[16]; + transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + uint32x4_t v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn4(v, msg_vecs, 0); + round_fn4(v, msg_vecs, 1); + round_fn4(v, msg_vecs, 2); + round_fn4(v, msg_vecs, 3); + round_fn4(v, msg_vecs, 4); + round_fn4(v, msg_vecs, 5); + round_fn4(v, msg_vecs, 6); + h_vecs[0] = xor_128(v[0], v[8]); + h_vecs[1] = xor_128(v[1], v[9]); + h_vecs[2] = xor_128(v[2], v[10]); + h_vecs[3] = xor_128(v[3], v[11]); + h_vecs[4] = xor_128(v[4], v[12]); + h_vecs[5] = xor_128(v[5], v[13]); + h_vecs[6] = xor_128(v[6], v[14]); + h_vecs[7] = xor_128(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_128(&h_vecs[0]); + transpose_vecs_128(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu_128(h_vecs[0], &out[0 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[4], &out[1 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[1], &out[2 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[5], &out[3 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[2], &out[4 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[6], &out[5 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[3], &out[6 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[7], &out[7 * sizeof(uint32x4_t)]); +} + +/* + * ---------------------------------------------------------------------------- + * hash_many_neon + * ---------------------------------------------------------------------------- + */ + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +INLINE void hash_one_neon(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, uint8_t flags_end, + uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + // TODO: Implement compress_neon. However note that according to + // https://github.com/BLAKE2/BLAKE2/commit/7965d3e6e1b4193438b8d3a656787587d2579227, + // compress_neon might not be any faster than compress_portable. + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= 4) { + blake3_hash4_neon(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 4; + } + inputs += 4; + num_inputs -= 4; + out = &out[4 * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_neon(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/ext/digest/blake3/blake3_portable.c b/ext/digest/blake3/blake3_portable.c new file mode 100644 index 00000000000000..062dd1b47fb642 --- /dev/null +++ b/ext/digest/blake3/blake3_portable.c @@ -0,0 +1,160 @@ +#include "blake3_impl.h" +#include + +INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); +} + +INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, + uint32_t x, uint32_t y) { + state[a] = state[a] + state[b] + x; + state[d] = rotr32(state[d] ^ state[a], 16); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 12); + state[a] = state[a] + state[b] + y; + state[d] = rotr32(state[d] ^ state[a], 8); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 7); +} + +INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { + // Select the message schedule based on the round. + const uint8_t *schedule = MSG_SCHEDULE[round]; + + // Mix the columns. + g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); + g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); + g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); + g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); + + // Mix the rows. + g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); + g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); + g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); + g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); +} + +INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + uint32_t block_words[16]; + block_words[0] = load32(block + 4 * 0); + block_words[1] = load32(block + 4 * 1); + block_words[2] = load32(block + 4 * 2); + block_words[3] = load32(block + 4 * 3); + block_words[4] = load32(block + 4 * 4); + block_words[5] = load32(block + 4 * 5); + block_words[6] = load32(block + 4 * 6); + block_words[7] = load32(block + 4 * 7); + block_words[8] = load32(block + 4 * 8); + block_words[9] = load32(block + 4 * 9); + block_words[10] = load32(block + 4 * 10); + block_words[11] = load32(block + 4 * 11); + block_words[12] = load32(block + 4 * 12); + block_words[13] = load32(block + 4 * 13); + block_words[14] = load32(block + 4 * 14); + block_words[15] = load32(block + 4 * 15); + + state[0] = cv[0]; + state[1] = cv[1]; + state[2] = cv[2]; + state[3] = cv[3]; + state[4] = cv[4]; + state[5] = cv[5]; + state[6] = cv[6]; + state[7] = cv[7]; + state[8] = IV[0]; + state[9] = IV[1]; + state[10] = IV[2]; + state[11] = IV[3]; + state[12] = counter_low(counter); + state[13] = counter_high(counter); + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + + round_fn(state, &block_words[0], 0); + round_fn(state, &block_words[0], 1); + round_fn(state, &block_words[0], 2); + round_fn(state, &block_words[0], 3); + round_fn(state, &block_words[0], 4); + round_fn(state, &block_words[0], 5); + round_fn(state, &block_words[0], 6); +} + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + cv[0] = state[0] ^ state[8]; + cv[1] = state[1] ^ state[9]; + cv[2] = state[2] ^ state[10]; + cv[3] = state[3] ^ state[11]; + cv[4] = state[4] ^ state[12]; + cv[5] = state[5] ^ state[13]; + cv[6] = state[6] ^ state[14]; + cv[7] = state[7] ^ state[15]; +} + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + + store32(&out[0 * 4], state[0] ^ state[8]); + store32(&out[1 * 4], state[1] ^ state[9]); + store32(&out[2 * 4], state[2] ^ state[10]); + store32(&out[3 * 4], state[3] ^ state[11]); + store32(&out[4 * 4], state[4] ^ state[12]); + store32(&out[5 * 4], state[5] ^ state[13]); + store32(&out[6 * 4], state[6] ^ state[14]); + store32(&out[7 * 4], state[7] ^ state[15]); + store32(&out[8 * 4], state[8] ^ cv[0]); + store32(&out[9 * 4], state[9] ^ cv[1]); + store32(&out[10 * 4], state[10] ^ cv[2]); + store32(&out[11 * 4], state[11] ^ cv[3]); + store32(&out[12 * 4], state[12] ^ cv[4]); + store32(&out[13 * 4], state[13] ^ cv[5]); + store32(&out[14 * 4], state[14] ^ cv[6]); + store32(&out[15 * 4], state[15] ^ cv[7]); +} + +INLINE void hash_one_portable(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + store_cv_words(out, cv); +} + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs > 0) { + hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/ext/digest/blake3/blake3_sse2.c b/ext/digest/blake3/blake3_sse2.c new file mode 100644 index 00000000000000..691e1c6806cce4 --- /dev/null +++ b/ext/digest/blake3/blake3_sse2.c @@ -0,0 +1,566 @@ +#include "blake3_impl.h" + +#include + +#define DEGREE 4 + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu(const uint8_t src[16]) { + return _mm_loadu_si128((const __m128i *)src); +} + +INLINE void storeu(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((__m128i *)dest, src); +} + +INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16(__m128i x) { + return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1); +} + +INLINE __m128i rot12(__m128i x) { + return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); +} + +INLINE __m128i rot8(__m128i x) { + return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8)); +} + +INLINE __m128i rot7(__m128i x) { + return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); +} + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot16(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot12(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot8(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot7(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) { + const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01); + __m128i mask = _mm_set1_epi16(imm8); + mask = _mm_and_si128(mask, bits); + mask = _mm_cmpeq_epi16(mask, bits); + return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu((uint8_t *)&cv[0]); + rows[1] = loadu((uint8_t *)&cv[4]); + rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_in_place_sse2(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +void blake3_compress_xof_sse2(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), &out[0]); + storeu(xorv(rows[1], rows[3]), &out[16]); + storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); + storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); +} + +INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m128i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[4]); + transpose_vecs(&out[8]); + transpose_vecs(&out[12]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); + const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); + const __m128i add1 = _mm_and_si128(mask, add0); + __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); + __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), + _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); + __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(&h_vecs[0]); + transpose_vecs(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +INLINE void hash_one_sse2(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash4_sse2(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/ext/digest/blake3/blake3_sse41.c b/ext/digest/blake3/blake3_sse41.c new file mode 100644 index 00000000000000..4653a856fe6c44 --- /dev/null +++ b/ext/digest/blake3/blake3_sse41.c @@ -0,0 +1,560 @@ +#include "blake3_impl.h" + +#include + +#define DEGREE 4 + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu(const uint8_t src[16]) { + return _mm_loadu_si128((const __m128i *)src); +} + +INLINE void storeu(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((__m128i *)dest, src); +} + +INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16(__m128i x) { + return _mm_shuffle_epi8( + x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); +} + +INLINE __m128i rot12(__m128i x) { + return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); +} + +INLINE __m128i rot8(__m128i x) { + return _mm_shuffle_epi8( + x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); +} + +INLINE __m128i rot7(__m128i x) { + return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); +} + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot16(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot12(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot8(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot7(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu((uint8_t *)&cv[0]); + rows[1] = loadu((uint8_t *)&cv[4]); + rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_in_place_sse41(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +void blake3_compress_xof_sse41(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), &out[0]); + storeu(xorv(rows[1], rows[3]), &out[16]); + storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); + storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); +} + +INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m128i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[4]); + transpose_vecs(&out[8]); + transpose_vecs(&out[12]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); + const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); + const __m128i add1 = _mm_and_si128(mask, add0); + __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); + __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), + _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); + __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(&h_vecs[0]); + transpose_vecs(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +INLINE void hash_one_sse41(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash4_sse41(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/ext/digest/blake3/blake3init.c b/ext/digest/blake3/blake3init.c new file mode 100644 index 00000000000000..639107a2611d60 --- /dev/null +++ b/ext/digest/blake3/blake3init.c @@ -0,0 +1,82 @@ +/* BLAKE3 binding for the Ruby digest framework. */ + +#include +#include "../digest.h" +#include "blake3.h" + +/* + * Thin adapters mapping BLAKE3's public API onto the signatures the + * digest framework expects (see rb_digest_metadata_t in ../digest.h). + * The signatures line up directly, so no function-pointer casts are + * needed. + */ + +static int +rb_digest_BLAKE3_init(void *ctx) +{ + blake3_hasher_init((blake3_hasher *)ctx); + return 1; +} + +static void +rb_digest_BLAKE3_update(void *ctx, unsigned char *ptr, size_t size) +{ + blake3_hasher_update((blake3_hasher *)ctx, ptr, size); +} + +static int +rb_digest_BLAKE3_finish(void *ctx, unsigned char *ptr) +{ + blake3_hasher_finalize((blake3_hasher *)ctx, ptr, BLAKE3_OUT_LEN); + return 1; +} + +static const rb_digest_metadata_t blake3 = { + RUBY_DIGEST_API_VERSION, + BLAKE3_OUT_LEN, + BLAKE3_BLOCK_LEN, + sizeof(blake3_hasher), + rb_digest_BLAKE3_init, + rb_digest_BLAKE3_update, + rb_digest_BLAKE3_finish, +}; + +/* + * Document-class: Digest::BLAKE3 < Digest::Base + * A class for calculating message digests using the BLAKE3 hash function, + * by Jack O'Connor, Jean-Philippe Aumasson, Samuel Neves, and Zooko + * Wilcox-O'Hearn. + * + * BLAKE3 produces a digest of 256 bits (32 bytes) by default. + * + * == Examples + * require 'digest' + * + * # Compute a complete digest + * Digest::BLAKE3.hexdigest 'abc' #=> "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85" + * + * # Compute digest by chunks + * blake3 = Digest::BLAKE3.new # =># + * blake3.update "ab" + * blake3 << "c" # alias for #update + * blake3.hexdigest # => "6437b3ac..." + * + * # Use the same object to compute another digest + * blake3.reset + * blake3 << "message" + * blake3.hexdigest + */ +void +Init_blake3(void) +{ + VALUE mDigest, cDigest_Base, cDigest_BLAKE3; + +#if 0 + mDigest = rb_define_module("Digest"); /* let rdoc know */ +#endif + mDigest = rb_digest_namespace(); + cDigest_Base = rb_const_get(mDigest, rb_intern_const("Base")); + + cDigest_BLAKE3 = rb_define_class_under(mDigest, "BLAKE3", cDigest_Base); + rb_iv_set(cDigest_BLAKE3, "metadata", rb_digest_make_metadata(&blake3)); +} diff --git a/ext/digest/blake3/extconf.rb b/ext/digest/blake3/extconf.rb new file mode 100644 index 00000000000000..df844d07cc4904 --- /dev/null +++ b/ext/digest/blake3/extconf.rb @@ -0,0 +1,102 @@ +# -*- coding: us-ascii -*- +# frozen_string_literal: false + +require "mkmf" + +# BLAKE3 build configuration. +# +# The binding is always built from the portable C code (blake3.c, +# blake3_dispatch.c, blake3_portable.c). On architectures BLAKE3 ships +# optimized backends for, we additionally compile the SIMD implementations +# and let blake3_dispatch.c select the fastest one supported by the CPU the +# program is actually running on (via CPUID on x86). This means a binary +# built on an AVX-512-capable machine still runs correctly on an older CPU. +# +# Each SIMD translation unit must be compiled with its own instruction-set +# flag, which a single global $CFLAGS can't express, so we emit one explicit +# object rule per backend at the end (see below). Any x86 backend we do NOT +# compile is disabled with -DBLAKE3_NO_; that macro is honoured both by +# the dispatcher and by sibling backends (e.g. blake3_avx2.c falls back to +# SSE4.1 helpers), keeping the set of referenced symbols consistent. + +objs = %w[blake3init blake3 blake3_dispatch blake3_portable] + +# Extra per-object compiler flags, keyed by object basename. +simd_cflags = {} + +# Probe used to confirm the compiler both accepts +flag+ and can compile the +# intrinsics the backend relies on. +def blake3_have_isa?(name, flag, snippet) + checking_for("#{name} intrinsics (#{flag})") do + try_compile(snippet, flag) + end +end + +case RbConfig::CONFIG["host_cpu"] +when /\A(x86_64|amd64|x64)\z/i + # Try to detect which SIMD features this x86 machine and compiler has + x86_backends = [ + ["blake3_sse2", "SSE2", "-msse2", + "#include \nint main(void){ volatile __m128i x = _mm_setzero_si128(); (void)x; return 0; }\n", + "BLAKE3_NO_SSE2"], + ["blake3_sse41", "SSE4.1", "-msse4.1", + "#include \nint main(void){ volatile __m128i x = _mm_setzero_si128(); return _mm_testz_si128(x, x); }\n", + "BLAKE3_NO_SSE41"], + ["blake3_avx2", "AVX2", "-mavx2", + "#include \nint main(void){ volatile __m256i x = _mm256_setzero_si256(); (void)x; return 0; }\n", + "BLAKE3_NO_AVX2"], + ["blake3_avx512", "AVX-512", "-mavx512f -mavx512vl", + "#include \nint main(void){ volatile __m512i x = _mm512_setzero_si512(); (void)x; return 0; }\n", + "BLAKE3_NO_AVX512"], + ] + + x86_backends.each do |obj, name, flag, snippet, no_macro| + if blake3_have_isa?(name, flag, snippet) + objs << obj + simd_cflags[obj] = flag + else + $defs << "-D#{no_macro}" + end + end +when /\A(aarch64|arm64)\z/i + # NEON is part of the AArch64 baseline, so no runtime detection or special + # compiler flag is needed. Leave BLAKE3_USE_NEON to auto-detect, which is + # 1 on little-endian AArch64 (see blake3_impl.h). + objs << "blake3_neon" +else + # No optimized backend wired up for this architecture (e.g. 32-bit x86, + # ppc): build portable-only. Disabling every x86 ISA keeps the dispatcher + # from referencing backends we didn't compile, and NEON is forced off. + $defs << "-DBLAKE3_NO_SSE2" + $defs << "-DBLAKE3_NO_SSE41" + $defs << "-DBLAKE3_NO_AVX2" + $defs << "-DBLAKE3_NO_AVX512" + $defs << "-DBLAKE3_USE_NEON=0" +end + +$objs = objs.map { |o| "#{o}.#{$OBJEXT}" } + +have_header("sys/cdefs.h") + +$preload = %w[digest] + +create_makefile("digest/blake3") + +# Emit one explicit compile rule per SIMD backend so each gets its own +# instruction-set flag. mkmf's implicit .c.o rule compiles every object with +# the same $(CFLAGS), which can't express e.g. -mavx2 for one file only; an +# explicit rule with a recipe overrides that implicit rule for these targets. +# The recipe mirrors mkmf's .c.o rule with the extra flag inserted after +# $(CFLAGS). +unless simd_cflags.empty? + File.open("Makefile", "a") do |mf| + mf.puts + mf.puts "# Per-file instruction-set flags for the BLAKE3 SIMD backends." + simd_cflags.each do |obj, flag| + target = "#{obj}.#{$OBJEXT}" + mf.puts "#{target}: $(srcdir)/#{obj}.c" + mf.puts "\t$(ECHO) compiling $(<)" + mf.puts "\t$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) #{flag} $(COUTFLAG)$@ -c $(CSRCFLAG)$(srcdir)/#{obj}.c" + end + end +end diff --git a/ext/digest/digest.gemspec b/ext/digest/digest.gemspec index 4a01c5fde16d05..033779fb894a69 100644 --- a/ext/digest/digest.gemspec +++ b/ext/digest/digest.gemspec @@ -37,6 +37,9 @@ Gem::Specification.new do |spec| spec.extensions = Dir["ext/digest/**/extconf.rb"] spec.files += Dir["ext/digest/**/{*.{rb,c,h,sh},depend}"] + # Vendored third-party license files (e.g. BLAKE3's LICENSE_*), which + # carry no filename extension and are missed by the glob above. + spec.files += Dir["ext/digest/**/LICENSE*"] spec.require_paths = %w[lib] end diff --git a/test/digest/test_digest.rb b/test/digest/test_digest.rb index 8a39a7bc27820a..c2b98f2f5bee3f 100644 --- a/test/digest/test_digest.rb +++ b/test/digest/test_digest.rb @@ -6,7 +6,7 @@ require 'tempfile' require 'digest' -%w[digest/md5 digest/rmd160 digest/sha1 digest/sha2 digest/bubblebabble digest/crc32].each do |lib| +%w[digest/md5 digest/rmd160 digest/sha1 digest/sha2 digest/bubblebabble digest/crc32 digest/blake3].each do |lib| begin require lib rescue LoadError @@ -279,6 +279,106 @@ def test_initialize_copy_into_frozen_raises end end + class TestBLAKE3 < Test::Unit::TestCase + include TestDigest + ALGO = Digest::BLAKE3 + DATA = { + Data1 => "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85", + Data2 => "c19012cc2aaf0dc3d8e5c45a1b79114d2df42abb2a410bf54be09e891af06ff8", + } + + # Input byte i has value (i % 251), matching the BLAKE3 + # test_vectors.json fixture. The expected value is the first 32 bytes + # (the default digest length) of each case's extended output hash. + TEST_VECTORS = { + 0 => "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", + 1 => "2d3adedff11b61f14c886e35afa036736dcd87a74d27b5c1510225d0f592e213", + 64 => "4eed7141ea4a5cd4b788606bd23f46e212af9cacebacdc7d1f4c6dc7f2511b98", + 65 => "de1e5fa0be70df6d2be8fffd0e99ceaa8eb6e8c93a63f2d8d1c30ecb6b263dee", + 1023 => "10108970eeda3eb932baac1428c7a2163b0e924c9a9e25b35bba72b28f70bd11", + 1024 => "42214739f095a406f3fc83deb889744ac00df831c10daa55189b5d121c855af7", + 1025 => "d00278ae47eb27b34faecf67b4fe263f82d5412916c1ffd97c8cb7fb814b8444", + 2048 => "e776b6028c7cd22a4d0ba182a8bf62205d2ef576467e838ed6f2529b85fba24a", + 3072 => "b98cb0ff3623be03326b373de6b9095218513e64f1ee2edd2525c7ad1e5cffd2", + 4096 => "015094013f57a5277b59d8475c0501042c0b642e531b0a1c8f58d2163229e969", + 8192 => "aae792484c8efe4f19e2ca7d371d8c467ffb10748d8a5a1ae579948f718a2a63", + 16384 => "f875d6646de28985646f34ee13be9a576fd515f76b5b0a26bb324735041ddde4", + 102400 => "bc3e3d41a1146b069abffad3c0d44860cf664390afce4d9661f7902e7943e085", + } + + # https://github.com/BLAKE3-team/BLAKE3/blob/93a431c78a52d7ccf0f366f106467f5070e6075e/test_vectors/src/lib.rs#L65-L73 + def paint_test_input(len) + len.times.map { |i| (i % 251).chr }.join + end + + def test_digest_length + assert_equal 32, Digest::BLAKE3.new.digest_length + end + + def test_block_length + assert_equal 64, Digest::BLAKE3.new.block_length + end + + def test_empty_string + assert_equal "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", + Digest::BLAKE3.hexdigest("") + end + + def test_known_vectors + # Exercises single-chunk, chunk-boundary and multi-chunk tree hashing + # against the upstream BLAKE3 test vectors. + TEST_VECTORS.each do |len, expected| + assert_equal expected, Digest::BLAKE3.hexdigest(paint_test_input(len)), + "BLAKE3 of #{len}-byte test input" + end + end + + def test_incremental_equals_one_shot + # Feed a multi-chunk input in awkward, unaligned pieces and compare + # against the one-shot digest for the same bytes. + input = paint_test_input(4097) + inc = Digest::BLAKE3.new + off = 0 + [1, 63, 64, 65, 900, 1024, 1080].each do |n| + inc << input[off, n] + off += n + end + inc << input[off..] + assert_equal Digest::BLAKE3.hexdigest(input), inc.hexdigest + end + + def test_clone_mid_stream_independence + d = Digest::BLAKE3.new + d << "ab" + copy = d.clone + d << "c" + copy << "c" + assert_equal d.hexdigest, copy.hexdigest + assert_equal Digest::BLAKE3.hexdigest("abc"), copy.hexdigest + end + + def test_reset + d = Digest::BLAKE3.new + d << "some other data" + d.reset + d << "abc" + assert_equal Digest::BLAKE3.hexdigest("abc"), d.hexdigest + end + + def test_digest_bang_resets_state + d = Digest::BLAKE3.new + d << "abc" + d.hexdigest! + assert_equal Digest::BLAKE3.hexdigest(""), d.hexdigest + end + + def test_initialize_copy_into_frozen_raises + dest = Digest::BLAKE3.allocate + dest.freeze + assert_raise(FrozenError) { dest.send(:initialize_copy, Digest::BLAKE3.new) } + end + end if defined?(Digest::BLAKE3) + class TestBase < Test::Unit::TestCase def test_base bug3810 = '[ruby-core:32231]' From 4490cedbf95f50f4abc30bc2071dad8bbf971ca8 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 14 Aug 2026 14:14:44 +0200 Subject: [PATCH 14/20] [ruby/digest] Don't use _Atomic in C99 mode ``` /github/workspace/src/ext/digest/blake3/blake3_dispatch.c:108:5: error: '_Atomic' is a C11 extension [-Werror,-Wc11-extensions] 108 | ATOMIC_INT g_cpu_features = UNDEFINED; | ^ /github/workspace/src/ext/digest/blake3/blake3_dispatch.c:34:20: note: expanded from macro 'ATOMIC_INT' 34 | #define ATOMIC_INT _Atomic int | ^ 1 error generated. ``` https://github.com/ruby/digest/commit/505c2d159f --- ext/digest/blake3/extconf.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ext/digest/blake3/extconf.rb b/ext/digest/blake3/extconf.rb index df844d07cc4904..4eec4c4bf6d044 100644 --- a/ext/digest/blake3/extconf.rb +++ b/ext/digest/blake3/extconf.rb @@ -78,6 +78,18 @@ def blake3_have_isa?(name, flag, snippet) have_header("sys/cdefs.h") +checking_for("C11 atomics") do + ok = try_compile(<<~C) + static _Atomic int atomic_int = 0; + static int t(void) {return atomic_int;} + #{MAIN_DOES_NOTHING 't'} + C + unless ok + $defs << "-DBLAKE3_ATOMICS=0" + end + ok +end + $preload = %w[digest] create_makefile("digest/blake3") From 2febd4bd301d62d29ec024c4ca813bf6a9d72b72 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sat, 15 Aug 2026 00:10:14 +0900 Subject: [PATCH 15/20] [ruby/digest] Pass config macros via command line The distributed files do not include `extconf.h` generated by `create_header`. https://github.com/ruby/digest/commit/d3bcdccba3 --- ext/digest/blake3/extconf.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ext/digest/blake3/extconf.rb b/ext/digest/blake3/extconf.rb index 4eec4c4bf6d044..6d1b8f54e34bc0 100644 --- a/ext/digest/blake3/extconf.rb +++ b/ext/digest/blake3/extconf.rb @@ -92,7 +92,12 @@ def blake3_have_isa?(name, flag, snippet) $preload = %w[digest] -create_makefile("digest/blake3") +create_makefile("digest/blake3") do |mk| + mk.grep(/^CPPFLAGS *=/) {|m| + m.sub!(/(?=-DRUBY_EXTCONF_H)/) {[$defs, ''].join(' ')} + } + mk +end # Emit one explicit compile rule per SIMD backend so each gets its own # instruction-set flag. mkmf's implicit .c.o rule compiles every object with From f0edab28988f19f03fefa89161059480a0e56d1e Mon Sep 17 00:00:00 2001 From: XrXr Date: Fri, 14 Aug 2026 10:57:11 -0400 Subject: [PATCH 16/20] ZJIT: Put stack map details behind `#if USE_ZJIT` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mainly, to dodge warnings on i686: ../src/zjit.h: In function ‘ZJIT_STACK_MAP_BASE_PTR_STACK_SIZE’: ../src/zjit.h:83:18: warning: right shift count >= width of type [-Wshift-count-overflow] 83 | return entry >> ZJIT_STACK_MAP_BASE_PTR_SIZE_SHIFT; | ^~ --- zjit.h | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/zjit.h b/zjit.h index 8bd5ec57c8fb01..f7928989c5ec18 100644 --- a/zjit.h +++ b/zjit.h @@ -11,6 +11,29 @@ # define ZJIT_STATS (USE_ZJIT && RUBY_DEBUG) #endif +// JITFrame is defined here as the single source of truth and imported into +// Rust via bindgen. C code reads fields directly; Rust uses an impl block. +typedef struct zjit_jit_frame { + // Program counter for this frame, used for backtraces and GC. + // NULL for C frames (they don't have a Ruby PC). + const VALUE *pc; + // The ISEQ this frame belongs to. Marked via rb_execution_context_mark. + // NULL for C frames. + const rb_iseq_t *iseq; + // Whether to materialize block_code when this frame is materialized. + // True when the ISEQ doesn't contain send/invokesuper/invokeblock + // (which write block_code themselves), so we must restore it. + // Always false for C frames. + bool materialize_block_code; + + // Number of stack map entries in stack[]. + uint32_t stack_size; + // Flexible array of stack map entries, executed in order by + // zjit_materialize_frames(). See the ZJIT_STACK_MAP_* opcodes above. + VALUE stack[]; +} zjit_jit_frame_t; + +#if USE_ZJIT // Stack map entries are opcodes for zjit_materialize_frames(), which walks them // in order while moving a cursor down the VM stack. An untagged entry is an // immediate Ruby VALUE to store; the tagged forms below copy from the native @@ -83,29 +106,6 @@ ZJIT_STACK_MAP_BASE_PTR_STACK_SIZE(VALUE entry) return entry >> ZJIT_STACK_MAP_BASE_PTR_SIZE_SHIFT; } -// JITFrame is defined here as the single source of truth and imported into -// Rust via bindgen. C code reads fields directly; Rust uses an impl block. -typedef struct zjit_jit_frame { - // Program counter for this frame, used for backtraces and GC. - // NULL for C frames (they don't have a Ruby PC). - const VALUE *pc; - // The ISEQ this frame belongs to. Marked via rb_execution_context_mark. - // NULL for C frames. - const rb_iseq_t *iseq; - // Whether to materialize block_code when this frame is materialized. - // True when the ISEQ doesn't contain send/invokesuper/invokeblock - // (which write block_code themselves), so we must restore it. - // Always false for C frames. - bool materialize_block_code; - - // Number of stack map entries in stack[]. - uint32_t stack_size; - // Flexible array of stack map entries, executed in order by - // zjit_materialize_frames(). See the ZJIT_STACK_MAP_* opcodes above. - VALUE stack[]; -} zjit_jit_frame_t; - -#if USE_ZJIT extern void *rb_zjit_entry; extern const zjit_jit_frame_t rb_zjit_c_frame; extern uint64_t rb_zjit_call_threshold; From 1b0c22c1151b48286a643719b4cc3a2b93f93a3b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 14 Aug 2026 19:55:47 +0200 Subject: [PATCH 17/20] hash.c: compact ar_table on freeze Since frozen ar_table can occupy smaller slots, shrinking the table on freeze potentially allow the Hash to be demoted to an even smaller slot. --- hash.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hash.c b/hash.c index 3c33081a424d0c..6d61079cc9ab71 100644 --- a/hash.c +++ b/hash.c @@ -102,10 +102,15 @@ static VALUE rb_hash_s_try_convert(VALUE, VALUE); * 2. Insert WBs */ +static int ar_compact_table(VALUE hash); + /* :nodoc: */ VALUE rb_hash_freeze(VALUE hash) { + if (!OBJ_FROZEN(hash) && RHASH_AR_TABLE_P(hash)) { + ar_compact_table(hash); + } return rb_obj_freeze(hash); } From 65a071aad931b685ef4d4771f57e58aad5b34e3f Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Fri, 14 Aug 2026 16:33:34 +0000 Subject: [PATCH 18/20] GC: fix page pool advised_count underflow on 64KiB-page systems page_pool_reclaim() advises (madvise) cold page bodies only when the OS page size is smaller than HEAP_PAGE_SIZE, because the first OS page of a body holds the in-body freelist link and the arena tag and must be spared. On systems whose page size is >= HEAP_PAGE_SIZE (64KiB pages on aarch64, e.g. the oci-aarch64 CI machine) Step A therefore never increments page_pool.advised_count -- but Step B still subtracted PAGE_POOL_ARENA_BODIES (32) from it for every munmap'd arena, so the counter went negative. advised_count is an int reported through GC.stat as page_pool_discarded_pages with SIZET2NUM(), so a negative value is converted to a huge unsigned value and GC.stat has to allocate a Bignum for it. That allocation happens after GC.stat has already read the slot counters, so any pair of "read stats, then read them again another way" observes one extra live object, which is exactly what these two tests do (both fail by exactly 1 in the same direction). Skip the Step B adjustment when the platform cannot advise at all. When it can, every body of an unmapped arena is on its cold freelist and was just advised by Step A, so subtracting 32 stays exact. Verified by simulating a 64KiB page size (forcing page_pool.os_page_size = HEAP_PAGE_SIZE) on x86_64: before the fix GC.stat[:page_pool_discarded_pages] reads 18446744073709551456 after arenas are unmapped and the test_stat comparison is off by 1; after the fix it reads 0 and test/ruby/test_gc.rb passes both with and without the simulation. Error: 1) Failure: TestGc#test_stat_heap_constraints [test/ruby/test_gc.rb:298]: <357949> expected but was <357950>. 2) Failure: TestGc#test_stat [test/ruby/test_gc.rb:178]: <356712> expected but was <356711>. (5 consecutive master@oci-aarch64 runs since 9ebb977651) CI: https://ci.rvm.jp/results/master@oci-aarch64/6449795 Log: https://ci.rvm.jp/logfiles/brlog.master.20260814-161514 Co-Authored-By: Claude Opus 5 (1M context) --- gc/default/default.c | 15 +++++++++++++-- test/ruby/test_gc.rb | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 89468669853880..21833210446f8e 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -2578,8 +2578,14 @@ page_pool_reclaim(rb_global_objspace_t *g) rb_native_mutex_lock(&g->page_pool.lock); + /* Advising spares the first OS page of a body (it holds the in-body freelist link + * and the arena tag), so it needs sub-page granularity: when the OS page size is + * >= HEAP_PAGE_SIZE (e.g. 64KiB pages on aarch64) no body is ever advised, and + * advised_count must not be adjusted anywhere either. */ + const bool can_advise = os_page_size < HEAP_PAGE_SIZE; + /* Step A — advise cold bodies (immediate release: drop RSS now if the platform allows). */ - if (os_page_size < HEAP_PAGE_SIZE) { + if (can_advise) { for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) { for (struct heap_page_body *body = a->cold_freelist; body; ) { asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false); @@ -2657,7 +2663,12 @@ page_pool_reclaim(rb_global_objspace_t *g) rb_bug("page_pool_reclaim: munmap failed"); } total_free -= PAGE_POOL_ARENA_BODIES; - g->page_pool.advised_count -= PAGE_POOL_ARENA_BODIES; + /* Every body of this arena is on its cold freelist, so Step A above has just + * advised all of them -- but only if this platform can advise at all. */ + if (can_advise) { + g->page_pool.advised_count -= PAGE_POOL_ARENA_BODIES; + GC_ASSERT(g->page_pool.advised_count >= 0); + } g->page_pool.arena_count--; g->page_pool.arenas_unmapped++; if (a == g->page_pool.arena_current) { diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index ef022650199f38..84202c57ccf7f7 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -304,6 +304,26 @@ def test_stat_heap_constraints assert_equal stat[:total_freed_objects], stat_heap_sum[:total_freed_objects] end + def test_page_pool_stat_consistency + omit 'no page pool' unless GC.stat.key?(:page_pool_total_pages) + + # Freeing arenas back to the OS must keep page_pool_discarded_pages in range. + # An underflowed counter wraps to a huge value, which also makes GC.stat + # allocate a Bignum and perturb the object counts it reports. + assert_separately([], __FILE__, __LINE__, <<~RUBY, timeout: 60) + 3.times do + ary = 200_000.times.map { "x" * 40 } + ary.clear + GC.start(full_mark: true, immediate_sweep: true) + GC.start(full_mark: true, immediate_sweep: true) + end + + stat = GC.stat + assert_operator stat[:page_pool_discarded_pages], :<=, stat[:page_pool_total_pages] + assert_operator stat[:page_pool_arenas], :>=, 0 # arenas is always 0 if doesn't have mmap + RUBY + end + def test_measure_total_time assert_separately([], __FILE__, __LINE__, <<~RUBY, timeout: 60) GC.measure_total_time = false From 3349f4107d268658fdf8cc6b979fbb1c923e597f Mon Sep 17 00:00:00 2001 From: XrXr Date: Fri, 14 Aug 2026 15:16:40 -0400 Subject: [PATCH 19/20] Filter TracePoint events in test_trace_optimized_methods to current file Same as test_settracefunc.rb. This should fix flaky CI failures like https://github.com/ruby/ruby/actions/runs/31826726042/job/94852494689 --- test/ruby/test_optimization.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ruby/test_optimization.rb b/test/ruby/test_optimization.rb index 5d26e16b44f43d..c114b98e4792de 100644 --- a/test/ruby/test_optimization.rb +++ b/test/ruby/test_optimization.rb @@ -211,7 +211,7 @@ def test_trace_optimized_methods :&, :|, :[], :[]=, :length, :empty?, :nil?, :succ, :!, :=~] [:c_call, :c_return].each do |type| methods = [] - tp = TracePoint.new(type) { |tp| methods << tp.method_id } + tp = TracePoint.new(type) { |tp| methods << tp.method_id if tp.path == __FILE__ } tp.enable do x = "a"; x = -x [1].max @@ -243,7 +243,7 @@ def test_trace_optimized_methods end methods = [] - tp = TracePoint.new(:c_call, :c_return) { |tp| methods << tp.method_id } + tp = TracePoint.new(:c_call, :c_return) { |tp| methods << tp.method_id if tp.path == __FILE__ } tp.enable do x = 1 x != 42 From 6cc5c920b029a686ef6fbf93ffb11f57fb312dee Mon Sep 17 00:00:00 2001 From: XrXr Date: Fri, 14 Aug 2026 19:48:39 -0400 Subject: [PATCH 20/20] ZJIT: Skip fiddle memory leak test when --zjit-call-threshold=1 This should avoid flaky failures such as --- tool/test-bundled-gems.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb index 75bae805ba5433..87e065719be81b 100644 --- a/tool/test-bundled-gems.rb +++ b/tool/test-bundled-gems.rb @@ -131,6 +131,13 @@ when "rexml" test_command[-2..-1] = %w[test/run.rb --ignore-name=/linear_performance/] + when "fiddle" + # When ZJIT is compile-happy, skip Fiddle::TestFunction#test_no_memory_leak + # since compiling uses more memory which the test does not expect. + if run_opts&.include?("--zjit-call-threshold=1") + test_command[-2..-1] = %w[test/run.rb --ignore-name=/\Atest_no_memory_leak\z/] + end + when "debug" # needs pty next unless /mswin|mingw/ =~ RUBY_PLATFORM