diff --git a/benchmark/hash_ar_aref.yml b/benchmark/hash_ar_aref.yml new file mode 100644 index 00000000000000..6cfc99e5e93cf1 --- /dev/null +++ b/benchmark/hash_ar_aref.yml @@ -0,0 +1,17 @@ +prelude: | + h = {a:1, b:2, c:3, d:4, e:5, f:6, g:7, h:8} + e = {} + +benchmark: + miss_empty: | + e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a] + hit_first: | + h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a] + hit_third: | + h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c] + hit_fourth: | + h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d] + hit_last: | + h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h] + miss: | + h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_] diff --git a/defs/gmake.mk b/defs/gmake.mk index 088de0e6774723..212c404560a636 100644 --- a/defs/gmake.mk +++ b/defs/gmake.mk @@ -198,6 +198,9 @@ $(SCRIPTBINDIR)%$(EXEEXT): bin/% $(STUBPROGRAM) \ $(SCRIPTBINDIR): $(Q) mkdir $@ +encdb.h: $(wildcard $(srcdir)/enc/*.[ch]) +transdb.h: $(wildcard $(srcdir)/enc/trans/*.trans) + .PHONY: commit COMMIT_PREPARE := $(subst :,\:,$(filter-out commit do-commit,$(MAKECMDGOALS))) up diff --git a/hash.c b/hash.c index 5a2c71f86d989a..3c33081a424d0c 100644 --- a/hash.c +++ b/hash.c @@ -607,49 +607,83 @@ ar_equal(VALUE x, VALUE y) return rb_any_cmp(x, y) == 0; } + +#if SIZEOF_VALUE == 8 +#define AR_HINT_BASE_MASK 0x101010101010101 +#define AR_HINT_NORMALIZE_MASK 0x7F7F7F7F7F7F7F7F +#ifdef WORDS_BIGENDIAN +#define AR_HINT_FIND_FIRST_ZERO_BYTE(x) (nlz_int64(x) / CHAR_BIT) +#else +#define AR_HINT_FIND_FIRST_ZERO_BYTE(x) (ntz_int64(x) / CHAR_BIT) +#endif +#else +#define AR_HINT_BASE_MASK 0x1010101 +#define AR_HINT_NORMALIZE_MASK 0x7F7F7F7F +#ifdef WORDS_BIGENDIAN +#define AR_HINT_FIND_FIRST_ZERO_BYTE(x) (nlz_int32(x) / CHAR_BIT) +#else +#define AR_HINT_FIND_FIRST_ZERO_BYTE(x) (ntz_int32(x) / CHAR_BIT) +#endif +#endif + +static inline unsigned int +ar_hint_first_match(ar_hint_t needle, VALUE haystack) +{ + // Common SWAR technique. + // First XOR all bytes so that matching ones are set to 0x00. + VALUE search_mask = AR_HINT_BASE_MASK * needle; + VALUE matches = haystack ^ search_mask; + + // Then turns 0x00 into 0x80, and any other bytes into 0x00. + matches = ~((((matches & AR_HINT_NORMALIZE_MASK) + AR_HINT_NORMALIZE_MASK) | matches) | AR_HINT_NORMALIZE_MASK); + unsigned index = AR_HINT_FIND_FIRST_ZERO_BYTE(matches); + RBIMPL_ASSERT_OR_ASSUME(index <= RHASH_AR_TABLE_MAX_SIZE); + return index; +} + // Returns the bin index if found, RHASH_AR_TABLE_MAX_BOUND if not found, // or RHASH_AR_TABLE_CONVERTED_TO_ST_TABLE if #eql? or a Thread converted the hash to st_table. static unsigned ar_find_entry_hint(VALUE hash, ar_hint_t hint, st_data_t key) { - /* if table is NULL, then bound also should be 0 */ + unsigned first_match = ar_hint_first_match(hint, RHASH_AR_TABLE(hash)->ar_hint.word); - for (unsigned i = 0; i < RHASH_AR_TABLE_BOUND(hash); i++) { - const ar_hint_t *hints = RHASH_AR_TABLE(hash)->ar_hint.ary; - if (hints[i] == hint) { - ar_table_pair *pair = RHASH_AR_TABLE_REF(hash, i); - int eq = ar_equal(key, pair->key); - if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) { - return RHASH_AR_TABLE_CONVERTED_TO_ST_TABLE; - } - if (eq) { - RB_DEBUG_COUNTER_INC(artable_hint_hit); - return i; - } - else { -#if 0 - static int pid; - static char fname[256]; - static FILE *fp; + if (LIKELY(first_match >= RHASH_AR_TABLE_BOUND(hash))) { + RB_DEBUG_COUNTER_INC(artable_hint_notfound); + return RHASH_AR_TABLE_MAX_BOUND; + } - if (pid != getpid()) { - snprintf(fname, sizeof(fname), "/tmp/ruby-armiss.%d", pid = getpid()); - if ((fp = fopen(fname, "w")) == NULL) rb_bug("fopen"); + RUBY_ASSERT(RHASH_AR_TABLE(hash)->ar_hint.ary[first_match] == hint); + int eq = ar_equal(key, RHASH_AR_TABLE_REF(hash, first_match)->key); + if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) { + return RHASH_AR_TABLE_CONVERTED_TO_ST_TABLE; + } + if (LIKELY(eq)) { + RB_DEBUG_COUNTER_INC(artable_hint_hit); + return first_match; + } + else { + // In theory we could extract all the matching indexes in `ar_hint_first_match`, + // and avoid this loop, but sine `ar_equal` may call back into arbitrary code, + // the `ar_hint` may have changed. + for (unsigned i = first_match + 1; i < RHASH_AR_TABLE_BOUND(hash); i++) { + const ar_hint_t *hints = RHASH_AR_TABLE(hash)->ar_hint.ary; + if (UNLIKELY(hints[i] == hint)) { + eq = ar_equal(key, RHASH_AR_TABLE_REF(hash, i)->key); + if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) { + return RHASH_AR_TABLE_CONVERTED_TO_ST_TABLE; + } + if (eq) { + RB_DEBUG_COUNTER_INC(artable_hint_hit); + return i; + } + else { + RB_DEBUG_COUNTER_INC(artable_hint_miss); } - - st_hash_t h1 = ar_do_hash(key); - st_hash_t h2 = ar_do_hash(pair->key); - - fprintf(fp, "miss: hash_eq:%d hints[%d]:%02x hint:%02x\n" - " key :%016lx %s\n" - " pair->key:%016lx %s\n", - h1 == h2, i, hints[i], hint, - h1, rb_obj_info(key), h2, rb_obj_info(pair->key)); -#endif - RB_DEBUG_COUNTER_INC(artable_hint_miss); } } } + RB_DEBUG_COUNTER_INC(artable_hint_notfound); return RHASH_AR_TABLE_MAX_BOUND; } diff --git a/internal/hash.h b/internal/hash.h index 0386a5009c08ea..a91a18785ab570 100644 --- a/internal/hash.h +++ b/internal/hash.h @@ -203,7 +203,7 @@ RHASH_AR_TABLE_BOUND(VALUE h) { RUBY_ASSERT(RHASH_AR_TABLE_P(h)); const unsigned int bound = RHASH_AR_TABLE_BOUND_RAW(h); - RUBY_ASSERT(bound <= RHASH_AR_TABLE_MAX_SIZE); + RBIMPL_ASSERT_OR_ASSUME(bound <= RHASH_AR_TABLE_MAX_SIZE); return bound; }