Skip to content

Commit

Permalink
some cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@d-netto
d-netto committed Feb 17, 2023
1 parent cc0a355 commit b10ab4f
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 62 deletions.
58 changes: 13 additions & 45 deletions src/gc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1792,80 +1792,48 @@ STATIC_INLINE void gc_mark_push_remset(jl_ptls_t ptls, jl_value_t *obj,
// Push a work item to the queue
STATIC_INLINE void gc_markqueue_push(jl_gc_markqueue_t *mq, jl_value_t *obj) JL_NOTSAFEPOINT
{
ws_array_t *old_a = ws_queue_push(&mq->q, obj);
ws_array_t *old_a = ws_queue_push(&mq->q, &obj, sizeof(void *));
if (__unlikely(old_a != NULL))
arraylist_push(&mq->reclaim_set, old_a);
}

// Pop from the mark queue
STATIC_INLINE jl_value_t *gc_markqueue_pop(jl_gc_markqueue_t *mq)
{
return (jl_value_t *)ws_queue_pop(&mq->q);
jl_value_t *v = NULL;
ws_queue_pop(&mq->q, &v, sizeof(void *));
return v;
}

// Steal from `mq2`
STATIC_INLINE jl_value_t *gc_markqueue_steal_from(jl_gc_markqueue_t *mq2)
{
return (jl_value_t *)ws_queue_steal_from(&mq2->q);
jl_value_t *v = NULL;
ws_queue_steal_from(&mq2->q, &v, sizeof(void *));
return v;
}

// Chunk-queue functions are almost verbatim copied from `work-stealing-queue.h`.
// Could be made less repetitive with the use of an extra element size parameter
// passed to the functions in `work-stealing-queue.h`, at the expense of debuggability

// Push chunk `*c` into chunk queue
STATIC_INLINE void gc_chunkqueue_push(jl_gc_markqueue_t *mq, jl_gc_chunk_t *c) JL_NOTSAFEPOINT
{
ws_queue_t *cq = &mq->cq;
ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor);
ws_array_t *ary = jl_atomic_load_relaxed(&cq->array);
if (anc.tail == ary->capacity) {
// Resize queue
ws_array_t *new_ary = create_ws_array(2 * ary->capacity, sizeof(jl_gc_chunk_t));
memcpy(new_ary->buffer, ary->buffer, anc.tail * sizeof(jl_gc_chunk_t));
jl_atomic_store_relaxed(&cq->array, new_ary);
arraylist_push(&mq->reclaim_set, ary);
ary = new_ary;
}
((jl_gc_chunk_t *)ary->buffer)[anc.tail] = *c;
anc.tail++;
anc.tag++;
jl_atomic_store_release(&cq->anchor, anc);
ws_array_t *old_a = ws_queue_push(&mq->cq, c, sizeof(jl_gc_chunk_t));
if (__unlikely(old_a != NULL))
arraylist_push(&mq->reclaim_set, old_a);
}

// Pop chunk from chunk queue
STATIC_INLINE jl_gc_chunk_t gc_chunkqueue_pop(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT
{
ws_queue_t *cq = &mq->cq;
jl_gc_chunk_t c = {.cid = GC_empty_chunk};
ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor);
ws_array_t *ary = jl_atomic_load_relaxed(&cq->array);
if (anc.tail == 0)
// Empty queue
return c;
anc.tail--;
c = ((jl_gc_chunk_t *)ary->buffer)[anc.tail];
jl_atomic_store_release(&cq->anchor, anc);
ws_queue_steal_from(&mq->cq, &c, sizeof(jl_gc_chunk_t));
return c;
}

// Steal chunk from `mq2`
STATIC_INLINE jl_gc_chunk_t gc_chunkqueue_steal_from(jl_gc_markqueue_t *mq2) JL_NOTSAFEPOINT
{
ws_queue_t *cq = &mq2->cq;
jl_gc_chunk_t c = {.cid = GC_empty_chunk};
ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor);
ws_array_t *ary = jl_atomic_load_acquire(&cq->array);
if (anc.tail == 0)
// Empty queue
return c;
c = ((jl_gc_chunk_t *)ary->buffer)[anc.tail - 1];
ws_anchor_t anc2 = {anc.tail - 1, anc.tag};
if (!jl_atomic_cmpswap(&cq->anchor, &anc, anc2)) {
// Steal failed
c.cid = GC_empty_chunk;
return c;
}
ws_queue_steal_from(&mq2->cq, &c, sizeof(jl_gc_chunk_t));
return c;
}

Expand Down Expand Up @@ -3335,7 +3303,7 @@ void jl_init_thread_heap(jl_ptls_t ptls)
jl_atomic_store_relaxed(&cq->anchor, anc);
jl_atomic_store_relaxed(&cq->array, wsa);
ws_queue_t *q = &mq->q;
ws_array_t *wsa2 = create_ws_array(MARK_QUEUE_INIT_SIZE, sizeof(void *));
ws_array_t *wsa2 = create_ws_array(MARK_QUEUE_INIT_SIZE, sizeof(jl_gc_chunk_t));
jl_atomic_store_relaxed(&q->anchor, anc);
jl_atomic_store_relaxed(&q->array, wsa2);

Expand Down
2 changes: 1 addition & 1 deletion src/gc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ typedef struct {
} jl_gc_num_t;

typedef enum {
GC_empty_chunk,
GC_empty_chunk = 0,
GC_objary_chunk,
GC_ary8_chunk,
GC_ary16_chunk,
Expand Down
32 changes: 16 additions & 16 deletions src/work-stealing-queue.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ extern "C" {
// =======

typedef struct {
void **buffer;
char *buffer;
int64_t capacity;
} ws_array_t;

static inline ws_array_t *create_ws_array(size_t capacity, size_t eltsz) JL_NOTSAFEPOINT
{
ws_array_t *a = (ws_array_t *)malloc_s(sizeof(ws_array_t));
a->buffer = (void **)malloc_s(capacity * eltsz);
a->buffer = (char *)malloc_s(capacity * eltsz);
a->capacity = capacity;
return a;
}
Expand All @@ -38,52 +38,52 @@ typedef struct {
_Atomic(ws_array_t *) array;
} ws_queue_t;

static inline ws_array_t *ws_queue_push(ws_queue_t *q, void *elt) JL_NOTSAFEPOINT
static inline ws_array_t *ws_queue_push(ws_queue_t *q, void *elt, size_t eltsz) JL_NOTSAFEPOINT
{
ws_anchor_t anc = jl_atomic_load_acquire(&q->anchor);
ws_array_t *ary = jl_atomic_load_relaxed(&q->array);
ws_array_t *old_ary = NULL;
if (anc.tail == ary->capacity) {
// Resize queue
ws_array_t *new_ary = create_ws_array(2 * ary->capacity, sizeof(void *));
memcpy(new_ary->buffer, ary->buffer, anc.tail * sizeof(void *));
ws_array_t *new_ary = create_ws_array(2 * ary->capacity, eltsz);
memcpy(new_ary->buffer, ary->buffer, anc.tail * eltsz);
jl_atomic_store_relaxed(&q->array, new_ary);
old_ary = ary;
ary = new_ary;
}
ary->buffer[anc.tail] = elt;
memcpy(ary->buffer + anc.tail * eltsz, elt, eltsz);
anc.tail++;
anc.tag++;
jl_atomic_store_release(&q->anchor, anc);
return old_ary;
}

static inline void *ws_queue_pop(ws_queue_t *q) JL_NOTSAFEPOINT
static inline void ws_queue_pop(ws_queue_t *q, void *dest, size_t eltsz) JL_NOTSAFEPOINT
{
ws_anchor_t anc = jl_atomic_load_acquire(&q->anchor);
ws_array_t *ary = jl_atomic_load_relaxed(&q->array);
if (anc.tail == 0)
// Empty queue
return NULL;
return;
anc.tail--;
void *elt = ary->buffer[anc.tail];
memcpy(dest, ary->buffer + anc.tail * eltsz, eltsz);
jl_atomic_store_release(&q->anchor, anc);
return elt;
}

static inline void *ws_queue_steal_from(ws_queue_t *q) JL_NOTSAFEPOINT
static inline void ws_queue_steal_from(ws_queue_t *q, void *dest, size_t eltsz) JL_NOTSAFEPOINT
{
ws_anchor_t anc = jl_atomic_load_acquire(&q->anchor);
ws_array_t *ary = jl_atomic_load_acquire(&q->array);
if (anc.tail == 0)
// Empty queue
return NULL;
void *elt = ary->buffer[anc.tail - 1];
return;
memcpy(dest, ary->buffer + (anc.tail - 1) * eltsz, eltsz);
ws_anchor_t anc2 = {anc.tail - 1, anc.tag};
if (!jl_atomic_cmpswap(&q->anchor, &anc, anc2))
if (!jl_atomic_cmpswap(&q->anchor, &anc, anc2)) {
// Steal failed
return NULL;
return elt;
memset(dest, 0, eltsz);
return;
}
}

#ifdef __cplusplus
Expand Down

0 comments on commit b10ab4f

Please sign in to comment.