loongarch: sljit_emit_atomic_load/store implementation

API_CHANGES

@@ -1,5 +1,10 @@
 This file is the short summary of the API changes:
 
+10.06.2023 - Backward compatible
+    Emulation for atomic operations in narrow bitwidths is
+    configurable at build time, SLJIT_HAS_ATOMIC_{8,16}BIT
+    provide availability at runtime.
+
 16.02.2022 - Non-backward compatible
     The sljit_emit_cmov operation is replaced
     by sljit_emit_select.

sljit_src/sljitConfig.h

@@ -148,6 +148,11 @@ extern "C" {
 #define SLJIT_VERBOSE 1
 #endif
 
+/* Atomic emulation level */
+#ifndef SLJIT_ATOMIC_EMULATION
+#define SLJIT_ATOMIC_EMULATION 0
+#endif
+
 /*
   SLJIT_IS_FPU_AVAILABLE
     The availability of the FPU can be controlled by SLJIT_IS_FPU_AVAILABLE.

sljit_src/sljitConfigInternal.h

@@ -779,13 +779,14 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr);
 
 #elif (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
 
-#define SLJIT_NUMBER_OF_REGISTERS 23
+#define SLJIT_NUMBER_OF_REGISTERS 22
 #define SLJIT_NUMBER_OF_SAVED_REGISTERS 10
 #define SLJIT_LOCALS_OFFSET_BASE 0
 #define SLJIT_NUMBER_OF_FLOAT_REGISTERS 30
 #define SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS 12
 #define SLJIT_MASKED_SHIFT 1
 #define SLJIT_MASKED_SHIFT32 1
+#define SLJIT_ATOMIC_WIDTH 32
 
 #elif (defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
 

sljit_src/sljitLir.h

@@ -658,6 +658,10 @@ static SLJIT_INLINE sljit_uw sljit_get_generated_code_size(struct sljit_compiler
 #define SLJIT_HAS_COPY_F32		9
 /* [Emulated] Copy from/to f64 operation is available (see sljit_emit_fcopy). */
 #define SLJIT_HAS_COPY_F64		10
+/* [Emulated] Loading/Storing 8bit atomics is available. */
+#define SLJIT_HAS_ATOMIC_8BIT		11
+/* [Emulated] Loading/Storing 16bit atomics is available. */
+#define SLJIT_HAS_ATOMIC_16BIT		12
 
 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
 /* [Not emulated] SSE2 support is available on x86. */
@@ -1804,7 +1808,11 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler
      - the memory operation (op) and the base address (stored in mem_reg)
        passed to the load/store operations must be the same (the mem_reg
        can be a different register, only its value must be the same)
-     - an store must always follow a load for the same transaction.
+     - an store must always follow a load for the same transaction, but
+       loads might be abandoned
+     - if the CPU defines a minimum bit width supported (SLJIT_ATOMIC_WIDTH)
+       then the memory address must be aligned to it; alternatively smaller
+       atomic types can be supported by enabling SLJIT_ATOMIC_EMULATION
 
    op must be between SLJIT_MOV and SLJIT_MOV_P, excluding all
      signed loads such as SLJIT_MOV32_S16

sljit_src/sljitNativeARM_32.c

@@ -974,6 +974,8 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
 	case SLJIT_HAS_PREFETCH:
 	case SLJIT_HAS_COPY_F32:
 	case SLJIT_HAS_COPY_F64:
+	case SLJIT_HAS_ATOMIC_8BIT:
+	case SLJIT_HAS_ATOMIC_16BIT:
 		return 1;
 
 	default:

sljit_src/sljitNativeARM_64.c

@@ -423,6 +423,8 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
 	case SLJIT_HAS_PREFETCH:
 	case SLJIT_HAS_COPY_F32:
 	case SLJIT_HAS_COPY_F64:
+	case SLJIT_HAS_ATOMIC_8BIT:
+	case SLJIT_HAS_ATOMIC_16BIT:
 		return 1;
 
 	default:

sljit_src/sljitNativeARM_T2_32.c

@@ -514,6 +514,8 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
 	case SLJIT_HAS_PREFETCH:
 	case SLJIT_HAS_COPY_F32:
 	case SLJIT_HAS_COPY_F64:
+	case SLJIT_HAS_ATOMIC_8BIT:
+	case SLJIT_HAS_ATOMIC_16BIT:
 		return 1;
 
 	default:

sljit_src/sljitNativeLOONGARCH_64.c

@@ -36,19 +36,20 @@ typedef sljit_u32 sljit_ins;
 #define TMP_REG1	(SLJIT_NUMBER_OF_REGISTERS + 2)
 #define TMP_REG2	(SLJIT_NUMBER_OF_REGISTERS + 3)
 #define TMP_REG3	(SLJIT_NUMBER_OF_REGISTERS + 4)
+#define TMP_REG4	(SLJIT_NUMBER_OF_REGISTERS + 5)
 #define TMP_ZERO	0
 
 /* Flags are kept in volatile registers. */
-#define EQUAL_FLAG	(SLJIT_NUMBER_OF_REGISTERS + 5)
+#define EQUAL_FLAG	(SLJIT_NUMBER_OF_REGISTERS + 6)
 #define RETURN_ADDR_REG	TMP_REG2
-#define OTHER_FLAG	(SLJIT_NUMBER_OF_REGISTERS + 6)
+#define OTHER_FLAG	(SLJIT_NUMBER_OF_REGISTERS + 7)
 
 #define TMP_FREG1	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
 #define TMP_FREG2	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
 
 
-static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 7] = {
-	0, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19, 20, 22, 31, 30, 29, 28, 27, 26, 25, 24, 23, 3, 13, 1, 14, 12, 15
+static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 8] = {
+	0, 4, 5, 6, 7, 8, 9, 10, 11, 17, 18, 19, 20, 22, 31, 30, 29, 28, 27, 26, 25, 24, 23, 3, 13, 1, 14, 16, 12, 15
 };
 
 static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
@@ -62,7 +63,7 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
 /*
 LoongArch instructions are 32 bits wide, belonging to 9 basic instruction formats (and variants of them):
 
-| Format name  | Composition 				 |
+| Format name  | Composition                 |
 | 2R           | Opcode + Rj + Rd            |
 | 3R           | Opcode + Rk + Rj + Rd       |
 | 4R           | Opcode + Ra + Rk + Rj + Rd  |
@@ -642,6 +643,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
 	case SLJIT_HAS_COPY_F64:
 		return 1;
 
+	case SLJIT_HAS_ATOMIC_8BIT:
+	case SLJIT_HAS_ATOMIC_16BIT:
+		return SLJIT_ATOMIC_EMULATION ? 2 : 0;
+
 	default:
 		return 0;
 	}
@@ -2449,8 +2454,10 @@ static sljit_ins get_jump_instruction(sljit_s32 type)
 {
 	switch (type) {
 	case SLJIT_EQUAL:
+	case SLJIT_ATOMIC_NOT_STORED:
 		return BNE | RJ(EQUAL_FLAG) | RD(TMP_ZERO);
 	case SLJIT_NOT_EQUAL:
+	case SLJIT_ATOMIC_STORED:
 		return BEQ | RJ(EQUAL_FLAG) | RD(TMP_ZERO);
 	case SLJIT_LESS:
 	case SLJIT_GREATER:
@@ -2734,6 +2741,12 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *co
 			FAIL_IF(push_inst(compiler, SLTUI | RD(dst_r) | RJ(EQUAL_FLAG) | IMM_I12(1)));
 			src_r = dst_r;
 			break;
+		case SLJIT_ATOMIC_STORED:
+		case SLJIT_ATOMIC_NOT_STORED:
+			FAIL_IF(push_inst(compiler, SLTUI | RD(dst_r) | RJ(EQUAL_FLAG) | IMM_I12(1)));
+			src_r = dst_r;
+			invert ^= 0x1;
+			break;
 		case SLJIT_OVERFLOW:
 		case SLJIT_NOT_OVERFLOW:
 			if (compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)) {
@@ -2933,15 +2946,52 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler
 	sljit_s32 dst_reg,
 	sljit_s32 mem_reg)
 {
-	SLJIT_UNUSED_ARG(compiler);
-	SLJIT_UNUSED_ARG(op);
-	SLJIT_UNUSED_ARG(dst_reg);
-	SLJIT_UNUSED_ARG(mem_reg);
+	sljit_ins ins = LL_W;
+	sljit_s32 dst = TMP_REG2;
+	sljit_s32 mem = mem_reg;
 
 	CHECK_ERROR();
 	CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
 
-	return SLJIT_ERR_UNSUPPORTED;
+	op = GET_OPCODE(op);
+	switch (op) {
+#if SLJIT_ATOMIC_EMULATION
+	case SLJIT_MOV_U8:
+	case SLJIT_MOV_U16:
+		mem = TMP_REG1;
+		FAIL_IF(push_inst(compiler, ANDI | RD(TMP_REG3) | RJ(mem_reg) | IMM_I12(0x3)));
+		FAIL_IF(push_inst(compiler, XOR | RD(mem) | RJ(TMP_REG3) | RK(mem_reg)));
+		break;
+#endif /* SLJIT_ATOMIC_EMULATION */
+	case SLJIT_MOV_P:
+	case SLJIT_MOV:
+		ins = LL_D;
+		dst = dst_reg;
+		break;
+	}
+
+	FAIL_IF(push_inst(compiler, ins | RD(dst) | RJ(mem)));
+
+	switch (op) {
+	case SLJIT_MOV_U8:
+#if SLJIT_ATOMIC_EMULATION
+		FAIL_IF(push_inst(compiler, SLLI_W | RD(TMP_REG3) | RJ(TMP_REG3) | IMM_I12(0x3)));
+		FAIL_IF(push_inst(compiler, SRL_W | RD(dst) | RJ(dst) | RK(TMP_REG3)));
+#endif /* SLJIT_ATOMIC_EMULATION */
+		FAIL_IF(push_inst(compiler, ANDI | RD(dst_reg) | RJ(dst) | IMM_I12(0xff)));
+		break;
+	case SLJIT_MOV_U16:
+#if SLJIT_ATOMIC_EMULATION
+		FAIL_IF(push_inst(compiler, BEQZ | RJ(TMP_REG3) | IMM_I21(1 + 2)));
+		FAIL_IF(push_inst(compiler, BSTRPICK_W | RD(dst_reg) | RJ(dst) | (31 << 16) | (16 << 10)));
+		FAIL_IF(push_inst(compiler, B | IMM_I26(1 + 1)));
+#endif /* SLJIT_ATOMIC_EMULATION */
+		return push_inst(compiler, BSTRPICK_W | RD(dst_reg) | RJ(dst) | (15 << 16));
+	case SLJIT_MOV_U32:
+	case SLJIT_MOV32:
+		return push_inst(compiler, BSTRPICK_D | RD(dst_reg) | RJ(dst) | (31 << 16));
+	}
+	return SLJIT_SUCCESS;
 }
 
 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler,
@@ -2950,16 +3000,65 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler
 	sljit_s32 mem_reg,
 	sljit_s32 temp_reg)
 {
-	SLJIT_UNUSED_ARG(compiler);
-	SLJIT_UNUSED_ARG(op);
-	SLJIT_UNUSED_ARG(src_reg);
-	SLJIT_UNUSED_ARG(mem_reg);
-	SLJIT_UNUSED_ARG(temp_reg);
+	sljit_ins ins = SC_W;
+	sljit_ins chk = ORI | RD(EQUAL_FLAG) | RJ(temp_reg) | RK(TMP_ZERO);
+	sljit_s32 mem = mem_reg;
 
 	CHECK_ERROR();
 	CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
 
-	return SLJIT_ERR_UNSUPPORTED;
+	op = GET_OPCODE(op);
+
+#if SLJIT_ATOMIC_EMULATION
+	switch (op) {
+	case SLJIT_MOV_U8:
+	case SLJIT_MOV_U16:
+		mem = TMP_REG1;
+		FAIL_IF(push_inst(compiler, ANDI | RD(TMP_REG3) | RJ(mem_reg) | IMM_I12(0x3)));
+		FAIL_IF(push_inst(compiler, XOR | RD(mem) | RJ(TMP_REG3) | RK(mem_reg)));
+	}
+#endif /* SLJIY_ATOMIC_EMULATION */
+
+	switch (op) {
+	case SLJIT_MOV_U8: {
+		sljit_s32 dst = temp_reg;
+#if SLJIT_ATOMIC_EMULATION
+		dst = TMP_REG4;
+		FAIL_IF(push_inst(compiler, SLLI_W | RD(TMP_REG3) | RJ(TMP_REG3) | IMM_I12(3)));
+		FAIL_IF(push_inst(compiler, LD_WU | RD(temp_reg) | RJ(mem)));
+		FAIL_IF(push_inst(compiler, ADDI_W | RD(TMP_REG2) | RJ(TMP_ZERO) | IMM_I12(0xff)));
+		FAIL_IF(push_inst(compiler, SLL_W | RD(TMP_REG2) | RJ(TMP_REG2) | RK(TMP_REG3)));
+		FAIL_IF(push_inst(compiler, ANDN | RD(temp_reg) | RJ(temp_reg) | RK(TMP_REG2)));
+#endif /* SLJIT_ATOMIC_EMULATION */
+		FAIL_IF(push_inst(compiler, ANDI | RD(dst) | RJ(src_reg) | IMM_I12(0xff)));
+#if SLJIT_ATOMIC_EMULATION
+		FAIL_IF(push_inst(compiler, SLL_W | RD(TMP_REG4) | RJ(src_reg) | RK(TMP_REG3)));
+		FAIL_IF(push_inst(compiler, OR | RD(temp_reg) | RJ(temp_reg) | RK(TMP_REG4)));
+#endif /* SLJIT_ATOMIC_EMULATION */
+		break; }
+	case SLJIT_MOV_U16:
+#if SLJIT_ATOMIC_EMULATION
+		FAIL_IF(push_inst(compiler, LD_WU | RD(temp_reg) | RJ(mem)));
+		FAIL_IF(push_inst(compiler, BNEZ | RJ(TMP_REG3) | IMM_I21(1 + 2)));
+#endif /* SLJIT_ATOMIC_EMULATION */
+		FAIL_IF(push_inst(compiler, BSTRINS_W | RD(temp_reg) | RJ(src_reg) | (15 << 16)));
+#if SLJIT_ATOMIC_EMULATION
+		FAIL_IF(push_inst(compiler, B | IMM_I26(1 + 1)));
+		FAIL_IF(push_inst(compiler, BSTRINS_W | RD(temp_reg) | RJ(src_reg) | (31 << 16) | (16 << 10)));
+#endif /* SLJIT_ATOMIC_EMULATION */
+		break;
+	case SLJIT_MOV_P:
+	case SLJIT_MOV:
+		ins = SC_D;
+		/* FALLTHRU */
+	default:
+		FAIL_IF(push_inst(compiler, ORI | RD(temp_reg) | RJ(src_reg) | RK(TMP_ZERO)));
+		break;
+	}
+
+	FAIL_IF(push_inst(compiler, ins | RD(temp_reg) | RJ(mem)));
+
+	return chk ? push_inst(compiler, chk) : SLJIT_SUCCESS;
 }
 
 static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw init_value, sljit_ins last_ins)

sljit_src/sljitNativeS390X.c

@@ -1663,6 +1663,8 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
 	case SLJIT_HAS_COPY_F32:
 	case SLJIT_HAS_COPY_F64:
 		return 1;
+	case SLJIT_HAS_ATOMIC_8BIT:
+	case SLJIT_HAS_ATOMIC_16BIT:
 	case SLJIT_HAS_CTZ:
 		return 2;
 	case SLJIT_HAS_CMOV:

sljit_src/sljitNativeX86_common.c

@@ -789,6 +789,8 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
 	case SLJIT_HAS_PREFETCH:
 	case SLJIT_HAS_COPY_F32:
 	case SLJIT_HAS_COPY_F64:
+	case SLJIT_HAS_ATOMIC_8BIT:
+	case SLJIT_HAS_ATOMIC_16BIT:
 		return 1;
 
 	case SLJIT_HAS_SSE2: