diff --git a/config/craftax_clean.ini b/config/craftax_clean.ini index 70d5427d39..b560b9461b 100644 --- a/config/craftax_clean.ini +++ b/config/craftax_clean.ini @@ -2,7 +2,7 @@ env_name = craftax_clean [vec] -total_agents = 16384 +total_agents = 2048 num_buffers = 16 num_threads = 16 @@ -11,23 +11,23 @@ reset_pool_size = 32 action_mask = 0 [train] -total_timesteps = 2_427_869_180 -learning_rate = 0.00577104278 -ent_coef = 0.0000466404454 -gamma = 0.998501122 -gae_lambda = 0.900757194 -horizon = 128 +total_timesteps = 1_872_097_150 +learning_rate = 0.00179971103 +ent_coef = 0.0000203385534 +gamma = 0.997127712 +gae_lambda = 0.960012317 +horizon = 64 minibatch_size = 65536 -replay_ratio = 1.06381893 -clip_coef = 0.2003396 -vf_coef = 1.89120328 +replay_ratio = 3.26567292 +clip_coef = 0.207467377 +vf_coef = 2.66733265 vf_clip_coef = 0.01 -max_grad_norm = 0.1 -momentum = 0.865953863 +max_grad_norm = 0.496576279 +momentum = 0.961853921 [policy] -hidden_size = 256 -num_layers = 2 +hidden_size = 1024 +num_layers = 4 [sweep] metric = score diff --git a/ocean/craftax/pack_textures.py b/ocean/craftax/pack_textures.py index 739d230256..bc949a8da5 100644 --- a/ocean/craftax/pack_textures.py +++ b/ocean/craftax/pack_textures.py @@ -22,6 +22,10 @@ [88..90] passive types: cow, bat, snail [91..98] ranged types: skeleton, gnome_archer, orc_mage, kobold, knight_archer, deep_thing, fire_elemental, ice_elemental [99..102] projectiles: dagger, fireball, iceball, slimeball + [103..104] sword enchant overlays: fire, ice + [105..106] arrow enchant overlays: fire, ice + [107..110] armour fire overlays: helmet, chest, pants, boots + [111..114] armour ice overlays: helmet, chest, pants, boots """ import os @@ -213,6 +217,21 @@ def find_assets() -> Path: "iceball.png", "slimeball.png", ] +# HUD enchant overlays. Level 0 is empty (no tile); fire then ice. +ENCHANT_FILES = [ + "sword_fire_enchantment.png", + "sword_ice_enchantment.png", + "arrow_fire_enchantment.png", + "arrow_ice_enchantment.png", + "helmet_fire_enchantment.png", + "chestplate_fire_enchantment.png", + "pants_fire_enchantment.png", + "boots_fire_enchantment.png", + "helmet_ice_enchantment.png", + "chestplate_ice_enchantment.png", + "pants_ice_enchantment.png", + "boots_ice_enchantment.png", +] def load_tile(name: str | None) -> np.ndarray: @@ -250,7 +269,7 @@ def main() -> None: MOB_FILES + ARROW_FILES + ARMOUR_FILES + WEAPON_FILES + POTION_FILES + HUD_ITEM_FILES + MELEE_TYPE_FILES + PASSIVE_TYPE_FILES + RANGED_TYPE_FILES - + PROJECTILE_TYPE_FILES + + PROJECTILE_TYPE_FILES + ENCHANT_FILES ): tiles.append(load_tile(f)) diff --git a/ocean/craftax_clean/constants.h b/ocean/craftax_clean/constants.h index 93171ad4b7..54c1d40a8f 100644 --- a/ocean/craftax_clean/constants.h +++ b/ocean/craftax_clean/constants.h @@ -453,7 +453,7 @@ static const DungeonConfig DUNGEON_LEVEL_CONFIGS[3] = { #define TEX_TILE_PX 16 #define TEX_SCALE 3 #define TEX_DRAW_PX (TEX_TILE_PX * TEX_SCALE) -#define TEX_NUM (37 + 5 + 5 + 3 + 4 + 8 + 9 + 6 + 3 + 8 + 3 + 8 + 4) +#define TEX_NUM (37 + 5 + 5 + 3 + 4 + 8 + 9 + 6 + 3 + 8 + 3 + 8 + 4 + 12) #define RENDER_ROWS 14 #define RENDER_COLS 16 #define ACTION_PANEL_W 280 @@ -488,3 +488,9 @@ static const DungeonConfig DUNGEON_LEVEL_CONFIGS[3] = { #define TEX_PROJ_FIREBALL 100 #define TEX_PROJ_ICEBALL 101 #define TEX_PROJ_SLIMEBALL 102 +#define TEX_SWORD_ENCHANT_FIRE 103 +#define TEX_SWORD_ENCHANT_ICE 104 +#define TEX_ARROW_ENCHANT_FIRE 105 +#define TEX_ARROW_ENCHANT_ICE 106 +#define TEX_ARMOUR_ENCHANT_FIRE 107 +#define TEX_ARMOUR_ENCHANT_ICE 111 diff --git a/ocean/craftax_clean/craftax_clean.c b/ocean/craftax_clean/craftax_clean.c index 39372e66b1..eba7cb850d 100644 --- a/ocean/craftax_clean/craftax_clean.c +++ b/ocean/craftax_clean/craftax_clean.c @@ -1,55 +1,83 @@ -#include "craftax_clean.h" +#ifdef CRAFTAX_CLEAN_IN_HEADER static int key_to_action(void) { - // Makes it human playable - if (IsKeyPressed(KEY_Q)) return ACTION_NOOP; - if (IsKeyPressed(KEY_W) || IsKeyPressed(KEY_UP)) return ACTION_UP; - if (IsKeyPressed(KEY_D) || IsKeyPressed(KEY_RIGHT)) return ACTION_RIGHT; - if (IsKeyPressed(KEY_S) || IsKeyPressed(KEY_DOWN)) return ACTION_DOWN; - if (IsKeyPressed(KEY_A) || IsKeyPressed(KEY_LEFT)) return ACTION_LEFT; - if (IsKeyPressed(KEY_SPACE)) return ACTION_DO; - if (IsKeyPressed(KEY_ONE)) return ACTION_MAKE_WOOD_PICKAXE; - if (IsKeyPressed(KEY_TWO)) return ACTION_MAKE_STONE_PICKAXE; - if (IsKeyPressed(KEY_THREE)) return ACTION_MAKE_IRON_PICKAXE; - if (IsKeyPressed(KEY_FOUR)) return ACTION_MAKE_DIAMOND_PICKAXE; - if (IsKeyPressed(KEY_FIVE)) return ACTION_MAKE_WOOD_SWORD; - if (IsKeyPressed(KEY_SIX)) return ACTION_MAKE_STONE_SWORD; - if (IsKeyPressed(KEY_SEVEN)) return ACTION_MAKE_IRON_SWORD; - if (IsKeyPressed(KEY_EIGHT)) return ACTION_MAKE_DIAMOND_SWORD; - if (IsKeyPressed(KEY_T)) return ACTION_PLACE_TABLE; - if (IsKeyPressed(KEY_TAB)) return ACTION_SLEEP; - if (IsKeyPressed(KEY_R)) return ACTION_PLACE_STONE; - if (IsKeyPressed(KEY_F)) return ACTION_PLACE_FURNACE; - if (IsKeyPressed(KEY_P)) return ACTION_PLACE_PLANT; - if (IsKeyPressed(KEY_E)) return ACTION_REST; - if (IsKeyPressed(KEY_COMMA)) return ACTION_ASCEND; - if (IsKeyPressed(KEY_PERIOD)) return ACTION_DESCEND; - if (IsKeyPressed(KEY_Y)) return ACTION_MAKE_IRON_ARMOUR; - if (IsKeyPressed(KEY_U)) return ACTION_MAKE_DIAMOND_ARMOUR; - if (IsKeyPressed(KEY_I)) return ACTION_SHOOT_ARROW; - if (IsKeyPressed(KEY_O)) return ACTION_MAKE_ARROW; - if (IsKeyPressed(KEY_G)) return ACTION_CAST_FIREBALL; - if (IsKeyPressed(KEY_H)) return ACTION_CAST_ICEBALL; - if (IsKeyPressed(KEY_J)) return ACTION_PLACE_TORCH; - if (IsKeyPressed(KEY_Z)) return ACTION_DRINK_POTION_RED; - if (IsKeyPressed(KEY_X)) return ACTION_DRINK_POTION_GREEN; - if (IsKeyPressed(KEY_C)) return ACTION_DRINK_POTION_BLUE; - if (IsKeyPressed(KEY_V)) return ACTION_DRINK_POTION_PINK; - if (IsKeyPressed(KEY_B)) return ACTION_DRINK_POTION_CYAN; - if (IsKeyPressed(KEY_N)) return ACTION_DRINK_POTION_YELLOW; - if (IsKeyPressed(KEY_M)) return ACTION_READ_BOOK; - if (IsKeyPressed(KEY_K)) return ACTION_ENCHANT_SWORD; - if (IsKeyPressed(KEY_L)) return ACTION_ENCHANT_ARMOUR; - if (IsKeyPressed(KEY_LEFT_BRACKET)) return ACTION_MAKE_TORCH; - if (IsKeyPressed(KEY_RIGHT_BRACKET)) return ACTION_LEVEL_UP_DEXTERITY; - if (IsKeyPressed(KEY_MINUS)) return ACTION_LEVEL_UP_STRENGTH; - if (IsKeyPressed(KEY_EQUAL)) return ACTION_LEVEL_UP_INTELLIGENCE; - if (IsKeyPressed(KEY_SEMICOLON)) return ACTION_ENCHANT_BOW; + static const int map[][2] = { + {KEY_Q, ACTION_NOOP}, + {KEY_W, ACTION_UP}, + {KEY_UP, ACTION_UP}, + {KEY_D, ACTION_RIGHT}, + {KEY_RIGHT, ACTION_RIGHT}, + {KEY_S, ACTION_DOWN}, + {KEY_DOWN, ACTION_DOWN}, + {KEY_A, ACTION_LEFT}, + {KEY_LEFT, ACTION_LEFT}, + {KEY_SPACE, ACTION_DO}, + {KEY_ONE, ACTION_MAKE_WOOD_PICKAXE}, + {KEY_TWO, ACTION_MAKE_STONE_PICKAXE}, + {KEY_THREE, ACTION_MAKE_IRON_PICKAXE}, + {KEY_FOUR, ACTION_MAKE_DIAMOND_PICKAXE}, + {KEY_FIVE, ACTION_MAKE_WOOD_SWORD}, + {KEY_SIX, ACTION_MAKE_STONE_SWORD}, + {KEY_SEVEN, ACTION_MAKE_IRON_SWORD}, + {KEY_EIGHT, ACTION_MAKE_DIAMOND_SWORD}, + {KEY_T, ACTION_PLACE_TABLE}, + {KEY_TAB, ACTION_SLEEP}, + {KEY_R, ACTION_PLACE_STONE}, + {KEY_F, ACTION_PLACE_FURNACE}, + {KEY_P, ACTION_PLACE_PLANT}, + {KEY_E, ACTION_REST}, + {KEY_COMMA, ACTION_ASCEND}, + {KEY_PERIOD, ACTION_DESCEND}, + {KEY_Y, ACTION_MAKE_IRON_ARMOUR}, + {KEY_U, ACTION_MAKE_DIAMOND_ARMOUR}, + {KEY_I, ACTION_SHOOT_ARROW}, + {KEY_O, ACTION_MAKE_ARROW}, + {KEY_G, ACTION_CAST_FIREBALL}, + {KEY_H, ACTION_CAST_ICEBALL}, + {KEY_J, ACTION_PLACE_TORCH}, + {KEY_Z, ACTION_DRINK_POTION_RED}, + {KEY_X, ACTION_DRINK_POTION_GREEN}, + {KEY_C, ACTION_DRINK_POTION_BLUE}, + {KEY_V, ACTION_DRINK_POTION_PINK}, + {KEY_B, ACTION_DRINK_POTION_CYAN}, + {KEY_N, ACTION_DRINK_POTION_YELLOW}, + {KEY_M, ACTION_READ_BOOK}, + {KEY_K, ACTION_ENCHANT_SWORD}, + {KEY_L, ACTION_ENCHANT_ARMOUR}, + {KEY_LEFT_BRACKET, ACTION_MAKE_TORCH}, + {KEY_RIGHT_BRACKET, ACTION_LEVEL_UP_DEXTERITY}, + {KEY_MINUS, ACTION_LEVEL_UP_STRENGTH}, + {KEY_EQUAL, ACTION_LEVEL_UP_INTELLIGENCE}, + {KEY_SEMICOLON, ACTION_ENCHANT_BOW}, + }; + for (int i = 0; i < (int)(sizeof(map) / sizeof(map[0])); i++) { + if (IsKeyPressed(map[i][0])) { + return map[i][1]; + } + } return -1; } -int main(void) { +// Shift + action-panel key. 1 = applied, 0 = policy, -1 = skip tick. +int craftax_clean_human_controls(Craftax* env) { + int shift = IsKeyDown(KEY_LEFT_SHIFT) || IsKeyDown(KEY_RIGHT_SHIFT); + if (!IsWindowReady() || !shift + || env->state.is_sleeping || env->state.is_resting) { + return 0; + } + int action = key_to_action(); + if (action < 0) { + return -1; + } + env->agents[0].actions[0] = (float)action; + return 1; +} +#else + +#include "craftax_clean.h" + +int main(void) { Craftax env; memset(&env, 0, sizeof(env)); env.num_agents = 1; @@ -61,12 +89,11 @@ int main(void) { env.agents[0].actions = (float*)calloc(1, sizeof(float)); env.agents[0].rewards = (float*)calloc(1, sizeof(float)); env.agents[0].terminals = (float*)calloc(1, sizeof(float)); - env.agents[0].action_mask = (unsigned char*)calloc(ATN_DIM, sizeof(unsigned char)); + env.agents[0].action_mask = (unsigned char*)calloc(ATN_DIM, 1); puf_reset(&env); env.agents[0].actions[0] = -1.0f; puf_render(&env); - while (!WindowShouldClose()) { int action = key_to_action(); if (action < 0 && (env.state.is_sleeping || env.state.is_resting)) { @@ -76,7 +103,6 @@ int main(void) { puf_render(&env); continue; } - env.agents[0].actions[0] = (float)action; puf_step(&env); puf_render(&env); @@ -90,3 +116,5 @@ int main(void) { free(env.agents[0].action_mask); return 0; } + +#endif diff --git a/ocean/craftax_clean/craftax_clean.h b/ocean/craftax_clean/craftax_clean.h index f94c410ba8..266093d830 100644 --- a/ocean/craftax_clean/craftax_clean.h +++ b/ocean/craftax_clean/craftax_clean.h @@ -1,4 +1,5 @@ // Full native Craftax port. +#pragma once #include #include @@ -208,6 +209,7 @@ struct Env { State* reset_pool; int reset_pool_size; int use_action_mask; + float predicted_value; }; Rng rng_seed(uint32_t seed) { @@ -1802,6 +1804,9 @@ void update_log_state(Craftax* env) { } } +#define CRAFTAX_CLEAN_IN_HEADER +#include "craftax_clean.c" + void puf_reset(Craftax* env) { env->agents[0].rewards[0] = 0.0f; env->agents[0].terminals[0] = 0.0f; @@ -1829,6 +1834,9 @@ void puf_reset(Craftax* env) { } void puf_step(Craftax* env) { + if (craftax_clean_human_controls(env) < 0) { + return; + } CLEAN_PROF_START(); env->agents[0].rewards[0] = 0.0f; env->agents[0].terminals[0] = 0.0f; @@ -2926,6 +2934,20 @@ static void draw_agent_obs(Craftax* env, int panel_x, int panel_y, player_tex = TEX_PLAYER_UP; } draw_tile(player_tex, grid_x + rc * px, grid_y + rr * px, px); + + float v = env->predicted_value; + float t = clampf(v, -1.0f, 1.0f); + unsigned char fade = (unsigned char)(255.0f * (1.0f - fabsf(t))); + Color vc = WHITE; + if (t < 0.0f) { + vc = (Color){255, fade, fade, 255}; + } else if (t > 0.0f) { + vc = (Color){fade, 255, fade, 255}; + } + const char* vlabel = TextFormat("V(o, h) = %.2f", v); + int tw = MeasureText(vlabel, 18); + DrawText(vlabel, panel_x + (panel_w - tw) / 2, + grid_y + OBS_ROWS * px + 10, 18, vc); } static void draw_icon_count(int tex_id, int value, int x, int y) { @@ -2933,12 +2955,15 @@ static void draw_icon_count(int tex_id, int value, int x, int y) { DrawText(TextFormat("%d", value), x + 23, y + 4, 14, RAYWHITE); } -static void draw_inv_slot(int tex_id, int x, int y) { +static void draw_inv_slot(int tex_id, int overlay, int x, int y) { DrawRectangle(x, y, 24, 24, (Color){32, 32, 32, 255}); DrawRectangleLines(x, y, 24, 24, (Color){80, 80, 80, 255}); if (tex_id >= 0) { draw_tile(tex_id, x, y, 24); } + if (overlay >= 0) { + draw_tile(overlay, x, y, 24); + } } static const char* craftax_clean_action_names[ATN_DIM] = { @@ -3054,6 +3079,7 @@ void puf_render(Craftax* env) { if (IsKeyDown(KEY_ESCAPE)) { exit(0); } + craftax_clean_human_controls(env); int level = clampi(env->state.player_level, 0, NUM_LEVELS - 1); int player_row = clampi(env->state.player_position[0], 0, MAP_SIZE - 1); @@ -3241,7 +3267,14 @@ void puf_render(Craftax* env) { if (alvl > 0) { tex = (alvl >= 2 ? TEX_ARMOUR_DIAMOND : TEX_ARMOUR_IRON) + slot; } - draw_inv_slot(tex, armour_x + slot * 30, inv_y); + int overlay = -1; + int ench = env->state.armour_enchantments[slot]; + if (ench == 1) { + overlay = TEX_ARMOUR_ENCHANT_FIRE + slot; + } else if (ench == 2) { + overlay = TEX_ARMOUR_ENCHANT_ICE + slot; + } + draw_inv_slot(tex, overlay, armour_x + slot * 30, inv_y); } int weap_y = hud_y + 90; int gear[] = { @@ -3250,25 +3283,44 @@ void puf_render(Craftax* env) { inv->bow > 0 ? TEX_BOW : -1, inv->arrows > 0 ? TEX_ARROW_UP : -1, }; + int overlays[4] = {-1, -1, -1, -1}; + if (env->state.sword_enchantment == 1) { + overlays[1] = TEX_SWORD_ENCHANT_FIRE; + } else if (env->state.sword_enchantment == 2) { + overlays[1] = TEX_SWORD_ENCHANT_ICE; + } + if (inv->arrows > 0) { + if (env->state.bow_enchantment == 1) { + overlays[3] = TEX_ARROW_ENCHANT_FIRE; + } else if (env->state.bow_enchantment == 2) { + overlays[3] = TEX_ARROW_ENCHANT_ICE; + } + } for (int i = 0; i < 4; i++) { - draw_inv_slot(gear[i], armour_x + 30 * i, weap_y); + draw_inv_slot(gear[i], overlays[i], armour_x + 30 * i, weap_y); } DrawText(TextFormat("%d", inv->arrows), armour_x + 117, weap_y + 6, 14, RAYWHITE); for (int p = 0; p < NUM_POTIONS; p++) { draw_icon_count(TEX_POTION + p, inv->potions[p], inv_x + 52 * p, weap_y); } + draw_inv_slot(env->state.learned_spells[0] ? TEX_PROJ_FIREBALL : -1, + -1, inv_x + 52 * 6, weap_y); + draw_inv_slot(env->state.learned_spells[1] ? TEX_PROJ_ICEBALL : -1, + -1, inv_x + 52 * 7, weap_y); + int human = IsKeyDown(KEY_LEFT_SHIFT) || IsKeyDown(KEY_RIGHT_SHIFT); DrawText( TextFormat( - "ach:%d/%d ret:%.2f len:%d", + "ach:%d/%d ret:%.2f len:%d %s", achievements, NUM_ACHIEVEMENTS, env->episode_return_accum, - env->episode_length_accum + env->episode_length_accum, + human ? "HUMAN" : "Hold SHIFT to take control" ), origin_x + 4, hud_y + 118, 14, - (Color){200, 200, 140, 255} + human ? (Color){255, 210, 40, 255} : (Color){200, 200, 140, 255} ); int panel_x = origin_x + view_w + OBS_PANEL_W; diff --git a/resources/craftax_clean/craftax_clean_weights.bin b/resources/craftax_clean/craftax_clean_weights.bin index cbda5e4250..c9167c8450 100644 Binary files a/resources/craftax_clean/craftax_clean_weights.bin and b/resources/craftax_clean/craftax_clean_weights.bin differ diff --git a/src/puffercpu.c b/src/puffercpu.c index 0beba5f624..8037986e79 100644 --- a/src/puffercpu.c +++ b/src/puffercpu.c @@ -638,7 +638,7 @@ void free_puffernet(PufferNet* net) { #include ENV_HEADER -#if !defined(PUF_NMMO3_NET) && !defined(PUF_ASTEROIDS_NET) && !defined(PUF_MINIMAL_NET) +#if !defined(PUF_NMMO3_NET) && !defined(PUF_ASTEROIDS_NET) && !defined(PUF_MINIMAL_NET) && !defined(PUF_CRAFTAX_NET) static int puf_align8(int n) { return (n + 7) & ~7; } @@ -724,7 +724,10 @@ int main(int argc, char** argv) { if (argc >= 2 && argv[1][0] && argv[1][0] != '-' && strchr(argv[1], '=') == NULL && strchr(argv[1], '/') == NULL && strstr(argv[1], ".bin") == NULL && strcmp(argv[1], "latest") != 0) { - env_name = argv[1]; + if (strcmp(argv[1], "eval") != 0 && strcmp(argv[1], "train") != 0 + && strcmp(argv[1], "match") != 0 && strcmp(argv[1], "sweep") != 0) { + env_name = argv[1]; + } argi = 2; } Ini ini = {0}; @@ -768,6 +771,8 @@ int main(int argc, char** argv) { int need = asteroids_weight_count(hidden_size, num_layers); #elif defined(PUF_MINIMAL_NET) int need = minimal_weight_count(hidden_size, num_layers); +#elif defined(PUF_CRAFTAX_NET) + int need = craftax_weight_count(hidden_size, num_layers); #else int need = puffernet_weight_count(OBS_SIZE, hidden_size, num_layers, act_sizes, num_actions); @@ -802,7 +807,7 @@ int main(int argc, char** argv) { size_t n_atn = (size_t)env.num_agents * (size_t)NUM_ATNS; size_t n_agt = (size_t)env.num_agents; obs_t* observations = (obs_t*)calloc(n_obs, sizeof(obs_t)); -#if !defined(PUF_NMMO3_NET) && !defined(PUF_ASTEROIDS_NET) && !defined(PUF_MINIMAL_NET) +#if !defined(PUF_NMMO3_NET) && !defined(PUF_ASTEROIDS_NET) && !defined(PUF_MINIMAL_NET) && !defined(PUF_CRAFTAX_NET) float* obs_f = (float*)calloc(n_obs, sizeof(float)); #endif float* actions = (float*)calloc(n_atn, sizeof(float)); @@ -843,6 +848,12 @@ int main(int argc, char** argv) { net = init_minimal_net(weights, env.num_agents, hidden_size, num_layers); } +#elif defined(PUF_CRAFTAX_NET) + CraftaxNet* net = NULL; + if (have_net) { + net = init_craftax_net(weights, env.num_agents, + hidden_size, num_layers); + } #else PufferNet* net = NULL; if (have_net) { @@ -901,6 +912,9 @@ int main(int argc, char** argv) { forward_asteroids(net, (float*)observations, terminals, actions); #elif defined(PUF_MINIMAL_NET) forward_minimal(net, (float*)observations, terminals, actions); +#elif defined(PUF_CRAFTAX_NET) + forward_craftax(net, (float*)observations, terminals, actions, masks); + env.predicted_value = craftax_value(net, 0); #else float* fwd = (float*)observations; if (sizeof(obs_t) != sizeof(float)) {