Skip to content

Commit

Permalink
📦 NEW: Add weight decay
Browse files Browse the repository at this point in the history
  • Loading branch information
@ianagbip1oti
ianagbip1oti committed Jan 23, 2024
1 parent 36c98ee commit a65e8c2
Show file tree
Hide file tree
Showing 8 changed files with 114,816 additions and 114,788 deletions.
8 changes: 4 additions & 4 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ arc-swap = "=1.6.0"
arrayvec = "=0.7.4"
dashmap = "=5.5.3"
fastapprox = "=0.3.1"
goober = { git = "https://github.com/jw1912/goober/", rev = "30ded2d" }
goober = { git = "https://github.com/ianagbip1oti/goober/", rev = "7b090e0" }
nohash-hasher = "=0.2.0"
memmap = "=0.7.0"
once_cell = "=1.19.0"
Expand Down
6 changes: 5 additions & 1 deletion src/bin/train-state.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ const LR: f32 = 0.001;
const LR_DROP_AT: usize = EPOCHS * 2 / 3;
const LR_DROP_FACTOR: f32 = 0.01;

const WEIGHT_DECAY: f32 = 0.99;

fn main() {
println!("Running...");
let mut args = env::args();
Expand Down Expand Up @@ -42,9 +44,11 @@ fn main() {
);

for epoch in 1..=EPOCHS {
println!("Epoch {}/{} (LR: {})...", epoch, EPOCHS, lr);
println!("\nEpoch {}/{} (LR: {})...", epoch, EPOCHS, lr);
let start = Instant::now();

network.decay_weights(WEIGHT_DECAY);

train(
&mut network,
lr,
Expand Down
96 changes: 48 additions & 48 deletions src/model/hidden_bias
View file
Original file line number Diff line number Diff line change
@@ -1,56 +1,56 @@
[
-105, -192, -204, -249, -228, -192, -387, -99,
58, -200, -511, -193, 16, -161, 46, -370,
-137, -270, -302, 13, 114, -313, 372, -352,
-264, 90, -100, 7, -147, 334, -159, 60,
0, -278, -158, -205, 92, -158, -210, -128,
295, 83, -334, -325, -108, -181, -109, 65,
-565, -198, 206, -55, -211, -171, -108, -258,
-493, -269, -192, -212, -187, -94, 342, -120,
-113, 46, -313, 130, -75, -105, -39, -136,
-38, 159, -155, -200, -133, -252, -139, -104,
-73, -54, -231, -120, -94, -57, 41, 71,
-150, 104, -96, -381, -176, -141, -298, -24,
-84, -114, -47, -298, 51, -142, -78, 125,
-163, 279, -174, -227, -229, -444, -20, 50,
-402, 82, -202, -106, -236, -372, -260, -100,
-254, -157, -156, 80, -279, -42, -25, -189,

174, -18, -168, -52, -140, -160, -562, -353,
-127, -19, 89, -70, -39, 16, -39, 73,
-676, -265, 113, -53, -308, -271, -187, -19,
-151, -279, -310, -77, -244, -498, -81, -282,
-259, -54, -399, 207, 156, 78, -270, -11,
-258, -348, 72, -118, -185, -207, -46, -135,
-145, -159, -145, -139, -212, -145, -114, 113,
-287, -336, 37, -186, -230, -43, -156, -14,
201, 287, -377, -84, -52, 21, -84, -186,
126, -150, -520, 219, -161, -269, -572, -188,
-59, 147, 97, -74, -63, 218, -67, -65,
-62, -165, -154, 298, 132, 65, 7, 77,
-66, -523, -67, -411, -211, -165, -115, -215,
40, -134, -359, -305, -228, -211, -131, -424,
-518, -132, -135, 70, -68, -82, 82, -408,
-152, -113, 50, -178, -240, -465, -107, -159,

-74, 177, 64, -575, -68, 367, -79, -115,
-173, 68, -243, -195, -305, -260, -128, -155,
-186, -25, 457, -380, -176, -170, -45, 295,
86, -64, -61, -4, -559, -249, 0, 6,
-212, -293, 190, -136, -163, -74, -29, -178,
-296, -204, -229, -131, -92, -25, -71, -83,
-238, -76, -91, -141, -64, 498, -288, -592,
35, -8, -209, -34, 284, -21, -53, 209,
-28, 103, -85, -18, 74, 396, -13, -266,
-200, -156, -42, 12, 249, -217, -111, -344,
-227, 167, -465, -272, 137, 177, 88, -206,
29, -29, 25, 173, -290, -174, -17, -100,
40, 19, -180, -15, -415, -205, -54, -177,
83, -291, -24, -82, 97, -490, -186, 0,
-377, -329, 24, -258, 100, -99, 226, -242,
102, -200, -504, -227, -167, -85, -219, 131,

-124, -320, -165, 102, -155, -255, 3, -158,
-649, -67, -219, 12, -285, -152, -187, -43,
-36, -564, 100, 94, -10, -249, -182, -395,
55, -79, -61, 208, 5, -244, -419, -185,
-23, -108, -174, 6, -296, 187, -478, -167,
3, 101, -337, 13, -315, -198, -291, 207,
-145, -203, -205, 56, -177, -107, -91, 92,
-439, -60, -163, -487, -116, -10, -47, -65,
240, -262, -100, -216, -246, 112, -285, -48,
130, -69, -1, -277, -292, -24, -325, 149,
94, -155, 6, -120, 188, -483, -72, 138,
-187, -98, 27, -41, -305, -145, 7, -254,
-100, -40, -560, 22, -278, -175, -234, -213,
-66, -167, 99, -25, 68, 0, -209, -91,
-245, -9, -239, -80, -468, -23, -459, -242,
13, 200, -72, -162, 132, -190, 126, -204,

-111, -357, 254, -139, -158, 133, -184, -134,
-117, -195, 62, 438, 24, -347, -183, 263,
144, 65, 36, -297, -144, -109, -147, -287,
-292, -349, 90, -177, -226, -317, -231, 41,
-72, -362, -191, -6, -45, -195, -215, 89,
240, -345, -295, -254, -197, 220, 37, 350,
-85, -166, -206, 68, -122, -241, -60, -193,
-165, -170, -122, -97, -91, -61, -332, -186,
-448, -153, -90, -238, -247, -232, -83, -156,
142, 2, -202, -4, 111, -135, 392, -130,
-90, -73, 19, -2, 32, -393, -150, -242,
-537, -210, 231, -4, -125, -60, -112, -159,
-294, -271, -198, -45, -214, 70, 168, -461,
-138, -306, 68, -324, -317, 176, -67, -220,
-56, -258, -169, 82, -478, -313, -208, 104,
-132, -136, -85, -127, -49, -78, -334, 286,

-522, -115, -225, 188, -182, -216, -76, -251,
-262, 158, -191, -212, -306, -245, -27, -303,
-83, -105, 155, -269, -64, 82, -58, 150,
-180, -223, -18, -547, -169, 88, -29, -186,
-350, -106, -285, 165, -28, -49, -130, -79,
-110, 272, -180, -163, -154, -238, -202, -99,
-104, 18, -436, -196, 157, -287, -572, 149,
-155, -194, -202, -17, -135, -226, 129, -44,
-156, -65, -121, -6, 45, -384, -123, -86,
-176, -148, 99, -426, -69, -330, -219, 23,
-14, -172, -124, -102, 4, -8, -513, -154,
-200, 53, -88, 195, -61, -77, -298, 10,
-177, -61, -246, -94, 14, -224, 470, 28,
-42, -91, 192, -195, -102, -208, 134, -109,
-33, -107, -173, -240, 109, -115, 249, -154,
-363, -262, 100, -254, 58, 29, -12, -231,

]
Loading

0 comments on commit a65e8c2

Please sign in to comment.