Welcome to part 2 of the deep Q-learning with Deep Q Networks (DQNs) tutorials. In the previous tutorial, we were working on our DQNAgent
class, and here we will pick back up where we left off.
Code up to this point:
from keras.models import Sequential from keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Activation, Flatten from keras.callbacks import TensorBoard from keras.optimizers import Adam from collections import deque import numpy as np import time REPLAY_MEMORY_SIZE = 50_000 MIN_REPLAY_MEMORY_SIZE = 1_000 MODEL_NAME = "256x2" # Own Tensorboard class class ModifiedTensorBoard(TensorBoard): # Overriding init to set initial step and writer (we want one log file for all .fit() calls) def __init__(self, **kwargs): super().__init__(**kwargs) self.step = 1 self.writer = tf.summary.FileWriter(self.log_dir) # Overriding this method to stop creating default log writer def set_model(self, model): pass # Overrided, saves logs with our step number # (otherwise every .fit() will start writing from 0th step) def on_epoch_end(self, epoch, logs=None): self.update_stats(**logs) # Overrided # We train for one batch only, no need to save anything at epoch end def on_batch_end(self, batch, logs=None): pass # Overrided, so won't close writer def on_train_end(self, _): pass # Custom method for saving own metrics # Creates writer, writes custom metrics and closes writer def update_stats(self, **stats): self._write_logs(stats, self.step) class DQNAgent: def __init__(self): # main model # gets trained every step self.model = self.create_model() # Target model this is what we .predict against every step self.target_model = self.create_model() self.target_model.set_weights(self.model.get_weights()) self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE) self.tensorboard = ModifiedTensorBoard(log_dir=f"logs/{MODEL_NAME}-{int(time.time())}") self.target_update_counter = 0 def create_model(self): model = Sequential() model.add(Conv2D(256, (3, 3), input_shape=env.OBSERVATION_SPACE_VALUES)) model.add(Activation("relu")) model.add(MaxPooling2D(2, 2)) model.add(Dropout(0.2)) model.add(Conv2D(256, (3, 3))) model.add(Activation("relu")) model.add(MaxPooling2D(2, 2)) model.add(Dropout(0.2)) model.add(Flatten()) model.add(Dense(64)) model.add(Dense(env.ACTION_SPACE_SIZE, activiation="linear")) model.compile(loss="mse", optimizer=Adam(lr=0.001), metrics=['accuracy']) return model def update_replay_memory(self, transition): self.replay_memory.append(transition) def get_qs(self, state, step): return self.model.predict(np.array(state).reshape(-1, *state.shape)/255)[0]
# Trains main network every step during episode def train(self, terminal_state, step): # Start training only if certain number of samples is already saved if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return
Next, if we do have the proper amount of data to train, we need to randomly select the data we want to train off of from our "memory" (which we'll wind up starting with using the past 1,000 steps, which is the previous ~5-15ish episodes depending on how many steps are taken in the episode, but we'll eventually populate this with up to 50,000 steps.
Then, we need to separate the states and matching q values from that minibatch of memory, including the "new" state + the future Qs of those future states.
# Get a minibatch of random samples from memory replay table minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE) # Get current states from minibatch, then query NN model for Q values current_states = np.array([transition[0] for transition in minibatch])/255 current_qs_list = self.model.predict(current_states) # Get future states from minibatch, then query NN model for Q values # When using target network, query it, otherwise main network should be queried new_current_states = np.array([transition[3] for transition in minibatch])/255 future_qs_list = self.target_model.predict(new_current_states)
With these values, we're ready to "update" our model:
X = [] y = [] # Now we need to enumerate our batches for index, (current_state, action, reward, new_current_state, done) in enumerate(minibatch): # If not a terminal state, get new q from future states, otherwise set it to 0 # almost like with Q Learning, but we use just part of equation here if not done: max_future_q = np.max(future_qs_list[index]) new_q = reward + DISCOUNT * max_future_q else: new_q = reward # Update Q value for given state current_qs = current_qs_list[index] current_qs[action] = new_q # And append to our training data X.append(current_state) y.append(current_qs) # Fit on all samples as one batch, log only on terminal state self.model.fit(np.array(X)/255, np.array(y), batch_size=MINIBATCH_SIZE, verbose=0, shuffle=False, callbacks=[self.tensorboard] if terminal_state else None)
Finally:
# Update target network counter every episode if terminal_state: self.target_update_counter += 1 # If counter reaches set value, update target network with weights of main network if self.target_update_counter > UPDATE_TARGET_EVERY: self.target_model.set_weights(self.model.get_weights()) self.target_update_counter = 0
Above, we're updating our counter as well as the target network (the one we use to make predictions) if we've had enough episodes.
Full code up to this point:
DISCOUNT = 0.99 REPLAY_MEMORY_SIZE = 50_000 # How many last steps to keep for model training MIN_REPLAY_MEMORY_SIZE = 1_000 # Minimum number of steps in a memory to start training MINIBATCH_SIZE = 64 # How many steps (samples) to use for training UPDATE_TARGET_EVERY = 5 # Terminal states (end of episodes) # Own Tensorboard class class ModifiedTensorBoard(TensorBoard): # Overriding init to set initial step and writer (we want one log file for all .fit() calls) def __init__(self, **kwargs): super().__init__(**kwargs) self.step = 1 self.writer = tf.summary.FileWriter(self.log_dir) # Overriding this method to stop creating default log writer def set_model(self, model): pass # Overrided, saves logs with our step number # (otherwise every .fit() will start writing from 0th step) def on_epoch_end(self, epoch, logs=None): self.update_stats(**logs) # Overrided # We train for one batch only, no need to save anything at epoch end def on_batch_end(self, batch, logs=None): pass # Overrided, so won't close writer def on_train_end(self, _): pass # Custom method for saving own metrics # Creates writer, writes custom metrics and closes writer def update_stats(self, **stats): self._write_logs(stats, self.step) # Agent class class DQNAgent: def __init__(self): # Main model self.model = self.create_model() # Target network self.target_model = self.create_model() self.target_model.set_weights(self.model.get_weights()) # An array with last n steps for training self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE) # Custom tensorboard object self.tensorboard = ModifiedTensorBoard(log_dir="logs/{}-{}".format(MODEL_NAME, int(time.time()))) # Used to count when to update target network with main network's weights self.target_update_counter = 0 def create_model(self): model = Sequential() model.add(Conv2D(256, (3, 3), input_shape=env.OBSERVATION_SPACE_VALUES)) # OBSERVATION_SPACE_VALUES = (10, 10, 3) a 10x10 RGB image. model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Conv2D(256, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors model.add(Dense(64)) model.add(Dense(env.ACTION_SPACE_SIZE, activation='linear')) # ACTION_SPACE_SIZE = how many choices (9) model.compile(loss="mse", optimizer=Adam(lr=0.001), metrics=['accuracy']) return model # Adds step's data to a memory replay array # (observation space, action, reward, new observation space, done) def update_replay_memory(self, transition): self.replay_memory.append(transition) # Trains main network every step during episode def train(self, terminal_state, step): # Start training only if certain number of samples is already saved if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return # Get a minibatch of random samples from memory replay table minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE) # Get current states from minibatch, then query NN model for Q values current_states = np.array([transition[0] for transition in minibatch])/255 current_qs_list = self.model.predict(current_states) # Get future states from minibatch, then query NN model for Q values # When using target network, query it, otherwise main network should be queried new_current_states = np.array([transition[3] for transition in minibatch])/255 future_qs_list = self.target_model.predict(new_current_states) X = [] y = [] # Now we need to enumerate our batches for index, (current_state, action, reward, new_current_state, done) in enumerate(minibatch): # If not a terminal state, get new q from future states, otherwise set it to 0 # almost like with Q Learning, but we use just part of equation here if not done: max_future_q = np.max(future_qs_list[index]) new_q = reward + DISCOUNT * max_future_q else: new_q = reward # Update Q value for given state current_qs = current_qs_list[index] current_qs[action] = new_q # And append to our training data X.append(current_state) y.append(current_qs) # Fit on all samples as one batch, log only on terminal state self.model.fit(np.array(X)/255, np.array(y), batch_size=MINIBATCH_SIZE, verbose=0, shuffle=False, callbacks=[self.tensorboard] if terminal_state else None) # Update target network counter every episode if terminal_state: self.target_update_counter += 1 # If counter reaches set value, update target network with weights of main network if self.target_update_counter > UPDATE_TARGET_EVERY: self.target_model.set_weights(self.model.get_weights()) self.target_update_counter = 0 # Queries main network for Q values given current observation space (environment state) def get_qs(self, state): return self.model.predict(np.array(state).reshape(-1, *state.shape)/255)[0] agent = DQNAgent()
Next, let's bring in our Blob
class. It is the same as before, with a couple minor tweaks:
class Blob: def __init__(self, size): self.size = size self.x = np.random.randint(0, size) self.y = np.random.randint(0, size) def __str__(self): return f"Blob ({self.x}, {self.y})" def __sub__(self, other): return (self.x-other.x, self.y-other.y) def __eq__(self, other): return self.x == other.x and self.y == other.y def action(self, choice): ''' Gives us 9 total movement options. (0,1,2,3,4,5,6,7,8) ''' if choice == 0: self.move(x=1, y=1) elif choice == 1: self.move(x=-1, y=-1) elif choice == 2: self.move(x=-1, y=1) elif choice == 3: self.move(x=1, y=-1) elif choice == 4: self.move(x=1, y=0) elif choice == 5: self.move(x=-1, y=0) elif choice == 6: self.move(x=0, y=1) elif choice == 7: self.move(x=0, y=-1) elif choice == 8: self.move(x=0, y=0) def move(self, x=False, y=False): # If no value for x, move randomly if not x: self.x += np.random.randint(-1, 2) else: self.x += x # If no value for y, move randomly if not y: self.y += np.random.randint(-1, 2) else: self.y += y # If we are out of bounds, fix! if self.x < 0: self.x = 0 elif self.x > self.size-1: self.x = self.size-1 if self.y < 0: self.y = 0 elif self.y > self.size-1: self.y = self.size-1
Now, we're allowing for 9 total moves (no move along with up, down, left, and right. We've also added an operator overload for the ==
comparison. It will allow us to determine in our environment if two blobs are touching by simply using ==
. Next, I am going to put in our blob env class. It's very much the same as before, just turned into a class so we can treat it as an object:
class BlobEnv: SIZE = 10 RETURN_IMAGES = True MOVE_PENALTY = 1 ENEMY_PENALTY = 300 FOOD_REWARD = 25 OBSERVATION_SPACE_VALUES = (SIZE, SIZE, 3) # 4 ACTION_SPACE_SIZE = 9 PLAYER_N = 1 # player key in dict FOOD_N = 2 # food key in dict ENEMY_N = 3 # enemy key in dict # the dict! (colors) d = {1: (255, 175, 0), 2: (0, 255, 0), 3: (0, 0, 255)} def reset(self): self.player = Blob(self.SIZE) self.food = Blob(self.SIZE) while self.food == self.player: self.food = Blob(self.SIZE) self.enemy = Blob(self.SIZE) while self.enemy == self.player or self.enemy == self.food: self.enemy = Blob(self.SIZE) self.episode_step = 0 if self.RETURN_IMAGES: observation = np.array(self.get_image()) else: observation = (self.player-self.food) + (self.player-self.enemy) return observation def step(self, action): self.episode_step += 1 self.player.action(action) #### MAYBE ### #enemy.move() #food.move() ############## if self.RETURN_IMAGES: new_observation = np.array(self.get_image()) else: new_observation = (self.player-self.food) + (self.player-self.enemy) if self.player == self.enemy: reward = -self.ENEMY_PENALTY elif self.player == self.food: reward = self.FOOD_REWARD else: reward = -self.MOVE_PENALTY done = False if reward == self.FOOD_REWARD or reward == -self.ENEMY_PENALTY or self.episode_step >= 200: done = True return new_observation, reward, done def render(self): img = self.get_image() img = img.resize((300, 300)) # resizing so we can see our agent in all its glory. cv2.imshow("image", np.array(img)) # show it! cv2.waitKey(1) # FOR CNN # def get_image(self): env = np.zeros((self.SIZE, self.SIZE, 3), dtype=np.uint8) # starts an rbg of our size env[self.food.x][self.food.y] = self.d[self.FOOD_N] # sets the food location tile to green color env[self.enemy.x][self.enemy.y] = self.d[self.ENEMY_N] # sets the enemy location to red env[self.player.x][self.player.y] = self.d[self.PLAYER_N] # sets the player tile to blue img = Image.fromarray(env, 'RGB') # reading to rgb. Apparently. Even tho color definitions are bgr. ??? return img
None of the code above for this class should have you lost or confused, as long as you've been following along. We can then initialize our environment with:
env = BlobEnv() # For stats ep_rewards = [-200] # For more repetitive results random.seed(1) np.random.seed(1) tf.set_random_seed(1) # Create models folder if not os.path.isdir('models'): os.makedirs('models')
Full code up to this point:
import numpy as np import keras.backend.tensorflow_backend as backend from keras.models import Sequential from keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Activation, Flatten from keras.optimizers import Adam from keras.callbacks import TensorBoard import tensorflow as tf from collections import deque import time import random from tqdm import tqdm import os from PIL import Image import cv2 DISCOUNT = 0.99 REPLAY_MEMORY_SIZE = 50_000 # How many last steps to keep for model training MIN_REPLAY_MEMORY_SIZE = 1_000 # Minimum number of steps in a memory to start training MINIBATCH_SIZE = 64 # How many steps (samples) to use for training UPDATE_TARGET_EVERY = 5 # Terminal states (end of episodes) MODEL_NAME = '2x256' MIN_REWARD = -200 # For model save MEMORY_FRACTION = 0.20 # Environment settings EPISODES = 20_000 # Exploration settings epsilon = 1 # not a constant, going to be decayed EPSILON_DECAY = 0.99975 MIN_EPSILON = 0.001 # Stats settings AGGREGATE_STATS_EVERY = 50 # episodes SHOW_PREVIEW = False class Blob: def __init__(self, size): self.size = size self.x = np.random.randint(0, size) self.y = np.random.randint(0, size) def __str__(self): return f"Blob ({self.x}, {self.y})" def __sub__(self, other): return (self.x-other.x, self.y-other.y) def __eq__(self, other): return self.x == other.x and self.y == other.y def action(self, choice): ''' Gives us 9 total movement options. (0,1,2,3,4,5,6,7,8) ''' if choice == 0: self.move(x=1, y=1) elif choice == 1: self.move(x=-1, y=-1) elif choice == 2: self.move(x=-1, y=1) elif choice == 3: self.move(x=1, y=-1) elif choice == 4: self.move(x=1, y=0) elif choice == 5: self.move(x=-1, y=0) elif choice == 6: self.move(x=0, y=1) elif choice == 7: self.move(x=0, y=-1) elif choice == 8: self.move(x=0, y=0) def move(self, x=False, y=False): # If no value for x, move randomly if not x: self.x += np.random.randint(-1, 2) else: self.x += x # If no value for y, move randomly if not y: self.y += np.random.randint(-1, 2) else: self.y += y # If we are out of bounds, fix! if self.x < 0: self.x = 0 elif self.x > self.size-1: self.x = self.size-1 if self.y < 0: self.y = 0 elif self.y > self.size-1: self.y = self.size-1 class BlobEnv: SIZE = 10 RETURN_IMAGES = True MOVE_PENALTY = 1 ENEMY_PENALTY = 300 FOOD_REWARD = 25 OBSERVATION_SPACE_VALUES = (SIZE, SIZE, 3) # 4 ACTION_SPACE_SIZE = 9 PLAYER_N = 1 # player key in dict FOOD_N = 2 # food key in dict ENEMY_N = 3 # enemy key in dict # the dict! (colors) d = {1: (255, 175, 0), 2: (0, 255, 0), 3: (0, 0, 255)} def reset(self): self.player = Blob(self.SIZE) self.food = Blob(self.SIZE) while self.food == self.player: self.food = Blob(self.SIZE) self.enemy = Blob(self.SIZE) while self.enemy == self.player or self.enemy == self.food: self.enemy = Blob(self.SIZE) self.episode_step = 0 if self.RETURN_IMAGES: observation = np.array(self.get_image()) else: observation = (self.player-self.food) + (self.player-self.enemy) return observation def step(self, action): self.episode_step += 1 self.player.action(action) #### MAYBE ### #enemy.move() #food.move() ############## if self.RETURN_IMAGES: new_observation = np.array(self.get_image()) else: new_observation = (self.player-self.food) + (self.player-self.enemy) if self.player == self.enemy: reward = -self.ENEMY_PENALTY elif self.player == self.food: reward = self.FOOD_REWARD else: reward = -self.MOVE_PENALTY done = False if reward == self.FOOD_REWARD or reward == -self.ENEMY_PENALTY or self.episode_step >= 200: done = True return new_observation, reward, done def render(self): img = self.get_image() img = img.resize((300, 300)) # resizing so we can see our agent in all its glory. cv2.imshow("image", np.array(img)) # show it! cv2.waitKey(1) # FOR CNN # def get_image(self): env = np.zeros((self.SIZE, self.SIZE, 3), dtype=np.uint8) # starts an rbg of our size env[self.food.x][self.food.y] = self.d[self.FOOD_N] # sets the food location tile to green color env[self.enemy.x][self.enemy.y] = self.d[self.ENEMY_N] # sets the enemy location to red env[self.player.x][self.player.y] = self.d[self.PLAYER_N] # sets the player tile to blue img = Image.fromarray(env, 'RGB') # reading to rgb. Apparently. Even tho color definitions are bgr. ??? return img env = BlobEnv() # For stats ep_rewards = [-200] # For more repetitive results random.seed(1) np.random.seed(1) tf.set_random_seed(1) # Memory fraction, used mostly when trai8ning multiple agents #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) #backend.set_session(tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))) # Create models folder if not os.path.isdir('models'): os.makedirs('models') # Own Tensorboard class class ModifiedTensorBoard(TensorBoard): # Overriding init to set initial step and writer (we want one log file for all .fit() calls) def __init__(self, **kwargs): super().__init__(**kwargs) self.step = 1 self.writer = tf.summary.FileWriter(self.log_dir) # Overriding this method to stop creating default log writer def set_model(self, model): pass # Overrided, saves logs with our step number # (otherwise every .fit() will start writing from 0th step) def on_epoch_end(self, epoch, logs=None): self.update_stats(**logs) # Overrided # We train for one batch only, no need to save anything at epoch end def on_batch_end(self, batch, logs=None): pass # Overrided, so won't close writer def on_train_end(self, _): pass # Custom method for saving own metrics # Creates writer, writes custom metrics and closes writer def update_stats(self, **stats): self._write_logs(stats, self.step) # Agent class class DQNAgent: def __init__(self): # Main model self.model = self.create_model() # Target network self.target_model = self.create_model() self.target_model.set_weights(self.model.get_weights()) # An array with last n steps for training self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE) # Custom tensorboard object self.tensorboard = ModifiedTensorBoard(log_dir="logs/{}-{}".format(MODEL_NAME, int(time.time()))) # Used to count when to update target network with main network's weights self.target_update_counter = 0 def create_model(self): model = Sequential() model.add(Conv2D(256, (3, 3), input_shape=env.OBSERVATION_SPACE_VALUES)) # OBSERVATION_SPACE_VALUES = (10, 10, 3) a 10x10 RGB image. model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Conv2D(256, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors model.add(Dense(64)) model.add(Dense(env.ACTION_SPACE_SIZE, activation='linear')) # ACTION_SPACE_SIZE = how many choices (9) model.compile(loss="mse", optimizer=Adam(lr=0.001), metrics=['accuracy']) return model # Adds step's data to a memory replay array # (observation space, action, reward, new observation space, done) def update_replay_memory(self, transition): self.replay_memory.append(transition) # Trains main network every step during episode def train(self, terminal_state, step): # Start training only if certain number of samples is already saved if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return # Get a minibatch of random samples from memory replay table minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE) # Get current states from minibatch, then query NN model for Q values current_states = np.array([transition[0] for transition in minibatch])/255 current_qs_list = self.model.predict(current_states) # Get future states from minibatch, then query NN model for Q values # When using target network, query it, otherwise main network should be queried new_current_states = np.array([transition[3] for transition in minibatch])/255 future_qs_list = self.target_model.predict(new_current_states) X = [] y = [] # Now we need to enumerate our batches for index, (current_state, action, reward, new_current_state, done) in enumerate(minibatch): # If not a terminal state, get new q from future states, otherwise set it to 0 # almost like with Q Learning, but we use just part of equation here if not done: max_future_q = np.max(future_qs_list[index]) new_q = reward + DISCOUNT * max_future_q else: new_q = reward # Update Q value for given state current_qs = current_qs_list[index] current_qs[action] = new_q # And append to our training data X.append(current_state) y.append(current_qs) # Fit on all samples as one batch, log only on terminal state self.model.fit(np.array(X)/255, np.array(y), batch_size=MINIBATCH_SIZE, verbose=0, shuffle=False, callbacks=[self.tensorboard] if terminal_state else None) # Update target network counter every episode if terminal_state: self.target_update_counter += 1 # If counter reaches set value, update target network with weights of main network if self.target_update_counter > UPDATE_TARGET_EVERY: self.target_model.set_weights(self.model.get_weights()) self.target_update_counter = 0 # Queries main network for Q values given current observation space (environment state) def get_qs(self, state): return self.model.predict(np.array(state).reshape(-1, *state.shape)/255)[0] agent = DQNAgent()
Alright, now we just need to use our agent to engage with this environment.
# Iterate over episodes for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'):
Here, we're using tqdm, we're changing the chars to use ASCII (for our Windows users) and then calling the "unit" an "episode." Next:
# Update tensorboard step every episode agent.tensorboard.step = episode # Restarting episode - reset episode reward and step number episode_reward = 0 step = 1 # Reset environment and get initial state current_state = env.reset()
After restarting the episode and state, we're ready to iterate over the steps per episode:
# Reset flag and start iterating until episode ends done = False while not done: # This part stays mostly the same, the change is to query a model for Q values if np.random.random() > epsilon: # Get action from Q table action = np.argmax(agent.get_qs(current_state)) else: # Get random action action = np.random.randint(0, env.ACTION_SPACE_SIZE) new_state, reward, done = env.step(action)
Next, we account for the reward and render if it's time / and rendering is turned on. Then, we update replay memory and finally run a .train()
, which will fit our agent.model
every step, and then fit our agent.target_model
if it has been 5 episodes. Then, we set our current state to the new one, and add one step to our counter:
# Transform new continous state to new discrete state and count reward episode_reward += reward if SHOW_PREVIEW and not episode % AGGREGATE_STATS_EVERY: env.render() # Every step we update replay memory and train main network agent.update_replay_memory((current_state, action, reward, new_state, done)) agent.train(done, step) current_state = new_state step += 1
Now we will handle for stats and save our model if it's performing how you want. In this case, we're checking for the average reward to be above some number.
# Append episode reward to a list and log stats (every given number of episodes) ep_rewards.append(episode_reward) if not episode % AGGREGATE_STATS_EVERY or episode == 1: average_reward = sum(ep_rewards[-AGGREGATE_STATS_EVERY:])/len(ep_rewards[-AGGREGATE_STATS_EVERY:]) min_reward = min(ep_rewards[-AGGREGATE_STATS_EVERY:]) max_reward = max(ep_rewards[-AGGREGATE_STATS_EVERY:]) agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, epsilon=epsilon) # Save model, but only when min reward is greater or equal a set value if average_reward >= MIN_REWARD: agent.model.save(f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model')
Finally, we will handle for epsilon:
# Decay epsilon if epsilon > MIN_EPSILON: epsilon *= EPSILON_DECAY epsilon = max(MIN_EPSILON, epsilon)
Full code is now:
import numpy as np import keras.backend.tensorflow_backend as backend from keras.models import Sequential from keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Activation, Flatten from keras.optimizers import Adam from keras.callbacks import TensorBoard import tensorflow as tf from collections import deque import time import random from tqdm import tqdm import os from PIL import Image import cv2 DISCOUNT = 0.99 REPLAY_MEMORY_SIZE = 50_000 # How many last steps to keep for model training MIN_REPLAY_MEMORY_SIZE = 1_000 # Minimum number of steps in a memory to start training MINIBATCH_SIZE = 64 # How many steps (samples) to use for training UPDATE_TARGET_EVERY = 5 # Terminal states (end of episodes) MODEL_NAME = '2x256' MIN_REWARD = -200 # For model save MEMORY_FRACTION = 0.20 # Environment settings EPISODES = 20_000 # Exploration settings epsilon = 1 # not a constant, going to be decayed EPSILON_DECAY = 0.99975 MIN_EPSILON = 0.001 # Stats settings AGGREGATE_STATS_EVERY = 50 # episodes SHOW_PREVIEW = False class Blob: def __init__(self, size): self.size = size self.x = np.random.randint(0, size) self.y = np.random.randint(0, size) def __str__(self): return f"Blob ({self.x}, {self.y})" def __sub__(self, other): return (self.x-other.x, self.y-other.y) def __eq__(self, other): return self.x == other.x and self.y == other.y def action(self, choice): ''' Gives us 9 total movement options. (0,1,2,3,4,5,6,7,8) ''' if choice == 0: self.move(x=1, y=1) elif choice == 1: self.move(x=-1, y=-1) elif choice == 2: self.move(x=-1, y=1) elif choice == 3: self.move(x=1, y=-1) elif choice == 4: self.move(x=1, y=0) elif choice == 5: self.move(x=-1, y=0) elif choice == 6: self.move(x=0, y=1) elif choice == 7: self.move(x=0, y=-1) elif choice == 8: self.move(x=0, y=0) def move(self, x=False, y=False): # If no value for x, move randomly if not x: self.x += np.random.randint(-1, 2) else: self.x += x # If no value for y, move randomly if not y: self.y += np.random.randint(-1, 2) else: self.y += y # If we are out of bounds, fix! if self.x < 0: self.x = 0 elif self.x > self.size-1: self.x = self.size-1 if self.y < 0: self.y = 0 elif self.y > self.size-1: self.y = self.size-1 class BlobEnv: SIZE = 10 RETURN_IMAGES = True MOVE_PENALTY = 1 ENEMY_PENALTY = 300 FOOD_REWARD = 25 OBSERVATION_SPACE_VALUES = (SIZE, SIZE, 3) # 4 ACTION_SPACE_SIZE = 9 PLAYER_N = 1 # player key in dict FOOD_N = 2 # food key in dict ENEMY_N = 3 # enemy key in dict # the dict! (colors) d = {1: (255, 175, 0), 2: (0, 255, 0), 3: (0, 0, 255)} def reset(self): self.player = Blob(self.SIZE) self.food = Blob(self.SIZE) while self.food == self.player: self.food = Blob(self.SIZE) self.enemy = Blob(self.SIZE) while self.enemy == self.player or self.enemy == self.food: self.enemy = Blob(self.SIZE) self.episode_step = 0 if self.RETURN_IMAGES: observation = np.array(self.get_image()) else: observation = (self.player-self.food) + (self.player-self.enemy) return observation def step(self, action): self.episode_step += 1 self.player.action(action) #### MAYBE ### #enemy.move() #food.move() ############## if self.RETURN_IMAGES: new_observation = np.array(self.get_image()) else: new_observation = (self.player-self.food) + (self.player-self.enemy) if self.player == self.enemy: reward = -self.ENEMY_PENALTY elif self.player == self.food: reward = self.FOOD_REWARD else: reward = -self.MOVE_PENALTY done = False if reward == self.FOOD_REWARD or reward == -self.ENEMY_PENALTY or self.episode_step >= 200: done = True return new_observation, reward, done def render(self): img = self.get_image() img = img.resize((300, 300)) # resizing so we can see our agent in all its glory. cv2.imshow("image", np.array(img)) # show it! cv2.waitKey(1) # FOR CNN # def get_image(self): env = np.zeros((self.SIZE, self.SIZE, 3), dtype=np.uint8) # starts an rbg of our size env[self.food.x][self.food.y] = self.d[self.FOOD_N] # sets the food location tile to green color env[self.enemy.x][self.enemy.y] = self.d[self.ENEMY_N] # sets the enemy location to red env[self.player.x][self.player.y] = self.d[self.PLAYER_N] # sets the player tile to blue img = Image.fromarray(env, 'RGB') # reading to rgb. Apparently. Even tho color definitions are bgr. ??? return img env = BlobEnv() # For stats ep_rewards = [-200] # For more repetitive results random.seed(1) np.random.seed(1) tf.set_random_seed(1) # Memory fraction, used mostly when trai8ning multiple agents #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) #backend.set_session(tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))) # Create models folder if not os.path.isdir('models'): os.makedirs('models') # Own Tensorboard class class ModifiedTensorBoard(TensorBoard): # Overriding init to set initial step and writer (we want one log file for all .fit() calls) def __init__(self, **kwargs): super().__init__(**kwargs) self.step = 1 self.writer = tf.summary.FileWriter(self.log_dir) # Overriding this method to stop creating default log writer def set_model(self, model): pass # Overrided, saves logs with our step number # (otherwise every .fit() will start writing from 0th step) def on_epoch_end(self, epoch, logs=None): self.update_stats(**logs) # Overrided # We train for one batch only, no need to save anything at epoch end def on_batch_end(self, batch, logs=None): pass # Overrided, so won't close writer def on_train_end(self, _): pass # Custom method for saving own metrics # Creates writer, writes custom metrics and closes writer def update_stats(self, **stats): self._write_logs(stats, self.step) # Agent class class DQNAgent: def __init__(self): # Main model self.model = self.create_model() # Target network self.target_model = self.create_model() self.target_model.set_weights(self.model.get_weights()) # An array with last n steps for training self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE) # Custom tensorboard object self.tensorboard = ModifiedTensorBoard(log_dir="logs/{}-{}".format(MODEL_NAME, int(time.time()))) # Used to count when to update target network with main network's weights self.target_update_counter = 0 def create_model(self): model = Sequential() model.add(Conv2D(256, (3, 3), input_shape=env.OBSERVATION_SPACE_VALUES)) # OBSERVATION_SPACE_VALUES = (10, 10, 3) a 10x10 RGB image. model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Conv2D(256, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors model.add(Dense(64)) model.add(Dense(env.ACTION_SPACE_SIZE, activation='linear')) # ACTION_SPACE_SIZE = how many choices (9) model.compile(loss="mse", optimizer=Adam(lr=0.001), metrics=['accuracy']) return model # Adds step's data to a memory replay array # (observation space, action, reward, new observation space, done) def update_replay_memory(self, transition): self.replay_memory.append(transition) # Trains main network every step during episode def train(self, terminal_state, step): # Start training only if certain number of samples is already saved if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return # Get a minibatch of random samples from memory replay table minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE) # Get current states from minibatch, then query NN model for Q values current_states = np.array([transition[0] for transition in minibatch])/255 current_qs_list = self.model.predict(current_states) # Get future states from minibatch, then query NN model for Q values # When using target network, query it, otherwise main network should be queried new_current_states = np.array([transition[3] for transition in minibatch])/255 future_qs_list = self.target_model.predict(new_current_states) X = [] y = [] # Now we need to enumerate our batches for index, (current_state, action, reward, new_current_state, done) in enumerate(minibatch): # If not a terminal state, get new q from future states, otherwise set it to 0 # almost like with Q Learning, but we use just part of equation here if not done: max_future_q = np.max(future_qs_list[index]) new_q = reward + DISCOUNT * max_future_q else: new_q = reward # Update Q value for given state current_qs = current_qs_list[index] current_qs[action] = new_q # And append to our training data X.append(current_state) y.append(current_qs) # Fit on all samples as one batch, log only on terminal state self.model.fit(np.array(X)/255, np.array(y), batch_size=MINIBATCH_SIZE, verbose=0, shuffle=False, callbacks=[self.tensorboard] if terminal_state else None) # Update target network counter every episode if terminal_state: self.target_update_counter += 1 # If counter reaches set value, update target network with weights of main network if self.target_update_counter > UPDATE_TARGET_EVERY: self.target_model.set_weights(self.model.get_weights()) self.target_update_counter = 0 # Queries main network for Q values given current observation space (environment state) def get_qs(self, state): return self.model.predict(np.array(state).reshape(-1, *state.shape)/255)[0] agent = DQNAgent() # Iterate over episodes for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): # Update tensorboard step every episode agent.tensorboard.step = episode # Restarting episode - reset episode reward and step number episode_reward = 0 step = 1 # Reset environment and get initial state current_state = env.reset() # Reset flag and start iterating until episode ends done = False while not done: # This part stays mostly the same, the change is to query a model for Q values if np.random.random() > epsilon: # Get action from Q table action = np.argmax(agent.get_qs(current_state)) else: # Get random action action = np.random.randint(0, env.ACTION_SPACE_SIZE) new_state, reward, done = env.step(action) # Transform new continous state to new discrete state and count reward episode_reward += reward if SHOW_PREVIEW and not episode % AGGREGATE_STATS_EVERY: env.render() # Every step we update replay memory and train main network agent.update_replay_memory((current_state, action, reward, new_state, done)) agent.train(done, step) current_state = new_state step += 1 # Append episode reward to a list and log stats (every given number of episodes) ep_rewards.append(episode_reward) if not episode % AGGREGATE_STATS_EVERY or episode == 1: average_reward = sum(ep_rewards[-AGGREGATE_STATS_EVERY:])/len(ep_rewards[-AGGREGATE_STATS_EVERY:]) min_reward = min(ep_rewards[-AGGREGATE_STATS_EVERY:]) max_reward = max(ep_rewards[-AGGREGATE_STATS_EVERY:]) agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, epsilon=epsilon) # Save model, but only when min reward is greater or equal a set value if min_reward >= MIN_REWARD: agent.model.save(f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') # Decay epsilon if epsilon > MIN_EPSILON: epsilon *= EPSILON_DECAY epsilon = max(MIN_EPSILON, epsilon)
Now it's time to train!