Initial commit

Author: maurock

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.idea/SnakeGA.iml

@@ -0,0 +1,88 @@
+from keras.models import Sequential
+from keras.layers import Dense
+from keras.optimizers import Adam
+import numpy as np
+import random
+
+# action = 1,2,3,4
+# state = right_free, left_free, , position_food
+# Deep Q-learning Agent
+
+
+class DQNAgent:
+    def state(self):
+
+    def __init__(self, state_size, action_size):
+        self.state_size = state_size
+        self.action_size = action_size
+        self.memory = []
+        self.gamma = 0.95    # discount rate
+        self.epsilon = 1.0  # exploration rate
+        self.epsilon_min = 0.01
+        self.epsilon_decay = 0.995
+        self.learning_rate = 0.001
+        self.model = self._build_model()
+
+    def _build_model(self):
+        # Neural Net for Deep-Q learning Model
+        model = Sequential()
+        model.add(Dense(24, input_dim=self.state_size, activation='relu'))
+        model.add(Dense(24, activation='relu'))
+        model.add(Dense(self.action_size, activation='linear'))
+        model.compile(loss='mse',
+                      optimizer=Adam(lr=self.learning_rate))
+        return model
+
+    def remember(self, state, action, reward, next_state, done):
+        self.memory.append((state, action, reward, next_state, done))
+
+    def act(self, state):
+        if np.random.rand() <= self.epsilon:
+            return random.randrange(self.action_size)
+        act_values = self.model.predict(state)
+        return np.argmax(act_values[0])  # returns action
+
+    def replay(self, batch_size):
+        minibatch = random.sample(self.memory, batch_size)
+        for state, action, reward, next_state, done in minibatch:
+            target = reward
+            if not done:
+              target = reward + self.gamma * \
+                       np.amax(self.model.predict(next_state)[0])
+            target_f = self.model.predict(state)
+            target_f[0][action] = target
+            self.model.fit(state, target_f, epochs=1, verbose=0)
+        if self.epsilon > self.epsilon_min:
+            self.epsilon *= self.epsilon_decay
+
+agent = DQNAgent(env)
+# Iterate the game
+for e in range(episodes):
+    # reset state in the beginning of each game
+    state = env.reset()
+    state = np.reshape(state, [1, 4])
+    # time_t represents each frame of the game
+    # Our goal is to keep the pole upright as long as possible until score of 500
+    # the more time_t the more score
+    for time_t in range(500):
+        # turn this on if you want to render
+        # env.render()
+        # Decide action
+        action = agent.act(state)
+        # Advance the game to the next frame based on the action.
+        # Reward is 1 for every frame the pole survived
+        next_state, reward, done, _ = env.step(action)
+        next_state = np.reshape(next_state, [1, 4])
+        # Remember the previous state, action, reward, and done
+        agent.remember(state, action, rewrd, next_state, done)
+        # make next_state the new current state for the next frame.
+        state = next_state
+        # done becomes True when the game ends
+        # ex) The agent drops the pole
+        if done:
+            # print the score and break out of the loop
+            print("episode: {}/{}, score: {}"
+                  .format(e, episodes, time_t))
+            break
+    # train the agent with the experience of the episode
+    agent.replay(32)

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,203 @@
+#import snakeClass
+import pygame
+from keras.models import Sequential,Model
+from keras.layers.core import Flatten, Dense, Dropout
+from keras.optimizers import RMSprop
+import random
+import numpy as np
+import pandas as pd
+import pygame
+from keras.models import Sequential,Model
+from keras.layers.core import Flatten, Dense, Dropout
+from keras.optimizers import RMSprop
+import random
+import numpy as np
+import pandas as pd
+import keras
+import keras.backend as K
+import copy
+
+pd.set_option('display.max_columns', 500)
+
+class DQNAgent(object):
+
+    def __init__(self):
+        self.reward = 0
+        self.gamma = 0.9
+        self.dataframe = pd.DataFrame()
+        self.short_memory = np.array([])
+        self.agent_target = 1
+        self.agent_predict = 0
+        self.model = self.network()
+        #self.model = self.network("weights.hdf5")
+        self.epsilon = 3
+        self.actual = []
+
+
+
+    def get_state(self, game, player, food):
+        state = [
+                    player.position[-1][0] - 20 in player.position or player.position[-1][0] - 20 < 0,                    # danger left
+                    player.position[-1][0] - 40 in player.position or player.position[-1][0] - 40 < 0,                    # danger 2 left
+                    player.position[-1][0] + 20 in player.position or player.position[-1][0] + 20 > game.display_width,   # danger right
+                    player.position[-1][0] + 40 in player.position or player.position[-1][0] + 40 > game.display_width,   # danger 2 right
+                    player.position[-1][1] - 20 in player.position or player.position[-1][1] - 20 < 0,                    # danger up
+                    player.position[-1][-1] - 40 in player.position or player.position[-1][-1] - 40 < 0,                  # danger 2 up
+                    player.position[-1][1] + 20 in player.position or player.position[-1][1] + 20 > game.display_height,  # danger down
+                    player.position[-1][-1] + 40 in player.position or player.position[-1][-1] + 40 > game.display_height,# danger 2 down
+                    player.x_change == -20,           # move left
+                    player.x_change == 20,            # move right
+                    player.y_change == -20,           # move up
+                    player.y_change == 20,            # move down
+                    food.x_food < player.x,           # food left
+                    food.x_food > player.x,           # food right
+                    food.y_food < player.y,           # food up
+                    food.y_food > player.y            # food down
+            ]
+
+        for i in range(len(state)):
+            if state[i]:
+                state[i]=1
+            else:
+                state[i]=0
+        return np.asarray(state)
+
+
+    def set_reward(self, game, player, food):
+        if game.crash:
+            self.reward = -10
+            return self.reward
+        if player.eaten:
+            self.reward = 10
+        elif (player.x_change < 0 and food.x_food < player.x) or (player.x_change > 0 and food.x_food > player.x) or (player.y_change < 0 and food.y_food < player.y) or (player.y_change > 0 and food.y_food > player.y):
+            self.reward = 2
+        else:
+            self.reward = -1
+        return self.reward
+
+    def possible_moves(self, player):
+        if player.x_change == -20:
+            return [0,2,3]
+        elif player.x_change == 20:
+            return [1,2,3]
+        elif player.y_change == -20:
+            return [0,1,2]
+        elif player.y_change == 20:
+            return [0,1,3]
+
+    def replay(self, game, player, food, actual):
+        player.position = copy.deepcopy(actual[0])
+        player.x, player.y, player.x_change, player.y_change, food.x_food, food.y_food, game.crash, player.eaten, player.food = actual[1:]
+
+    '''
+    def next_state(self, game, player, food, i):
+        actual = [player.position, player.x, player.y, player.x_change, player.y_change, food.x_food, food.y_food, game.crash, player.eaten]
+        original_state = self.get_state(game, player, food)
+        player.do_move(i, player.x, player.y, game, food)
+        player.display_player(player.x, player.y,player.food,game,player)
+        array = [original_state, i, self.set_reward(game, player), self.get_state(game, player, food)]
+        pygame.time.wait(500)
+        self.replay(game, player, food, actual)
+        player.display_player(player.x, player.y, player.food, game, player)
+        return array
+    '''
+
+    def next_state(self, game, player, food, i):
+        actual = [player.position, player.x, player.y, player.x_change, player.y_change, food.x_food, food.y_food, game.crash, player.eaten, player.food]
+        original_state = self.get_state(game, player, food)
+        player.do_move(i, player.x, player.y, game, food)
+        player.display_player(player.x, player.y,player.food,game,player)
+        array = [original_state, i, self.set_reward(game, player), self.get_state(game, player, food)]
+        pygame.time.wait(500)
+        self.replay(game, player, food, actual)
+        player.display_player(player.x, player.y, player.food, game, player)
+        return array
+
+    def loss(self, target, state, action):
+        return K.mean(K.square(target - self.predict_q(self.model, state, action)), axis=-1)
+
+    def network(self,weights=None):
+        model = Sequential()
+        model.add(Dense(output_dim=50, activation='relu', input_dim=17))
+        model.add(Dense(output_dim=50, activation='relu'))
+        model.add(Dense(output_dim=1, activation='softmax'))
+        opt = RMSprop(lr=0.025)
+        model.compile(loss='mse', optimizer=opt)
+
+        if weights:
+            model.load_weights(weights)
+        # [self.loss(self.agent_target, self.agent_predict)]
+        return model
+
+    def act(self, state):
+        if random.random(0, 1) < self.epsilon:
+            return random.randint(0, 4)
+        else:
+            return np.argmax(self.brain.predictOne(state))
+
+    def observe(self, sequence):  # in (s, a, r, s_) format
+        self.memory.add(sequence)
+
+    def q_parameter(self):
+        q = self.reward + self.gamma * np.argmax(self.fit_q())
+
+    def predict_q(self, model, state, action):
+        predictor = np.array([np.hstack(np.array([state, action]))])
+        q = model.predict(predictor)
+        return q
+
+    def train_q(self, storage, state, action):
+        train = np.array([storage[:17]])
+        test = np.array([storage[17]])
+        self.model.compile(loss='mse', optimizer=RMSprop(lr=0.025))
+        self.model.fit(train, test, epochs=1)
+
+    def train2_q(self,training, test):
+        training = training.values
+        test = test.values
+        self.model.fit(training, test, epochs=1)
+
+
+    def initialize_dataframe(self):
+        state = [0]*16
+        for i in range(16):
+            state[i]= random.choice([0, 1])
+        move = random.randint(1,4)
+        reward = random.choice([-1, -10, 10])
+        future_state = [0]*16
+        for i in range(16):
+            future_state[i] = random.choice([True, False])
+        Q = 1
+        array = [state, move, reward, future_state, Q]
+        self.dataframe = self.dataframe.append([array])
+
+    def store_memory(self, state, action, q):
+        self.short_memory = np.hstack(np.array([state, action, q]))
+        #print(self.short_memory)
+
+
+
+
+
+'''
+def run():
+    pygame.init()
+    while True:
+        game = Game(400, 400)
+        player1 = game.player
+        food1 = game.food
+        display(player1, food1, game)
+        initial_move(player1,game,food1)
+        display(player1, food1, game)
+        pygame.time.wait(game.speed)
+        while not game.crash:
+            loop(player1, food1, game)
+            display(player1, food1, game)
+            pygame.time.wait(game.speed)
+
+run()
+'''
+
+
+
+

.idea/misc.xml

@@ -0,0 +1,83 @@
+import pygame
+
+pygame.init()
+display_width= 800
+display_height = 800
+
+black = (0,0,0)
+white = (255,255,255)
+red = (255,0,0)
+
+gameDisplay = pygame.display.set_mode((display_width,display_height))
+pygame.display.set_caption('SnakeGen')
+clock = pygame.time.Clock()
+
+img = cv2.imread('racecar.png')
+carImg = pygame.image.load(img)
+
+
+def car(x,y):
+    gameDisplay.blit(carImg,(x,y))
+
+
+
+def text_objects(text, font):
+    textSurface = font.render(text, True, black)
+    return textSurface, textSurface.get_rect()
+
+
+def message_display(text):
+    largeText = pygame.font.Font('freesansbold.ttf',115)
+    TextSurf, TextRect = text_objects(text, largeText)
+    TextRect.center = ((display_width/2),(display_height/2))
+    gameDisplay.blit(TextSurf, TextRect)
+
+def crash():
+    message_display('You Crashed')
+
+
+def game_loop():
+    x = display_width * 0.45
+    y = display_height * 0.5
+
+    x_change = 0
+    y_change = 0
+    gameExit = False
+
+    while not gameExit:
+        for event in pygame.event.get():
+            if event.type == pygame.QUIT:
+                crashed = True
+
+            if event.type == pygame.KEYDOWN:
+                if event.key == pygame.K_LEFT:
+                    x_change = -25
+                elif event.key == pygame.K_RIGHT:
+                    x_change = 25
+                elif event.key == pygame.K_UP:
+                    y_change = - 25
+                elif event.key == pygame.K_DOWN:
+                    y_change = 25
+
+            if event.type == pygame.KEYUP:
+                if event.key == pygame.K_LEFT or event.key == pygame.K_RIGHT or event.key == pygame.K_UP or event.key == pygame.K_DOWN:
+                    x_change = 0
+                    y_change = 0
+            print(event)
+        x = x + x_change
+        y = y + y_change
+        gameDisplay.fill(white)
+        car(x,y)
+
+        if x < 0 or x > display_width:
+            gameExit = True
+
+        if y == 50:
+            crash()
+
+        pygame.display.update()
+        clock.tick(60)
+
+game_loop()
+pygame.quit()
+quit()

.idea/modules.xml

@@ -0,0 +1,103 @@
+import snakeGenetic
+import keras
+import pygame
+from keras.models import Sequential
+from keras.layers.core import Flatten, Dense, Dropout
+from keras.models import load_model
+import pandas as pd
+import numpy as np
+
+modello = load_model('snake8.h5')
+
+df = pd.DataFrame()
+t=0
+while t<50:
+    food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = snakeGenetic.initialize()
+    horizontal=1
+    vertical=0
+    if x_food < x:
+        right_food = 0
+        left_food = 1
+    else:
+        right_food = 1
+        left_food = 0
+    if y_food < y:
+        up_food = 1
+        down_food = 0
+    else:
+        up_food = 0
+        down_food = 1
+    if [(x + 20), y] not in position and (x + 20) < snakeGenetic.display_width:
+        right_free = 1
+    else:
+        right_free = 0
+    if [(x - 20), y] not in position and (x - 20) > 0:
+        left_free = 1
+    else:
+        left_free = 0
+    if [x, (y - 20)] not in position and (y - 20) > 0:
+        up_free = 1
+    else:
+        up_free = 0
+    if [x, (y + 20)] not in position and (y + 20) < snakeGenetic.display_height:
+        down_free = 1
+    else:
+        down_free = 0
+    food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = snakeGenetic.loop_move(
+        food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, 1)
+
+    array2 = [right_food, left_food, up_food, down_food, right_free, left_free, up_free, down_free, vertical, horizontal]
+    move=1
+    while gameExit == False:
+        move = [move]
+        array1 = np.concatenate([array2,move], axis=0)
+        if x_food < x:
+            right_food = 0
+            left_food = 1
+        else:
+            right_food = 1
+            left_food = 0
+        if y_food < y:
+            up_food = 1
+            down_food = 0
+        else:
+            up_food = 0
+            down_food = 1
+        if [(x + 20), y] not in position and (x + 20) < snakeGenetic.display_width:
+            right_free = 1
+        else:
+            right_free = 0
+        if [(x - 20), y] not in position and (x - 20) > 0:
+            left_free = 1
+        else:
+            left_free = 0
+        if [x, (y - 20)] not in position and (y - 20) > 0:
+            up_free = 1
+        else:
+            up_free = 0
+        if [x, (y + 20)] not in position and (y + 20) < snakeGenetic.display_height:
+            down_free = 1
+        else:
+            down_free = 0
+
+
+        array2 = [right_food, left_food, up_food, down_food, right_free, left_free, up_free, down_free, vertical, horizontal]
+
+        array = np.concatenate([array1,array2], axis = 0)
+        array = np.array([array])
+        pred = modello.predict(array)
+        pos = np.where(np.amax(pred[0])==pred[0])
+        print(pred)
+        move_attempt =1+ pos[0][0]
+        print(move_attempt)
+        food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = snakeGenetic.loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, move_attempt)
+        if x_change == 20:
+            move = 1
+        elif x_change == -20:
+            move = 2
+        elif y_change == -20:
+            move = 3
+        elif y_change == 20:
+            move = 4
+
+    t=t+1

.idea/workspace.xml

@@ -0,0 +1,45 @@
+import keras
+from keras.models import Sequential,Model
+from keras.layers.core import Flatten, Dense, Dropout
+import numpy as np
+import pandas as pd
+
+pd.set_option('display.max_columns', 500)
+
+traintest=pd.read_csv('snakemove6.csv')
+x=traintest.drop(['3.1'], axis=1)
+#x=x.iloc[1:,:]
+y=traintest['3.1']
+
+def network(weights_path=None):
+    model = Sequential()
+    model.add(Dense(21,input_dim = 21,activation='relu'))
+    model.add(Dense(40, activation='relu'))
+    model.add(Dense(4, activation='softmax'))
+
+    if weights_path:
+        model.load_weights(weights_path)
+
+    return model
+
+def fitness(x_food, x, y_food, y, food):
+    score = (1-(np.sqrt((x_food-x)**2 + (y_food - y)**2))/566 + food)
+    return score
+
+model = network()
+
+
+x=x.values
+y=y-1
+y_onehot = keras.utils.to_categorical(y, num_classes=4)
+#print(y_onehot[:])
+lr_init = 0.0001
+optimizer = keras.optimizers.Adam(lr= lr_init, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
+model.compile(loss= 'sparse_categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
+model.fit(x,y, epochs=20 ,validation_split=0.2)
+model.save('snake8.h5')
+
+#snake5.h5 = 408 variables
+#snake6.h5 = 8 variables (big architecture)
+#snake7.h5 = 8 variables (small architecture)
+#snake8.h5 = 17 variables (small architecture)

DNQ_internet.py

@@ -0,0 +1,217 @@
+import pygame
+import cv2
+from random import randint
+import pandas as pd
+
+
+pygame.init()
+display_width = 400
+display_height = 400
+
+black = (0, 0, 0)
+white = (255, 255, 255)
+
+position = []
+
+gameDisplay = pygame.display.set_mode((display_width,display_height))
+pygame.display.set_caption('SnakeGen')
+clock = pygame.time.Clock()
+
+img = cv2.imread('ball.png')
+img = cv2.resize(img, (20, 20))
+cv2.imwrite('ball_sized.png',img)
+carImg = pygame.image.load('ball_sized.png')
+img = cv2.imread('pizza.png')
+img = cv2.resize(img, (20, 20))
+cv2.imwrite('pizza_sized.png',img)
+pizzaImg = pygame.image.load('pizza_sized.png')
+
+
+def car(x,y):
+    gameDisplay.blit(carImg,(x, y))
+
+
+def text_objects(text, font):
+    textSurface = font.render(text, True, black)
+    return textSurface, textSurface.get_rect()
+
+
+def message_display(text):
+    largeText = pygame.font.Font('freesansbold.ttf',30)
+    TextSurf, TextRect = text_objects(text, largeText)
+    TextRect.center = ((display_width/2),(display_height/2))
+    gameDisplay.blit(TextSurf, TextRect)
+    pygame.display.update()
+    pygame.time.wait(1000)
+
+
+def crash():
+    message_display('OPS SEI MORTO!')
+
+def food_coord():
+    x_rand = randint(20, display_width - 20)
+    x_rand = x_rand - x_rand % 20
+    y_rand = randint(20, display_width - 20)
+    y_rand = y_rand - y_rand % 20
+    return x_rand, y_rand
+
+def display_food(x,y):
+    gameDisplay.blit(pizzaImg, (x, y))
+
+def initialize():
+    position=[]
+    x = 0.45 * display_width
+    y = 0.5 * display_height
+    x = x - x % 20
+    y = y - y % 20
+
+
+    position.append([x,y])
+
+    food_coord()
+    x_food = food_coord()[0]
+    y_food = food_coord()[1]
+    return 1, False, True, False, False, 0, position, x, y, x_food, y_food, -20 ,0 , 1
+
+def loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change,move):
+
+    for event in pygame.event.get():
+        if event.type == pygame.KEYDOWN:
+            if event.key == pygame.K_LEFT and horizontal == False:
+                x_change = -20
+                y_change = 0
+                vertical = False
+                horizontal = True
+                move=1
+            elif event.key == pygame.K_RIGHT and horizontal == False:
+                x_change = 20
+                y_change = 0
+                vertical = False
+                horizontal = True
+                move=2
+            elif event.key == pygame.K_UP and vertical == False:
+                y_change = - 20
+                x_change = 0
+                vertical = True
+                horizontal = False
+                move=3
+            elif event.key == pygame.K_DOWN and vertical == False:
+                y_change = 20
+                x_change = 0
+                vertical = True
+                horizontal = False
+                move=4
+
+    x = x + x_change
+    y = y + y_change
+    if (position[-1][0] != x or position[-1][1] != y):
+        if [x,y] in position:
+            crash()
+            gameExit = True
+
+        if food>1:
+            for i in range(0,food-1):
+                position[i][0]= position[i+1][0]
+                position[i][1] = position[i+1][1]
+        position[-1][0] = x
+        position[-1][1] = y
+
+    if eaten == True:
+        food_coord()
+        x_food = food_coord()[0]
+        y_food = food_coord()[1]
+        eaten = False
+
+    if x == x_food and y == y_food:
+        eaten= True
+        food = food + 1
+        position.append([x,y])
+
+    gameDisplay.fill(white)
+    display_food(x_food, y_food)
+
+    for i in range(food):
+        print(len(position))
+        x_temp=position[len(position)-1-i][0]
+        y_temp=position[len(position)-1-i][1]
+        car(x_temp, y_temp)
+    pygame.time.wait(100)
+    pygame.display.update()
+
+
+    if x < 0 or x > display_width-20 or y<0 or y > display_height-20:
+        crash()
+        gameExit = True
+
+    clock.tick(100)
+    return food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change,move
+
+
+
+dataset=pd.DataFrame()
+t=0
+while t <10:
+    food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change,move = initialize()
+    while gameExit==False:
+        #print(position)
+        print('1',food)
+        print('2',gameExit)
+        print('3',horizontal)
+        print('4',vertical)
+        print('5',eaten)
+        print('6',cont)
+        print('7',position)
+        print('8',x)
+        print('9',y)
+        print('10',x_food)
+        print('11',x_change)
+        print('12',y_change)
+
+        food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change,move= loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change,move)
+
+        if x_food < x:
+            right_food = 0
+            left_food = 1
+        else:
+            right_food = 1
+            left_food = 0
+        if y_food < y:
+            up_food = 1
+            down_food = 0
+        else:
+            up_food = 0
+            down_food = 1
+        if [(x+20),y] not in position and (x+20)<display_width:
+            right_free = 1
+        else:
+            right_free = 0
+        if [(x-20),y] not in position and (x-20)>0:
+            left_free = 1
+        else:
+            left_free=0
+        if [x,(y-20)] not in position and (y-20)>0:
+            up_free = 1
+        else:
+            up_free = 0
+        if [x,(y+20)] not in position and (y+20)<display_height:
+            down_free = 1
+        else:
+            down_free=0
+        array = [right_food,left_food,up_food,down_food,right_free,left_free,up_free,down_free,move]
+        print(array)
+
+        dataset = dataset.append([array])
+    t=t+1
+
+colnames=['right_food','left_food','up_food','down_food','right_free','left_free','up_free','down_free','move']
+dataset.to_csv(path_or_buf='snakemove2.csv',sep = ',',index=False,header=colnames)
+
+pygame.quit()
+quit()
+
+'''
+#Execute game
+game_loop()
+pygame.quit()
+quit()
+'''

DQN.py

@@ -0,0 +1,335 @@
+import pygame
+from random import randint
+from DQN import DQNAgent
+import numpy as np
+import copy
+
+
+class Game:
+
+    def __init__(self, display_width, display_height):
+        pygame.display.set_caption('SnakeGen')
+        self.display_width = display_width
+        self.display_height = display_height
+        self.gameDisplay = pygame.display.set_mode((display_width, display_height))
+        self.crash = False
+        self.player = Player(self)
+        self.food = Food(self, self.player)
+        self.speed = 1
+
+
+class Player(object):
+
+    def __init__(self, game):
+        x = 0.45 * game.display_width
+        y = 0.5 * game.display_height
+        self.x = x - x % 20
+        self.y = y - y % 20
+        self.position = []
+        self.position.append([self.x, self.y])
+        self.food = 1
+        self.eaten = False
+        self.image = pygame.image.load('snakeBody.png')
+        self.x_change = 20
+        self.y_change = 0
+
+    def update_position(self, x, y,agent):
+        if self.position[-1][0] != x or self.position[-1][1] != y:
+            if self.food > 1:
+                for i in range(0, self.food - 1):
+                    self.position[i][0], self.position[i][1] = self.position[i + 1]
+            self.position[-1][0] = x
+            self.position[-1][1] = y
+
+    def do_move(self, move, x, y, game, food,agent):
+        move_array = [self.x_change, self.y_change]
+
+        if self.eaten:
+            self.position.append([self.x, self.y])
+            self.eaten = False
+            self.food = self.food + 1
+        if move == 0 and self.x_change == 0:  # left
+            move_array = [-20, 0]
+        elif move == 1 and self.x_change == 0:  # right
+            move_array = [20, 0]
+        elif move == 2 and self.y_change == 0:  # top
+            move_array = [0, -20]
+        elif move == 3 and self.y_change == 0:  # bottom
+            move_array = [0, 20]
+        self.x_change, self.y_change = move_array
+        self.x = x + self.x_change
+        self.y = y + self.y_change
+
+        #print(self.x_change, self.y_change, self.x, self.y, self.position)
+        if self.x < 0 or self.x == game.display_width or self.y < 0 or self.y == game.display_height or [self.x, self.y] in self.position:
+            game.crash = True
+        eat(self, food, game)
+
+        self.update_position(self.x, self.y,agent)
+
+
+
+
+    def display_player(self, x, y, food, game, player):
+        self.position[-1][0] = x
+        self.position[-1][1] = y
+
+        for i in range(food):
+            x_temp, y_temp = self.position[len(self.position) - 1 - i]
+            game.gameDisplay.blit(player.image, (x_temp, y_temp))
+        update_screen()
+
+
+class Food(object):
+
+    def __init__(self, game, player):
+        #self.x_food, self.y_food = self.food_coord(game, player)
+        self.x_food = 240
+        self.y_food = 200
+        self.image = pygame.image.load('food.png')
+
+    def food_coord(self, game, player):
+        x_rand = randint(20, game.display_width - 20)
+        self.x_food = x_rand - x_rand % 20
+        y_rand = randint(20, game.display_height - 20)
+        self.y_food = y_rand - y_rand % 20
+        if [self.x_food, self.y_food] not in player.position:
+            return self.x_food, self.y_food
+        else:
+            self.food_coord(game,player)
+
+    def display_food(self, x, y, game):
+        game.gameDisplay.blit(self.image, (x, y))
+        update_screen()
+
+
+def eat(player, food, game):
+    if player.x == food.x_food and player.y == food.y_food:
+        food.food_coord(game, player)
+        player.eaten = True
+
+
+def display(player, food, game):
+    game.gameDisplay.fill((255, 255, 255))
+    player.display_player(player.position[-1][0], player.position[-1][1], player.food, game, player)
+    food.display_food(food.x_food, food.y_food, game)
+
+
+def update_screen():
+    pygame.display.update()
+
+def initial_move(player, game, food,agent):
+    player.do_move(1, player.x, player.y, game, food,agent)
+
+# def loop(player, food, game,agent):
+#     move = 0
+#     if food.x_food < player.x and [(player.x - 20), player.y] not in player.position and (player.x - 20) > 0:
+#         move = 0
+#
+#     elif food.x_food > player.x and [(player.x + 20), player.y] not in player.position and (
+#             player.x + 20) < game.display_width:
+#         move=1
+#
+#     elif food.y_food < player.y and [player.x, (player.y - 20)] not in player.position and (player.y - 20) > 0:
+#         move =2
+#
+#     elif food.y_food > player.y and [player.x, (player.y + 20)] not in player.position and (
+#             player.y + 20) < game.display_height:
+#         move = 3
+#
+#     elif [(player.x - 20), player.y] not in player.position and player.x_change == 0 and (player.x - 20) > 0:
+#         move = 0
+#
+#     elif [(player.x + 20), player.y] not in player.position and player.x_change == 0 and (
+#             player.x + 20) < game.display_width:
+#         move = 1
+#
+#     elif [player.x, (player.y - 20)] not in player.position and player.y_change == 0 and (player.y - 20) > 0:
+#         move = 2
+#
+#     elif [player.x, (player.y + 20)] not in player.position and player.y_change == 0 and (
+#             player.y + 20) < game.display_height:
+#         move = 3
+#
+#     else:
+#         move = randint(1, 4)
+#
+#     player.do_move(move, player.x, player.y, game, food,agent)
+
+
+def run():
+pygame.init()
+agent = DQNAgent()
+counter_games = 0
+while counter_games < 10:
+    #Initialize game
+    game = Game(400, 400)
+    player1 = game.player
+    food1 = game.food
+    #agent.reward = 0
+    #Initialize storage to train first network
+    state1 = agent.get_state(game,player1,food1)
+    action = 1
+    agent.store_memory(state1, action, 1)
+    agent.dataframe = agent.dataframe.append([np.hstack(np.array([state1, action, 1]))])
+    # agent.dataframe = agent.dataframe.append([np.hstack(np.array([state1, 2, 3]))])
+    # agent.dataframe = agent.dataframe.append([np.hstack(np.array([state1, 3, 5]))])
+    agent.train2_q(agent.dataframe[agent.dataframe.columns[:17]], agent.dataframe[agent.dataframe.columns[17]])
+    #Performn first move
+    display(player1, food1, game)
+    initial_move(player1, game, food1,agent)
+    display(player1, food1, game)
+    while not game.crash:
+        state2 = agent.get_state(game,player1,food1)
+        agent.actual = copy.deepcopy([player1.position, player1.x, player1.y, player1.x_change, player1.y_change, food1.x_food,
+                   food1.y_food,game.crash, player1.eaten, player1.food])
+        primary_q_array = []
+        if randint(0,10)<agent.epsilon:
+            final_move=randint(0,3)
+        else:
+            for i in agent.possible_moves(player1):
+                player1.eaten = False
+                player1.do_move(i, player1.x, player1.y, game, food1,agent)
+                if game.crash==True:
+                    primary_q_array.append(agent.set_reward(game,player1, food1))
+                else:
+                    primary_q = agent.predict_q(agent.model, state2, i)
+                    primary_q_array.append(primary_q)
+                agent.replay(game,player1,food1,agent.actual)
+            predicted_q =max(primary_q_array)
+            max_index = primary_q_array.index(predicted_q)
+            final_move = agent.possible_moves(player1)[max_index]
+        player1.do_move(final_move, player1.x, player1.y, game, food1,agent)
+        temp_reward = agent.reward + agent.set_reward(game, player1, food1)
+        print('REWARD: ', temp_reward)
+        temp_state = agent.get_state(game, player1, food1)
+        agent.actual = copy.deepcopy([player1.position, player1.x, player1.y, player1.x_change, player1.y_change, food1.x_food,
+                   food1.y_food,game.crash, player1.eaten, player1.food])
+        secondary_q_array = []
+
+        if not game.crash:
+            for j in agent.possible_moves(player1):
+                player1.eaten = False
+                player1.do_move(j, player1.x, player1.y, game, food1, agent)
+
+                if game.crash == True:
+                    secondary_q_array.append(agent.reward)
+                    print('CRASH', secondary_q_array)
+
+                else:
+
+                    secondary_q = agent.predict_q(agent.model, temp_state, j)
+                    secondary_q_array.append(secondary_q)
+                agent.replay(game, player1, food1, agent.actual)
+            max_target_q = max(secondary_q_array)
+            target_q = [temp_reward + agent.gamma * max_target_q]
+            print('1', target_q)
+
+        else:
+            target_q = temp_reward
+            print('2', target_q)
+        #agent.agent_target = target_q
+        #agent.agent_predict = predicted_q
+        #agent.store_memory(state2, final_move, target_q[0])
+        print('3', state2)
+        print('3', final_move)
+        print('3', target_q[0])
+        print('3', [np.hstack(np.array([state2, final_move, target_q[0][0]]))])
+        agent.dataframe = agent.dataframe.append([np.hstack(np.array([state2, final_move, target_q[0][0]]))])
+        agent.train2_q(agent.dataframe[agent.dataframe.columns[:17]], agent.dataframe[agent.dataframe.columns[17]])
+        display(player1, food1, game)
+        pygame.time.wait(game.speed)
+    print('GAME: ',counter_games)
+    counter_games = counter_games + 1
+
+agent.model.save_weights('weights.hdf5')
+
+
+
+# pygame.init()
+# agent = DQNAgent()
+# counter_games = 0
+# while counter_games < 10:
+#     #Initialize game
+#     game = Game(400, 400)
+#     player1 = game.player
+#     food1 = game.food
+#     agent.reward = 0
+#     #Initialize storage to train first network
+#     state1 = agent.get_state(game,player1,food1)
+#     action = 1
+#     agent.store_memory(state1, action, 1)
+#     agent.dataframe = agent.dataframe.append([np.hstack(np.array([state1, action, 1]))])
+#     # agent.dataframe = agent.dataframe.append([np.hstack(np.array([state1, 2, 3]))])
+#     # agent.dataframe = agent.dataframe.append([np.hstack(np.array([state1, 3, 5]))])
+#     agent.train2_q(agent.dataframe[agent.dataframe.columns[:17]], agent.dataframe[agent.dataframe.columns[17]])
+#     #Performn first move
+#     display(player1, food1, game)
+#     initial_move(player1, game, food1,agent)
+#     display(player1, food1, game)
+#     while not game.crash:
+#         state2 = agent.get_state(game,player1,food1)
+#         agent.actual = copy.deepcopy([player1.position, player1.x, player1.y, player1.x_change, player1.y_change, food1.x_food,
+#                    food1.y_food,game.crash, player1.eaten, player1.food])
+#         primary_q_array = []
+#         if randint(0,10)<agent.epsilon:
+#             final_move=randint(0,3)
+#         else:
+#             for i in agent.possible_moves(player1):
+#                 player1.eaten = False
+#                 player1.do_move(i, player1.x, player1.y, game, food1,agent)
+#                 if game.crash==True:
+#                     primary_q_array.append(agent.set_reward(game,player1, food1))
+#                 else:
+#                     primary_q = agent.predict_q(agent.model, state2, i)
+#                     primary_q_array.append(primary_q)
+#                 agent.replay(game,player1,food1,agent.actual)
+#             predicted_q =max(primary_q_array)
+#             max_index = primary_q_array.index(predicted_q)
+#             final_move = agent.possible_moves(player1)[max_index]
+#         player1.do_move(final_move, player1.x, player1.y, game, food1,agent)
+#         temp_reward = agent.reward + agent.set_reward(game, player1, food1)
+#         print('REWARD: ', temp_reward)
+#         temp_state = agent.get_state(game, player1, food1)
+#         agent.actual = copy.deepcopy([player1.position, player1.x, player1.y, player1.x_change, player1.y_change, food1.x_food,
+#                    food1.y_food,game.crash, player1.eaten, player1.food])
+#         secondary_q_array = []
+#
+#         if not game.crash:
+#             for j in agent.possible_moves(player1):
+#                 player1.eaten = False
+#                 player1.do_move(j, player1.x, player1.y, game, food1, agent)
+#
+#                 if game.crash == True:
+#                     secondary_q_array.append(agent.reward)
+#                     print('CRASH', secondary_q_array)
+#
+#                 else:
+#
+#                     secondary_q = agent.predict_q(agent.model, temp_state, j)
+#                     secondary_q_array.append(secondary_q)
+#                 agent.replay(game, player1, food1, agent.actual)
+#             max_target_q = max(secondary_q_array)
+#             target_q = [temp_reward + agent.gamma * max_target_q]
+#             print('1', target_q)
+#
+#         else:
+#             target_q = temp_reward
+#             print('2', target_q)
+#         #agent.agent_target = target_q
+#         #agent.agent_predict = predicted_q
+#         #agent.store_memory(state2, final_move, target_q[0])
+#         print('3', state2)
+#         print('3', final_move)
+#         print('3', target_q[0])
+#         print('3', [np.hstack(np.array([state2, final_move, target_q[0][0]]))])
+#         agent.dataframe = agent.dataframe.append([np.hstack(np.array([state2, final_move, target_q[0][0]]))])
+#         agent.train2_q(agent.dataframe[agent.dataframe.columns[:17]], agent.dataframe[agent.dataframe.columns[17]])
+#         display(player1, food1, game)
+#         pygame.time.wait(game.speed)
+#     print('GAME: ',counter_games)
+#     counter_games = counter_games + 1
+#
+# agent.model.save_weights('weights.hdf5')
+
+run()

__pycache__/DQN.cpython-36.pyc

@@ -0,0 +1,243 @@
+import pygame
+import cv2
+from random import randint
+import csv
+import numpy as np
+import pandas as pd
+
+
+pygame.init()
+display_width= 400
+display_height = 400
+
+black = (0,0,0)
+white = (255,255,255)
+
+position = []
+
+gameDisplay = pygame.display.set_mode((display_width,display_height))
+pygame.display.set_caption('SnakeGen')
+clock = pygame.time.Clock()
+
+
+carImg = pygame.image.load('ball_sized2.png')
+pizzaImg = pygame.image.load('pizza_sized.png')
+
+
+def car(x,y):
+    gameDisplay.blit(carImg,(x,y))
+
+
+def text_objects(text, font):
+    textSurface = font.render(text, True, black)
+    return textSurface, textSurface.get_rect()
+
+
+def message_display(text):
+    largeText = pygame.font.Font('freesansbold.ttf',30)
+    TextSurf, TextRect = text_objects(text, largeText)
+    TextRect.center = ((display_width/2),(display_height/2))
+    gameDisplay.blit(TextSurf, TextRect)
+    #pygame.display.update()
+    #pygame.time.wait(100)
+
+
+def crash():
+    message_display('OPS SEI MORTO!')
+
+def food_coord():
+    x_rand = randint(20, display_width - 20)
+    x_rand = x_rand - x_rand % 20
+    y_rand = randint(20, display_width - 20)
+    y_rand = y_rand - y_rand % 20
+    return x_rand, y_rand
+
+def display_food(x,y):
+    gameDisplay.blit(pizzaImg, (x, y))
+
+
+def initialize():
+    position=[]
+    x = 0.45 * display_width
+    y = 0.5 * display_height
+    x = x - x % 20
+    y = y - y % 20
+
+    position.append([x,y])
+
+    food_coord()
+    x_food = food_coord()[0]
+    y_food = food_coord()[1]
+    return 1, False, 0, 0, False, 0, position, x, y, x_food, y_food, 0 ,0
+    '''
+    food = 1
+
+    x_change = 0
+    y_change = 0
+    gameExit = False
+    horizontal = False
+    vertical = False
+    eaten = True
+    cont = 0
+    '''
+
+def loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change,move):
+    if move == 1 and horizontal == 0:       #left
+        x_change = -20
+        y_change = 0
+        vertical = 0
+        horizontal = 1
+    elif move == 2 and horizontal == 0:        #right
+        x_change = 20
+        y_change = 0
+        vertical = 0
+        horizontal = 1
+    elif move == 3 and vertical == 0:           #top
+        y_change = - 20
+        x_change = 0
+        vertical = 1
+        horizontal = 0
+    elif move == 4 and vertical == 0:           #bottom
+        y_change = 20
+        x_change = 0
+        vertical = 1
+        horizontal = 0
+
+    x = x + x_change
+    y = y + y_change
+    if (position[-1][0] != x or position[-1][1] != y):
+        if [x,y] in position:
+            crash()
+            pygame.time.wait(100)
+            gameExit = True
+
+        if food>1:
+            for i in range(0,food-1):
+                position[i][0]= position[i+1][0]
+                position[i][1] = position[i+1][1]
+        position[-1][0] = x
+        position[-1][1] = y
+
+    if eaten == True:
+        food_coord()
+        x_food = food_coord()[0]
+        y_food = food_coord()[1]
+        eaten = False
+
+    if x == x_food and y == y_food:
+        eaten= True
+        food = food + 1
+        position.append([x,y])
+
+    gameDisplay.fill(white)
+    display_food(x_food, y_food)
+
+    for i in range(food):
+        #print(len(position))
+        x_temp=position[len(position)-1-i][0]
+        y_temp=position[len(position)-1-i][1]
+        car(x_temp, y_temp)
+    pygame.time.wait(100)
+    pygame.display.update()
+
+
+    if x < 0 or x > display_width-20 or y<0 or y > display_height-20:
+        crash()
+        gameExit = True
+
+    clock.tick(1000)
+    cont = cont+1
+
+    return food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change
+
+def fitness(x_food, x, y_food, y, food):
+    score = (1-(np.sqrt((x_food-x)**2 + (y_food - y)**2))/566 + food)
+    return score
+
+
+#Execute game
+#food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = initialize()
+#l r t b
+'''
+dataset=pd.DataFrame()
+t=1
+while(t<100):
+    gameExit=False
+
+    food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = initialize()
+
+    while gameExit==False:
+        #print(position)
+        if x_food < x and [(x-20),y] not in position and (x-20)>0:
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change,1)
+            move=1
+        elif x_food > x and [(x+20),y] not in position and (x+20)<display_width:
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, 2)
+            move=2
+        elif y_food < y and [x,(y-20)] not in position and (y-20)>0:
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, 3)
+            move=3
+        elif y_food > y and [x,(y+20)] not in position and (y+20) < display_height:
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, 4)
+            move=4
+        elif [(x-20),y] not in position and horizontal==False and (x-20)>0:
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, 1)
+            move=1
+        elif [(x+20),y] not in position and horizontal==False and (x+20)<display_width:
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, 2)
+            move=2
+        elif [x,(y-20)] not in position and vertical==False and (y-20)>0:
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, 3)
+            move=3
+        elif [x,(y+20)] not in position and vertical==False and (y+20) < display_height:
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, 4)
+            move=4
+        else:
+            move = randint(1,4)
+            food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change = loop_move(food, gameExit, horizontal, vertical, eaten, cont, position, x, y, x_food, y_food, x_change, y_change, move)
+
+
+        if x_food < x:
+            right_food = 0
+            left_food = 1
+        else:
+            right_food = 1
+            left_food = 0
+        if y_food < y:
+            up_food = 1
+            down_food = 0
+        else:
+            up_food = 0
+            down_food = 1
+        if [(x+20),y] not in position and (x+20)<display_width:
+            right_free = 1
+        else:
+            right_free = 0
+        if [(x-20),y] not in position and (x-20)>0:
+            left_free = 1
+        else:
+            left_free=0
+        if [x,(y-20)] not in position and (y-20)>0:
+            up_free = 1
+        else:
+            up_free = 0
+        if [x,(y+20)] not in position and (y+20)<display_height:
+            down_free = 1
+        else:
+            down_free=0
+
+
+
+        array = [right_food,left_food,up_food,down_food,right_free,left_free,up_free,down_free, vertical, horizontal, move]
+        #for i in range(len(grid)):
+            #array.append(grid[i])
+        dataset = dataset.append([array])
+
+    print(t)
+    t=t+1
+
+dataset.to_csv(path_or_buf='snakemove6.csv',sep = ',',index=False, header=False)
+
+pygame.quit()
+quit()
+'''

__pycache__/snake.cpython-36.pyc

@@ -0,0 +1,53 @@
+import pygame
+from keras.models import Sequential,Model
+from keras.layers.core import Flatten, Dense, Dropout
+from keras.optimizers import RMSprop
+import random
+import numpy as np
+import pandas as pd
+from DQN import DQNAgent
+from snakeClass import *
+import copy
+
+class Agent(object):
+    def __init__(self):
+        self.actual = []
+
+class Play(object):
+
+    def __init__(self):
+        self.x = 0.45 * 400
+        self.y = 0.5 * 400
+        self.position = []
+        self.position.append([self.x, self.y])
+        self.x_change = 20
+        self.y_change = 0
+
+    def do_move(self, move, x, y):
+        move_array = [self.x_change, self.y_change]
+
+        if move == 0 and self.x_change == 0:  # left
+            move_array = [-20, 0]
+        elif move == 1 and self.x_change == 0:  # right
+            move_array = [20, 0]
+        self.x_change, self.y_change = move_array
+        self.x = x + self.x_change
+        self.y = y + self.y_change
+
+        self.update_position(self.x, self.y)
+
+    def update_position(self, x, y):
+        self.position[-1][0] = x
+        self.position[-1][1] = y
+
+
+def run():
+    agent = Agent()
+    player1 = Play()
+    agent.actual = copy.deepcopy([player1.position, player1.x])
+    print(agent.actual[0])
+    i = 1
+    player1.do_move(i, player1.x, player1.y)
+    print(agent.actual[0])
+
+run()