I'm having two problems with my ValueError in my number game.

import random

def game():
    # generate a random number between 1 and 10
    secret_num = random.randint(1, 10)
    # limit guesses
    guesses = []

    while len(guesses) < 5:
        # safely make an int
        try:
            # get a number guess from the player
            guess = int(input("Guess a number between 1 and 10: "))
        # print hit/miss
        except ValueError:
            print("{} isn't a number!".format(guess))
        else:
            # compare guess to secret number
            if guess == secret_num:
                print("You got it! My secret number was {}.".format(secret_num))
                break
            # too high message
            elif guess < secret_num:
                print("My number is higher than {}!".format(guess))
            # too low message
            else:
                print("My number is lower than {}!".format(guess))
            guesses.append(guess)
    else:
        print("You didn't get it! My number was {}.".format(secret_num))
    # play again
    play_again = input("Do you want to play again? Y/n ")
    if play_again.lower() != 'n':
        game()
    else:
        print("BYE.")
game()