You need to create a custom module and also you need a .info file change 20 to the number you want

function custom_tweaks_menu_alter(&$menu) {
  if (isset($menu['nodereference/autocomplete'])) {
    $menu['nodereference/autocomplete']['page callback'] = 'custom_tweaks_nodereference_autocomplete';
  }
}

function custom_tweaks_nodereference_autocomplete($field_name, $string = '') {
  $fields = content_fields();
  $field = $fields[$field_name];
  $match = isset($field['widget']['autocomplete_match']) ? $field['widget']['autocomplete_match'] : 'contains';
  $matches = array();

  $references = _nodereference_potential_references($field, $string, $match, array(), 20);
  foreach ($references as $id => $row) {
    // Add a class wrapper for a few required CSS overrides.
    $matches[$row['title'] ." [nid:$id]"] = '<div class="reference-autocomplete">'. $row['rendered'] . '</div>';
  }
  drupal_json($matches);
}

Taken from this source