Slow Query- Performance issue

Getting Performace issue on the query and its internally calling getUnrestrictedNids function.

SQL query:
"{{SELECT tid, rid FROM permissions_by_term_role
UNION
SELECT tid, uid FROM permissions_by_term_user;}}"

Drupal Function:

private function getUnrestrictedNids() {
    $tidsRestrictedUserQuery = $this->database->select('permissions_by_term_user', 'u')
      ->fields('u', ['tid']);

    $restrictedTids = $this->database->select('permissions_by_term_role', 'r')
      ->fields('r', ['tid'])
      ->union($tidsRestrictedUserQuery)
      ->execute()
      ->fetchCol();

    if (empty($restrictedTids)) {
      return $this->getAllNids();
    }

    $restrictedNids = $this->database->select('taxonomy_index', 't')
      ->fields('t', ['nid'])
      ->condition('t.tid', $restrictedTids, 'IN')
      ->distinct(TRUE)
      ->execute()
      ->fetchCol();

    if (empty($restrictedNids)) {
      return $this->getAllNids();
    }

    $unrestrictedNids = $this->database->select('taxonomy_index', 't')
      ->fields('t', ['nid'])
      ->condition('t.nid', $restrictedNids, 'NOT IN')
      ->distinct(TRUE)
      ->execute()
      ->fetchCol();

    return $unrestrictedNids;
  }