Edit: Not needed lol.

echasnovski's response guides to a better approach. Post will remain up just in case someone else can't search the docs like me.

Not an actual plugin, but I thought it might be useful to someone.

It will list the inactive plugins in a buffer and the user might modify it to remove any of the list (e.g. for the cases when some plugin is lazy loaded).

Lua is definitely not the language I'm most proficient on, so it probably could be optimized.

~80 LoC snippet below. It follows pretty much what vim.pack does by default on vim.pack.update(), the difference is being able to modify the buffer.

--- @return vim.pack.PlugData[]
local function get_inactive_plugins()
  if not vim.pack or type(vim.pack.get) ~= "function" then
    vim.notify("vim.pack.get is not available", vim.log.levels.ERROR)
    return {}
  end

  local all_plugins = vim.pack.get()
  local inactive = {}

  for _, plugin in ipairs(all_plugins) do
    if not plugin.active then table.insert(inactive, plugin) end
  end

  return inactive
end

local function open_inactive_plugins_buffer()
  local inactive_plugins = get_inactive_plugins()

  if #inactive_plugins == 0 then
    vim.notify("No inactive plugins found")
    return
  end

  local inactive_plugin_names = {}

  for _, inactive_plugin in ipairs(inactive_plugins) do
    table.insert(inactive_plugin_names, inactive_plugin.spec.name)
  end

  local buf = vim.api.nvim_create_buf(false, true)

  vim.api.nvim_buf_set_lines(buf, 0, -1, false, inactive_plugin_names)

  local buf_name = "nvim-pack://inactive" .. buf
  vim.api.nvim_buf_set_name(buf, buf_name)

  local win_id = vim.api.nvim_get_current_win()
  vim.cmd.sbuffer({ buf, mods = { tab = vim.fn.tabpagenr() } })

  local delete_buffer = vim.schedule_wrap(function()
    pcall(vim.api.nvim_win_close, win_id, true)
    pcall(vim.api.nvim_buf_delete, buf, { force = true })

    vim.cmd.redraw()
  end)

  vim.api.nvim_create_autocmd("BufWriteCmd", {
    pattern = buf_name,
    callback = function()
      local to_delete = vim.api.nvim_buf_get_lines(buf, 0, -1, false)
      local deleted_count = 0

      for _, plugin in ipairs(to_delete) do
        if plugin ~= "" then
          vim.pack.del({ plugin })
          deleted_count = deleted_count + 1
        end
      end

      if deleted_count > 0 then
        vim.notify("Finished deleting " .. deleted_count .. " plugins")
      else
        vim.notify("No plugins deleted")
      end

      delete_buffer()
    end,
  })

  vim.bo[buf].modified = false
  vim.bo[buf].modifiable = true
  vim.bo[buf].buftype = "acwrite"
  vim.bo[buf].filetype = "nvim-pack"
end

Just map the open_inactive_plugins_buffer function to some key and that's all.