I have even more items in the INBOX of one IMAP account but never faced the issues you described. Maybe my configuration (parts of it) helps:

  (setq gnus-select-method '(nnnil nil))
  (setq gnus-secondary-select-methods
      '((nnimap "account1"
                (nnimap-address "mail.account1")
                (nnimap-server-port "imaps")
                (nnimap-stream ssl)
                (nnir-search-engine imap)
                (nnmail-expiry-target "nnimap+account1:Trash")
                (nnmail-expiry-wait immediate))
        (nnimap "account2"
                (nnimap-address "imap.account2")
                (nnimap-server-port "imaps")
                (nnimap-stream ssl)
                (nnir-search-engine imap)
                (nnmail-expiry-target "nnimap+account2:Trash")
                (nnmail-expiry-wait immediate))
        (nndiary "diary")))